Browse Source

HUE-5304 [indexer] Move api2 to solr_api

Romain Rigaux 8 years ago
parent
commit
4a76f84

+ 0 - 184
desktop/libs/indexer/src/indexer/api2.py

@@ -1,184 +0,0 @@
-#!/usr/bin/env python
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import csv
-import json
-import logging
-
-from django.utils.translation import ugettext as _
-
-from desktop.lib.django_util import JsonResponse
-from desktop.lib.exceptions_renderable import PopupException
-from libsolr.api import SolrApi
-from search.conf import SOLR_URL, SECURITY_ENABLED
-
-from indexer.solr_client import SolrClient
-from indexer.utils import get_default_fields
-
-
-LOG = logging.getLogger(__name__)
-
-
-def create_index(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  name = request.POST.get('name')
-
-  if name:
-    searcher = SolrClient(request.user)
-
-    try:
-      collection = searcher.create_index(
-          name,
-          request.POST.get('fields', get_default_fields()),
-          request.POST.get('uniqueKeyField', 'id'),
-          request.POST.get('df', 'text')
-      )
-
-      response['status'] = 0
-      response['collection'] = collection
-      response['message'] = _('Index created!')
-    except Exception, e:
-      response['message'] = _('Index could not be created: %s') % e
-  else:
-    response['message'] = _('Index requires a name field.')
-
-  return JsonResponse(response)
-
-
-def delete_indexes(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  indexes = json.loads(request.POST.get('indexes', '[]'))
-
-  if not indexes:
-    response['message'] = _('No indexes to remove.')
-  else:
-    searcher = SolrClient(request.user)
-
-    for index in indexes:
-      if index['type'] == 'collection':
-        searcher.delete_index(index['name'])
-      elif index['type'] == 'alias':
-        searcher.delete_alias(index['name'])
-      else:
-        LOG.warn('We could not delete: %s' % index)
-
-    response['status'] = 0
-    response['message'] = _('Indexes removed!')
-
-  return JsonResponse(response)
-
-
-def create_or_edit_alias(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  alias = request.POST.get('alias', '')
-  collections = json.loads(request.POST.get('collections', '[]'))
-
-  api = SolrApi(SOLR_URL.get(), request.user, SECURITY_ENABLED.get())
-
-  try:
-    api.create_or_modify_alias(alias, collections)
-    response['status'] = 0
-    response['message'] = _('Alias created or modified!')
-  except Exception, e:
-    response['message'] = _('Alias could not be created or modified: %s') % e
-
-  return JsonResponse(response)
-
-
-def create_wizard_get_sample(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  wizard = json.loads(request.POST.get('wizard', '{}'))
-
-  f = request.fs.open(wizard['path'])
-
-  response['status'] = 0
-  response['data'] = _read_csv(f)
-
-  return JsonResponse(response)
-
-
-def create_wizard_create(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  wizard = json.loads(request.POST.get('wizard', '{}'))
-
-  f = request.fs.open(wizard['path'])
-
-  response['status'] = 0
-  response['data'] = _read_csv(f)
-
-  return JsonResponse(response)
-
-
-def design_schema(request, index):
-  if request.method == 'POST':
-    pass # TODO: Support POST for update?
-
-  result = {'status': -1, 'message': ''}
-
-  try:
-    searcher = SolrClient(request.user)
-    unique_key, fields = searcher.get_index_schema(index)
-
-    result['status'] = 0
-    formatted_fields = []
-    for field in fields:
-      formatted_fields.append({
-        'name': field,
-        'type': fields[field]['type'],
-        'required': fields[field].get('required', None),
-        'indexed': fields[field].get('indexed', None),
-        'stored': fields[field].get('stored', None),
-        'multivalued': fields[field].get('multivalued', None),
-      })
-    result['fields'] = formatted_fields
-    result['unique_key'] = unique_key
-  except Exception, e:
-    result['message'] = _('Could not get index schema: %s') % e
-
-  return JsonResponse(result)
-
-
-def _read_csv(f):
-  content = f.read(1024 * 1024)
-
-  dialect = csv.Sniffer().sniff(content)
-  lines = content.splitlines()[:5]
-  reader = csv.reader(lines, delimiter=dialect.delimiter)
-  
-  return [row for row in reader]
-
-

+ 157 - 1
desktop/libs/indexer/src/indexer/solr_api.py

@@ -15,16 +15,21 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
+import csv
+import json
 import logging
 import logging
 
 
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_GET, require_POST
 from django.views.decorators.http import require_GET, require_POST
 
 
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.django_util import JsonResponse
+from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.i18n import smart_unicode
-
 from libsolr.api import SolrApi
 from libsolr.api import SolrApi
 
 
+from indexer.solr_client import SolrClient
+from indexer.utils import get_default_fields
+
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
@@ -72,3 +77,154 @@ def delete_collections(request):
   response['status'] = 0
   response['status'] = 0
 
 
   return JsonResponse(response)
   return JsonResponse(response)
+
+
+def create_index(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  name = request.POST.get('name')
+
+  if name:
+    searcher = SolrClient(request.user)
+
+    try:
+      collection = searcher.create_index(
+          name,
+          request.POST.get('fields', get_default_fields()),
+          request.POST.get('uniqueKeyField', 'id'),
+          request.POST.get('df', 'text')
+      )
+
+      response['status'] = 0
+      response['collection'] = collection
+      response['message'] = _('Index created!')
+    except Exception, e:
+      response['message'] = _('Index could not be created: %s') % e
+  else:
+    response['message'] = _('Index requires a name field.')
+
+  return JsonResponse(response)
+
+
+def delete_indexes(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  indexes = json.loads(request.POST.get('indexes', '[]'))
+
+  if not indexes:
+    response['message'] = _('No indexes to remove.')
+  else:
+    searcher = SolrClient(request.user)
+
+    for index in indexes:
+      if index['type'] == 'collection':
+        searcher.delete_index(index['name'])
+      elif index['type'] == 'alias':
+        searcher.delete_alias(index['name'])
+      else:
+        LOG.warn('We could not delete: %s' % index)
+
+    response['status'] = 0
+    response['message'] = _('Indexes removed!')
+
+  return JsonResponse(response)
+
+
+def create_or_edit_alias(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  alias = request.POST.get('alias', '')
+  collections = json.loads(request.POST.get('collections', '[]'))
+
+  api = SolrApi(SOLR_URL.get(), request.user, SECURITY_ENABLED.get())
+
+  try:
+    api.create_or_modify_alias(alias, collections)
+    response['status'] = 0
+    response['message'] = _('Alias created or modified!')
+  except Exception, e:
+    response['message'] = _('Alias could not be created or modified: %s') % e
+
+  return JsonResponse(response)
+
+
+def create_wizard_get_sample(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  wizard = json.loads(request.POST.get('wizard', '{}'))
+
+  f = request.fs.open(wizard['path'])
+
+  response['status'] = 0
+  response['data'] = _read_csv(f)
+
+  return JsonResponse(response)
+
+
+def create_wizard_create(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  wizard = json.loads(request.POST.get('wizard', '{}'))
+
+  f = request.fs.open(wizard['path'])
+
+  response['status'] = 0
+  response['data'] = _read_csv(f)
+
+  return JsonResponse(response)
+
+
+def design_schema(request, index):
+  if request.method == 'POST':
+    pass # TODO: Support POST for update?
+
+  result = {'status': -1, 'message': ''}
+
+  try:
+    searcher = SolrClient(request.user)
+    unique_key, fields = searcher.get_index_schema(index)
+
+    result['status'] = 0
+    formatted_fields = []
+    for field in fields:
+      formatted_fields.append({
+        'name': field,
+        'type': fields[field]['type'],
+        'required': fields[field].get('required', None),
+        'indexed': fields[field].get('indexed', None),
+        'stored': fields[field].get('stored', None),
+        'multivalued': fields[field].get('multivalued', None),
+      })
+    result['fields'] = formatted_fields
+    result['unique_key'] = unique_key
+  except Exception, e:
+    result['message'] = _('Could not get index schema: %s') % e
+
+  return JsonResponse(result)
+
+
+def _read_csv(f):
+  content = f.read(1024 * 1024)
+
+  dialect = csv.Sniffer().sniff(content)
+  lines = content.splitlines()[:5]
+  reader = csv.reader(lines, delimiter=dialect.delimiter)
+  
+  return [row for row in reader]
+
+

+ 5 - 9
desktop/libs/indexer/src/indexer/solr_client.py

@@ -103,14 +103,14 @@ class SolrClient(object):
     Create schema.xml file so that we can set UniqueKey field.
     Create schema.xml file so that we can set UniqueKey field.
     """
     """
     if self.is_solr_cloud_mode():
     if self.is_solr_cloud_mode():
-      self._create_solr_cloud_index(name, fields, unique_key_field, df)
-
-    else:  # Non-solrcloud mode
+      self._create_solr_cloud_index_config_name(name, fields, unique_key_field, df)
+      if not self.api.create_collection(name):
+        raise Exception('Failed to create collection: %s' % name)
+    else:
       self._create_non_solr_cloud_index(name, fields, unique_key_field, df)
       self._create_non_solr_cloud_index(name, fields, unique_key_field, df)
 
 
-    return name
 
 
-  def _create_solr_cloud_index(self, name, fields, unique_key_field, df):
+  def _create_solr_cloud_index_config_name(self, name, fields, unique_key_field, df):
     with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
     with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
       tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
       tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
 
 
@@ -122,16 +122,12 @@ class SolrClient(object):
         if is_enabled():
         if is_enabled():
           with open(os.path.join(config_root_path, 'solrconfig.xml.secure')) as f:
           with open(os.path.join(config_root_path, 'solrconfig.xml.secure')) as f:
             zc.set(os.path.join(root_node, 'conf', 'solrconfig.xml'), f.read())
             zc.set(os.path.join(root_node, 'conf', 'solrconfig.xml'), f.read())
-
-        if not self.api.create_collection(name):
-          raise Exception('Failed to create collection: %s' % name)
       except Exception, e:
       except Exception, e:
         if zc.path_exists(root_node):
         if zc.path_exists(root_node):
           # Remove the root node from Zookeeper
           # Remove the root node from Zookeeper
           zc.delete_path(root_node)
           zc.delete_path(root_node)
         raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
         raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
       finally:
       finally:
-        # Remove tmp config directory
         shutil.rmtree(tmp_path)
         shutil.rmtree(tmp_path)
 
 
 
 

+ 9 - 8
desktop/libs/indexer/src/indexer/urls.py

@@ -30,6 +30,7 @@ urlpatterns = patterns('indexer.views',
   url(r'^importer/prefill/(?P<source_type>[^/]+)/(?P<target_type>[^/]+)/(?P<target_path>[^/]+)?$', 'importer_prefill', name='importer_prefill'),
   url(r'^importer/prefill/(?P<source_type>[^/]+)/(?P<target_type>[^/]+)/(?P<target_path>[^/]+)?$', 'importer_prefill', name='importer_prefill'),
 )
 )
 
 
+# Current v1
 urlpatterns += patterns('indexer.api',
 urlpatterns += patterns('indexer.api',
   url(r'^api/fields/parse/$', 'parse_fields', name='api_parse_fields'),
   url(r'^api/fields/parse/$', 'parse_fields', name='api_parse_fields'),
   url(r'^api/autocomplete/$', 'autocomplete', name='api_autocomplete'),
   url(r'^api/autocomplete/$', 'autocomplete', name='api_autocomplete'),
@@ -43,7 +44,7 @@ urlpatterns += patterns('indexer.api',
 )
 )
 
 
 
 
-urlpatterns += patterns('indexer.api2',
+urlpatterns += patterns('indexer.solr_api',
   # V2
   # V2
   url(r'^api/aliases/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/aliases/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/indexes/create/$', 'create_index', name='create_index'),
   url(r'^api/indexes/create/$', 'create_index', name='create_index'),
@@ -53,17 +54,17 @@ urlpatterns += patterns('indexer.api2',
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
 )
 )
 
 
+urlpatterns += patterns('indexer.solr_api',
+  url(r'^api/collections/list/$', 'list_collections', name='list_collections'),
+  url(r'^api/collections/delete/$', 'delete_collections', name='delete_collections'),
+)
+
+
 urlpatterns += patterns('indexer.api3',
 urlpatterns += patterns('indexer.api3',
-  # V3
+  # Importer
   url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
   url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
   url(r'^api/indexer/index_file/$', 'index_file', name='index_file'),
   url(r'^api/indexer/index_file/$', 'index_file', name='index_file'),
   url(r'^api/indexer/guess_field_types/$', 'guess_field_types', name='guess_field_types'),
   url(r'^api/indexer/guess_field_types/$', 'guess_field_types', name='guess_field_types'),
 
 
   url(r'^api/importer/submit', 'importer_submit', name='importer_submit')
   url(r'^api/importer/submit', 'importer_submit', name='importer_submit')
 )
 )
-
-
-urlpatterns += patterns('indexer.solr_api',
-  url(r'^api/collections/list/$', 'list_collections', name='list_collections'),
-  url(r'^api/collections/delete/$', 'delete_collections', name='delete_collections'),
-)