Преглед на файлове

HUE-5304 [solr] Refactor create a collection to support managed templates

Romain Rigaux преди 8 години
родител
ревизия
e61ae6c

+ 1 - 0
desktop/libs/indexer/src/indexer/controller.py

@@ -263,6 +263,7 @@ class CollectionManagerController(object):
         else:
           raise PopupException(_('Could not update index. Unknown type %s') % data_type)
         fh.close()
+
       if not api.update(collection_or_core_name, data, content_type=content_type):
         raise PopupException(_('Could not update index. Check error logs for more info.'))
     else:

+ 13 - 64
desktop/libs/indexer/src/indexer/solr_api.py

@@ -15,7 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import csv
 import json
 import logging
 
@@ -27,8 +26,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_unicode
 from libsolr.api import SolrApi
 
-from indexer.solr_client import SolrClient
-from indexer.utils import get_default_fields
+from indexer.solr_client import SolrClient, _get_fields
 
 
 LOG = logging.getLogger(__name__)
@@ -79,32 +77,24 @@ def delete_collections(request):
   return JsonResponse(response)
 
 
+@require_POST
+@api_error_handler
 def create_index(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
   response = {'status': -1}
 
   name = request.POST.get('name')
+  fields = json.loads(request.POST.get('fields', '[]'))
 
-  if name:
-    searcher = SolrClient(request.user)
+  client = SolrClient(request.user)
 
-    try:
-      collection = searcher.create_index(
-          name,
-          request.POST.get('fields', get_default_fields()),
-          request.POST.get('uniqueKeyField', 'id'),
-          request.POST.get('df', 'text')
-      )
-
-      response['status'] = 0
-      response['collection'] = collection
-      response['message'] = _('Index created!')
-    except Exception, e:
-      response['message'] = _('Index could not be created: %s') % e
-  else:
-    response['message'] = _('Index requires a name field.')
+  collection = client.create_index(
+      name=name,
+      fields=request.POST.get('fields', _get_fields(name='id', type='string')),
+  )
+
+  response['status'] = 0
+  response['collection'] = collection
+  response['message'] = _('Index created!')
 
   return JsonResponse(response)
 
@@ -157,37 +147,6 @@ def create_or_edit_alias(request):
   return JsonResponse(response)
 
 
-def create_wizard_get_sample(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  wizard = json.loads(request.POST.get('wizard', '{}'))
-
-  f = request.fs.open(wizard['path'])
-
-  response['status'] = 0
-  response['data'] = _read_csv(f)
-
-  return JsonResponse(response)
-
-
-def create_wizard_create(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  wizard = json.loads(request.POST.get('wizard', '{}'))
-
-  f = request.fs.open(wizard['path'])
-
-  response['status'] = 0
-  response['data'] = _read_csv(f)
-
-  return JsonResponse(response)
-
 
 def design_schema(request, index):
   if request.method == 'POST':
@@ -218,13 +177,3 @@ def design_schema(request, index):
   return JsonResponse(result)
 
 
-def _read_csv(f):
-  content = f.read(1024 * 1024)
-
-  dialect = csv.Sniffer().sniff(content)
-  lines = content.splitlines()[:5]
-  reader = csv.reader(lines, delimiter=dialect.delimiter)
-  
-  return [row for row in reader]
-
-

+ 40 - 13
desktop/libs/indexer/src/indexer/solr_client.py

@@ -46,14 +46,28 @@ ZK_SOLR_CONFIG_NAMESPACE = 'configs'
 IS_SOLR_CLOUD = None
 
 
+DEFAULT_FIELD = {
+  'name': None,
+  'type': 'text',
+  'indexed': True,
+  'stored': True,
+  'required': False,
+  'multiValued': False
+}
+
+
 class SolrClientException(Exception):
   pass
 
 
+def _get_fields(name='id', type='text'):
+  default_field = DEFAULT_FIELD.copy()
+  default_field.update({'name': name, 'type': type})
+  return [default_field]
+
+
 class SolrClient(object):
-  """
-  Glue the models to the views.
-  """
+
   def __init__(self, user):
     self.user = user
     self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
@@ -61,9 +75,10 @@ class SolrClient(object):
 
   def is_solr_cloud_mode(self):
     global IS_SOLR_CLOUD
+
     if IS_SOLR_CLOUD is None:
-      print self.api.info_system()
       IS_SOLR_CLOUD = self.api.info_system().get('mode', 'solrcloud') == 'solrcloud'
+
     return IS_SOLR_CLOUD
 
 
@@ -76,9 +91,10 @@ class SolrClient(object):
         for name in collections:
           indexes.append({'name': name, 'type': 'collection', 'collections': []})
 
-#       solr_cores = self.api.cores()
-#       for name in solr_cores:
-#         indexes.append({'name': name, 'type': 'core', 'collections': []})
+      if not self.is_solr_cloud_mode():
+        solr_cores = self.api.cores()
+        for name in solr_cores:
+          indexes.append({'name': name, 'type': 'core', 'collections': []})
 
       if self.is_solr_cloud_mode():
         try:
@@ -97,20 +113,31 @@ class SolrClient(object):
     return indexes
 
 
-  def create_index(self, name, fields, unique_key_field='id', df='text'):
+  def create_index(self, name, fields, config_name=None):
     """
     Create solr collection or core and instance dir.
     Create schema.xml file so that we can set UniqueKey field.
     """
+    unique_key_field = 'id'
+    df = 'text'
+
     if self.is_solr_cloud_mode():
-      self._create_solr_cloud_index_config_name(name, fields, unique_key_field, df)
-      if not self.api.create_collection(name):
-        raise Exception('Failed to create collection: %s' % name)
+      if config_name is None:
+        self._create_cloud_config(name, fields, unique_key_field, df) # Create config set
+      
+      self.api.create_collection2(name, config_name=config_name)
+      fields = [{
+          'name': field['name'],
+          'type': field['type'],
+          'stored': field.get('stored', True)
+        } for field in fields
+      ]
+      self.api.add_fields(name, fields)
     else:
       self._create_non_solr_cloud_index(name, fields, unique_key_field, df)
 
 
-  def _create_solr_cloud_index_config_name(self, name, fields, unique_key_field, df):
+  def _create_cloud_config(self, name, fields, unique_key_field, df):
     with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
       tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
 
@@ -152,7 +179,7 @@ class SolrClient(object):
     finally:
       # Delete instance directory if we couldn't create the core.
       shutil.rmtree(instancedir)
-#
+
 
   def delete_index(self, name):
     """

+ 8 - 24
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -156,11 +156,17 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     self.create = function() {
       $.post("${ url('indexer:create_index') }", {
         "name": self.name
-      }, function() {
-        window.location.reload();
+      }, function (data) {
+        if (data.status == 0) {
+          window.location.reload();
+        } else {
+          $(document).trigger("error", data.message);
+        }
       }).fail(function (xhr, textStatus, errorThrown) {
         $(document).trigger("error", xhr.responseText);
+        self.status('failed');
       });
+
     }
   };
 
@@ -233,28 +239,6 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     self.wizard(self.fileWizard);
     self.availableWizards = ko.observableArray([self.fileWizard, self.hiveWizard]);
 
-    self.getSample = function() {
-      $.post("${ url('indexer:create_wizard_get_sample') }", {
-        "wizard": ko.mapping.toJSON(self.wizard)
-      }, function(resp) {
-        self.wizard().sample(resp.data);
-        self.showCreate(true);
-      }).fail(function (xhr, textStatus, errorThrown) {
-        $(document).trigger("error", xhr.responseText);
-      });
-    }
-
-    self.create = function() {
-      $.post("${ url('indexer:create_wizard_create') }", {
-        "wizard": ko.mapping.toJSON(self.wizard)
-      }, function(resp) {
-        self.wizard().sample(resp.data);
-        self.showCreate(true);
-      }).fail(function (xhr, textStatus, errorThrown) {
-        $(document).trigger("error", xhr.responseText);
-      });
-    }
-
     self.edit = function() {
       self.show(true);
     }

+ 0 - 2
desktop/libs/indexer/src/indexer/urls.py

@@ -49,8 +49,6 @@ urlpatterns += patterns('indexer.solr_api',
   url(r'^api/aliases/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/indexes/create/$', 'create_index', name='create_index'),
   url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes'),
-  url(r'^api/indexes/create_wizard_get_sample/$', 'create_wizard_get_sample', name='create_wizard_get_sample'),
-  url(r'^api/indexes/create_wizard_create/$', 'create_wizard_create', name='create_wizard_create'),
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
 )
 

+ 3 - 2
desktop/libs/indexer/src/indexer/utils.py

@@ -35,8 +35,7 @@ from desktop.lib.i18n import force_unicode, smart_str
 from libsentry.conf import is_enabled
 
 from indexer import conf
-from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES,\
-                           DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
+from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES, DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
 
 
 LOG = logging.getLogger(__name__)
@@ -60,6 +59,7 @@ def get_config_template_path(solr_cloud_mode):
 
 
 class SchemaXml(object):
+
   def __init__(self, xml):
     self.xml = xml
     self.unique_key_field = None
@@ -80,6 +80,7 @@ class SchemaXml(object):
 
 
 class SolrConfigXml(object):
+
   def __init__(self, xml):
     self.xml = xml
 

+ 49 - 11
desktop/libs/libsolr/src/libsolr/api.py

@@ -25,15 +25,12 @@ from itertools import groupby
 
 from django.utils.translation import ugettext as _
 
+from dashboard.facet_builder import _compute_range_facet
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.conf import SERVER_USER
-from desktop.lib.conf import BoundConfig
 from desktop.lib.i18n import force_unicode
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest import resource
-from dashboard.facet_builder import _compute_range_facet
-
-from search.conf import EMPTY_QUERY, SECURITY_ENABLED, SOLR_URL
 
 from libsolr.conf import SSL_CERT_CA_VERIFY
 
@@ -41,24 +38,25 @@ from libsolr.conf import SSL_CERT_CA_VERIFY
 LOG = logging.getLogger(__name__)
 
 
+try:
+  from search.conf import EMPTY_QUERY, SECURITY_ENABLED, SOLR_URL
+except ImportError, e:
+  LOG.warn('Solr Search is not enabled')
+
+
 def utf_quoter(what):
   return urllib.quote(unicode(what).encode('utf-8'), safe='~@#$&()*!+=;,.?/\'')
 
-def search_enabled():
-  return type(SECURITY_ENABLED) == BoundConfig
-
 
 class SolrApi(object):
   """
   http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
   """
-  def __init__(self, solr_url=SOLR_URL.get(), user=None,
-               security_enabled=SECURITY_ENABLED.get() if search_enabled() else SECURITY_ENABLED.default,
-               ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()):
+  def __init__(self, solr_url=SOLR_URL.get(), user=None, security_enabled=False, ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()):
     self._url = solr_url
     self._user = user
     self._client = HttpClient(self._url, logger=LOG)
-    self.security_enabled = security_enabled
+    self.security_enabled = security_enabled or SECURITY_ENABLED.get()
 
     if self.security_enabled:
       self._client.set_kerberos_auth()
@@ -367,6 +365,7 @@ class SolrApi(object):
     except Exception, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
+  # Deprecated
   def create_collection(self, name, shards=1, replication=1):
     try:
       params = self._get_params() + (
@@ -387,6 +386,44 @@ class SolrApi(object):
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
+
+  def create_collection2(self, name, config_name=None, shards=1, replication=1, **kwargs):
+    try:
+      params = self._get_params() + (
+        ('action', 'CREATE'),
+        ('name', name),
+        ('numShards', shards),
+        ('replicationFactor', replication),        
+        ('wt', 'json')
+      )
+      if config_name:
+        params += (
+          ('collection.configName', config_name),
+        )
+      if kwargs:
+        params += ((key, val) for key, val in kwargs.iteritems())
+        
+
+      response = self._root.post('admin/collections', params=params, contenttype='application/json')
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  def add_fields(self, name, fields):
+    try:
+      params = self._get_params() + (        
+        ('wt', 'json')
+      )
+
+      data = {'add-field': fields}
+
+      response = self._root.post('%(collection)s/schema' % name, params=params, data=json.dumps(data), contenttype='application/json')
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
   def create_core(self, name, instance_dir, shards=1, replication=1):
     try:
       params = self._get_params() + (
@@ -812,6 +849,7 @@ class SolrApi(object):
         response = json.loads(response.replace('\x00', ''))
     return response
 
+
   def uniquekey(self, collection):
     try:
       params = self._get_params() + (