浏览代码

HUE-5304 [solr] Refactor create a collection to support managed templates

Romain Rigaux 8 年之前
父节点
当前提交
e61ae6c

+ 1 - 0
desktop/libs/indexer/src/indexer/controller.py

@@ -263,6 +263,7 @@ class CollectionManagerController(object):
         else:
         else:
           raise PopupException(_('Could not update index. Unknown type %s') % data_type)
           raise PopupException(_('Could not update index. Unknown type %s') % data_type)
         fh.close()
         fh.close()
+
       if not api.update(collection_or_core_name, data, content_type=content_type):
       if not api.update(collection_or_core_name, data, content_type=content_type):
         raise PopupException(_('Could not update index. Check error logs for more info.'))
         raise PopupException(_('Could not update index. Check error logs for more info.'))
     else:
     else:

+ 13 - 64
desktop/libs/indexer/src/indexer/solr_api.py

@@ -15,7 +15,6 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-import csv
 import json
 import json
 import logging
 import logging
 
 
@@ -27,8 +26,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.i18n import smart_unicode
 from libsolr.api import SolrApi
 from libsolr.api import SolrApi
 
 
-from indexer.solr_client import SolrClient
-from indexer.utils import get_default_fields
+from indexer.solr_client import SolrClient, _get_fields
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -79,32 +77,24 @@ def delete_collections(request):
   return JsonResponse(response)
   return JsonResponse(response)
 
 
 
 
+@require_POST
+@api_error_handler
 def create_index(request):
 def create_index(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
   response = {'status': -1}
   response = {'status': -1}
 
 
   name = request.POST.get('name')
   name = request.POST.get('name')
+  fields = json.loads(request.POST.get('fields', '[]'))
 
 
-  if name:
-    searcher = SolrClient(request.user)
+  client = SolrClient(request.user)
 
 
-    try:
-      collection = searcher.create_index(
-          name,
-          request.POST.get('fields', get_default_fields()),
-          request.POST.get('uniqueKeyField', 'id'),
-          request.POST.get('df', 'text')
-      )
-
-      response['status'] = 0
-      response['collection'] = collection
-      response['message'] = _('Index created!')
-    except Exception, e:
-      response['message'] = _('Index could not be created: %s') % e
-  else:
-    response['message'] = _('Index requires a name field.')
+  collection = client.create_index(
+      name=name,
+      fields=request.POST.get('fields', _get_fields(name='id', type='string')),
+  )
+
+  response['status'] = 0
+  response['collection'] = collection
+  response['message'] = _('Index created!')
 
 
   return JsonResponse(response)
   return JsonResponse(response)
 
 
@@ -157,37 +147,6 @@ def create_or_edit_alias(request):
   return JsonResponse(response)
   return JsonResponse(response)
 
 
 
 
-def create_wizard_get_sample(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  wizard = json.loads(request.POST.get('wizard', '{}'))
-
-  f = request.fs.open(wizard['path'])
-
-  response['status'] = 0
-  response['data'] = _read_csv(f)
-
-  return JsonResponse(response)
-
-
-def create_wizard_create(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  wizard = json.loads(request.POST.get('wizard', '{}'))
-
-  f = request.fs.open(wizard['path'])
-
-  response['status'] = 0
-  response['data'] = _read_csv(f)
-
-  return JsonResponse(response)
-
 
 
 def design_schema(request, index):
 def design_schema(request, index):
   if request.method == 'POST':
   if request.method == 'POST':
@@ -218,13 +177,3 @@ def design_schema(request, index):
   return JsonResponse(result)
   return JsonResponse(result)
 
 
 
 
-def _read_csv(f):
-  content = f.read(1024 * 1024)
-
-  dialect = csv.Sniffer().sniff(content)
-  lines = content.splitlines()[:5]
-  reader = csv.reader(lines, delimiter=dialect.delimiter)
-  
-  return [row for row in reader]
-
-

+ 40 - 13
desktop/libs/indexer/src/indexer/solr_client.py

@@ -46,14 +46,28 @@ ZK_SOLR_CONFIG_NAMESPACE = 'configs'
 IS_SOLR_CLOUD = None
 IS_SOLR_CLOUD = None
 
 
 
 
+DEFAULT_FIELD = {
+  'name': None,
+  'type': 'text',
+  'indexed': True,
+  'stored': True,
+  'required': False,
+  'multiValued': False
+}
+
+
 class SolrClientException(Exception):
 class SolrClientException(Exception):
   pass
   pass
 
 
 
 
+def _get_fields(name='id', type='text'):
+  default_field = DEFAULT_FIELD.copy()
+  default_field.update({'name': name, 'type': type})
+  return [default_field]
+
+
 class SolrClient(object):
 class SolrClient(object):
-  """
-  Glue the models to the views.
-  """
+
   def __init__(self, user):
   def __init__(self, user):
     self.user = user
     self.user = user
     self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
     self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
@@ -61,9 +75,10 @@ class SolrClient(object):
 
 
   def is_solr_cloud_mode(self):
   def is_solr_cloud_mode(self):
     global IS_SOLR_CLOUD
     global IS_SOLR_CLOUD
+
     if IS_SOLR_CLOUD is None:
     if IS_SOLR_CLOUD is None:
-      print self.api.info_system()
       IS_SOLR_CLOUD = self.api.info_system().get('mode', 'solrcloud') == 'solrcloud'
       IS_SOLR_CLOUD = self.api.info_system().get('mode', 'solrcloud') == 'solrcloud'
+
     return IS_SOLR_CLOUD
     return IS_SOLR_CLOUD
 
 
 
 
@@ -76,9 +91,10 @@ class SolrClient(object):
         for name in collections:
         for name in collections:
           indexes.append({'name': name, 'type': 'collection', 'collections': []})
           indexes.append({'name': name, 'type': 'collection', 'collections': []})
 
 
-#       solr_cores = self.api.cores()
-#       for name in solr_cores:
-#         indexes.append({'name': name, 'type': 'core', 'collections': []})
+      if not self.is_solr_cloud_mode():
+        solr_cores = self.api.cores()
+        for name in solr_cores:
+          indexes.append({'name': name, 'type': 'core', 'collections': []})
 
 
       if self.is_solr_cloud_mode():
       if self.is_solr_cloud_mode():
         try:
         try:
@@ -97,20 +113,31 @@ class SolrClient(object):
     return indexes
     return indexes
 
 
 
 
-  def create_index(self, name, fields, unique_key_field='id', df='text'):
+  def create_index(self, name, fields, config_name=None):
     """
     """
     Create solr collection or core and instance dir.
     Create solr collection or core and instance dir.
     Create schema.xml file so that we can set UniqueKey field.
     Create schema.xml file so that we can set UniqueKey field.
     """
     """
+    unique_key_field = 'id'
+    df = 'text'
+
     if self.is_solr_cloud_mode():
     if self.is_solr_cloud_mode():
-      self._create_solr_cloud_index_config_name(name, fields, unique_key_field, df)
-      if not self.api.create_collection(name):
-        raise Exception('Failed to create collection: %s' % name)
+      if config_name is None:
+        self._create_cloud_config(name, fields, unique_key_field, df) # Create config set
+      
+      self.api.create_collection2(name, config_name=config_name)
+      fields = [{
+          'name': field['name'],
+          'type': field['type'],
+          'stored': field.get('stored', True)
+        } for field in fields
+      ]
+      self.api.add_fields(name, fields)
     else:
     else:
       self._create_non_solr_cloud_index(name, fields, unique_key_field, df)
       self._create_non_solr_cloud_index(name, fields, unique_key_field, df)
 
 
 
 
-  def _create_solr_cloud_index_config_name(self, name, fields, unique_key_field, df):
+  def _create_cloud_config(self, name, fields, unique_key_field, df):
     with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
     with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
       tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
       tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
 
 
@@ -152,7 +179,7 @@ class SolrClient(object):
     finally:
     finally:
       # Delete instance directory if we couldn't create the core.
       # Delete instance directory if we couldn't create the core.
       shutil.rmtree(instancedir)
       shutil.rmtree(instancedir)
-#
+
 
 
   def delete_index(self, name):
   def delete_index(self, name):
     """
     """

+ 8 - 24
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -156,11 +156,17 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     self.create = function() {
     self.create = function() {
       $.post("${ url('indexer:create_index') }", {
       $.post("${ url('indexer:create_index') }", {
         "name": self.name
         "name": self.name
-      }, function() {
-        window.location.reload();
+      }, function (data) {
+        if (data.status == 0) {
+          window.location.reload();
+        } else {
+          $(document).trigger("error", data.message);
+        }
       }).fail(function (xhr, textStatus, errorThrown) {
       }).fail(function (xhr, textStatus, errorThrown) {
         $(document).trigger("error", xhr.responseText);
         $(document).trigger("error", xhr.responseText);
+        self.status('failed');
       });
       });
+
     }
     }
   };
   };
 
 
@@ -233,28 +239,6 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     self.wizard(self.fileWizard);
     self.wizard(self.fileWizard);
     self.availableWizards = ko.observableArray([self.fileWizard, self.hiveWizard]);
     self.availableWizards = ko.observableArray([self.fileWizard, self.hiveWizard]);
 
 
-    self.getSample = function() {
-      $.post("${ url('indexer:create_wizard_get_sample') }", {
-        "wizard": ko.mapping.toJSON(self.wizard)
-      }, function(resp) {
-        self.wizard().sample(resp.data);
-        self.showCreate(true);
-      }).fail(function (xhr, textStatus, errorThrown) {
-        $(document).trigger("error", xhr.responseText);
-      });
-    }
-
-    self.create = function() {
-      $.post("${ url('indexer:create_wizard_create') }", {
-        "wizard": ko.mapping.toJSON(self.wizard)
-      }, function(resp) {
-        self.wizard().sample(resp.data);
-        self.showCreate(true);
-      }).fail(function (xhr, textStatus, errorThrown) {
-        $(document).trigger("error", xhr.responseText);
-      });
-    }
-
     self.edit = function() {
     self.edit = function() {
       self.show(true);
       self.show(true);
     }
     }

+ 0 - 2
desktop/libs/indexer/src/indexer/urls.py

@@ -49,8 +49,6 @@ urlpatterns += patterns('indexer.solr_api',
   url(r'^api/aliases/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/aliases/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/indexes/create/$', 'create_index', name='create_index'),
   url(r'^api/indexes/create/$', 'create_index', name='create_index'),
   url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes'),
   url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes'),
-  url(r'^api/indexes/create_wizard_get_sample/$', 'create_wizard_get_sample', name='create_wizard_get_sample'),
-  url(r'^api/indexes/create_wizard_create/$', 'create_wizard_create', name='create_wizard_create'),
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
 )
 )
 
 

+ 3 - 2
desktop/libs/indexer/src/indexer/utils.py

@@ -35,8 +35,7 @@ from desktop.lib.i18n import force_unicode, smart_str
 from libsentry.conf import is_enabled
 from libsentry.conf import is_enabled
 
 
 from indexer import conf
 from indexer import conf
-from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES,\
-                           DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
+from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES, DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -60,6 +59,7 @@ def get_config_template_path(solr_cloud_mode):
 
 
 
 
 class SchemaXml(object):
 class SchemaXml(object):
+
   def __init__(self, xml):
   def __init__(self, xml):
     self.xml = xml
     self.xml = xml
     self.unique_key_field = None
     self.unique_key_field = None
@@ -80,6 +80,7 @@ class SchemaXml(object):
 
 
 
 
 class SolrConfigXml(object):
 class SolrConfigXml(object):
+
   def __init__(self, xml):
   def __init__(self, xml):
     self.xml = xml
     self.xml = xml
 
 

+ 49 - 11
desktop/libs/libsolr/src/libsolr/api.py

@@ -25,15 +25,12 @@ from itertools import groupby
 
 
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
+from dashboard.facet_builder import _compute_range_facet
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.conf import SERVER_USER
 from desktop.conf import SERVER_USER
-from desktop.lib.conf import BoundConfig
 from desktop.lib.i18n import force_unicode
 from desktop.lib.i18n import force_unicode
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest import resource
 from desktop.lib.rest import resource
-from dashboard.facet_builder import _compute_range_facet
-
-from search.conf import EMPTY_QUERY, SECURITY_ENABLED, SOLR_URL
 
 
 from libsolr.conf import SSL_CERT_CA_VERIFY
 from libsolr.conf import SSL_CERT_CA_VERIFY
 
 
@@ -41,24 +38,25 @@ from libsolr.conf import SSL_CERT_CA_VERIFY
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 
 
+try:
+  from search.conf import EMPTY_QUERY, SECURITY_ENABLED, SOLR_URL
+except ImportError, e:
+  LOG.warn('Solr Search is not enabled')
+
+
 def utf_quoter(what):
 def utf_quoter(what):
   return urllib.quote(unicode(what).encode('utf-8'), safe='~@#$&()*!+=;,.?/\'')
   return urllib.quote(unicode(what).encode('utf-8'), safe='~@#$&()*!+=;,.?/\'')
 
 
-def search_enabled():
-  return type(SECURITY_ENABLED) == BoundConfig
-
 
 
 class SolrApi(object):
 class SolrApi(object):
   """
   """
   http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
   http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
   """
   """
-  def __init__(self, solr_url=SOLR_URL.get(), user=None,
-               security_enabled=SECURITY_ENABLED.get() if search_enabled() else SECURITY_ENABLED.default,
-               ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()):
+  def __init__(self, solr_url=SOLR_URL.get(), user=None, security_enabled=False, ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()):
     self._url = solr_url
     self._url = solr_url
     self._user = user
     self._user = user
     self._client = HttpClient(self._url, logger=LOG)
     self._client = HttpClient(self._url, logger=LOG)
-    self.security_enabled = security_enabled
+    self.security_enabled = security_enabled or SECURITY_ENABLED.get()
 
 
     if self.security_enabled:
     if self.security_enabled:
       self._client.set_kerberos_auth()
       self._client.set_kerberos_auth()
@@ -367,6 +365,7 @@ class SolrApi(object):
     except Exception, e:
     except Exception, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+  # Deprecated
   def create_collection(self, name, shards=1, replication=1):
   def create_collection(self, name, shards=1, replication=1):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (
@@ -387,6 +386,44 @@ class SolrApi(object):
     except RestException, e:
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
       raise PopupException(e, title=_('Error while accessing Solr'))
 
 
+
+  def create_collection2(self, name, config_name=None, shards=1, replication=1, **kwargs):
+    try:
+      params = self._get_params() + (
+        ('action', 'CREATE'),
+        ('name', name),
+        ('numShards', shards),
+        ('replicationFactor', replication),        
+        ('wt', 'json')
+      )
+      if config_name:
+        params += (
+          ('collection.configName', config_name),
+        )
+      if kwargs:
+        params += ((key, val) for key, val in kwargs.iteritems())
+        
+
+      response = self._root.post('admin/collections', params=params, contenttype='application/json')
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  def add_fields(self, name, fields):
+    try:
+      params = self._get_params() + (        
+        ('wt', 'json')
+      )
+
+      data = {'add-field': fields}
+
+      response = self._root.post('%(collection)s/schema' % name, params=params, data=json.dumps(data), contenttype='application/json')
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
   def create_core(self, name, instance_dir, shards=1, replication=1):
   def create_core(self, name, instance_dir, shards=1, replication=1):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (
@@ -812,6 +849,7 @@ class SolrApi(object):
         response = json.loads(response.replace('\x00', ''))
         response = json.loads(response.replace('\x00', ''))
     return response
     return response
 
 
+
   def uniquekey(self, collection):
   def uniquekey(self, collection):
     try:
     try:
       params = self._get_params() + (
       params = self._get_params() + (