瀏覽代碼

[indexer] Rename collections to more generic indexes

WIP for schema design API
Jenny Kim 10 年之前
父節點
當前提交
702dd3036e

+ 61 - 33
desktop/libs/indexer/src/indexer/api2.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import csv
 import json
 import logging
 
@@ -25,15 +26,14 @@ from desktop.lib.exceptions_renderable import PopupException
 from libsolr.api import SolrApi
 from search.conf import SOLR_URL, SECURITY_ENABLED
 
-from indexer.controller2 import CollectionController
+from indexer.controller2 import IndexController
 from indexer.utils import get_default_fields
-import csv
 
 
 LOG = logging.getLogger(__name__)
 
 
-def create_collection(request):
+def create_index(request):
   if request.method != 'POST':
     raise PopupException(_('POST request required.'))
 
@@ -42,42 +42,21 @@ def create_collection(request):
   name = request.POST.get('name')
 
   if name:
-    searcher = CollectionController(request.user)
+    searcher = IndexController(request.user)
 
     try:
-      collection = searcher.create_collection(name,
-                                              request.POST.get('fields', get_default_fields()),
-                                              request.POST.get('uniqueKeyField', 'id'),
-                                              request.POST.get('df', 'text'))
+      collection = searcher.create_index(name,
+                                         request.POST.get('fields', get_default_fields()),
+                                         request.POST.get('uniqueKeyField', 'id'),
+                                         request.POST.get('df', 'text'))
 
       response['status'] = 0
       response['collection'] = collection
-      response['message'] = _('Collection created!')
+      response['message'] = _('Index created!')
     except Exception, e:
-      response['message'] = _('Collection could not be created: %s') % e
+      response['message'] = _('Index could not be created: %s') % e
   else:
-    response['message'] = _('Collection requires a name field.')
-
-  return JsonResponse(response)
-
-
-def create_or_edit_alias(request):
-  if request.method != 'POST':
-    raise PopupException(_('POST request required.'))
-
-  response = {'status': -1}
-
-  alias = request.POST.get('alias', '')
-  collections = json.loads(request.POST.get('collections', '[]'))
-
-  api = SolrApi(SOLR_URL.get(), request.user, SECURITY_ENABLED.get())
-
-  try:
-    api.create_or_modify_alias(alias, collections)
-    response['status'] = 0
-    response['message'] = _('Alias created or modified!')
-  except Exception, e:
-    response['message'] = _('Alias could not be created or modified: %s') % e
+    response['message'] = _('Index requires a name field.')
 
   return JsonResponse(response)
 
@@ -93,7 +72,7 @@ def delete_indexes(request):
   if not indexes:
     response['message'] = _('No indexes to remove.')
   else:
-    searcher = CollectionController(request.user)
+    searcher = IndexController(request.user)
 
     for index in indexes:
       if index['type'] == 'collection':
@@ -109,6 +88,26 @@ def delete_indexes(request):
   return JsonResponse(response)
 
 
+def create_or_edit_alias(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  alias = request.POST.get('alias', '')
+  collections = json.loads(request.POST.get('collections', '[]'))
+
+  api = SolrApi(SOLR_URL.get(), request.user, SECURITY_ENABLED.get())
+
+  try:
+    api.create_or_modify_alias(alias, collections)
+    response['status'] = 0
+    response['message'] = _('Alias created or modified!')
+  except Exception, e:
+    response['message'] = _('Alias could not be created or modified: %s') % e
+
+  return JsonResponse(response)
+
 
 def create_wizard_get_sample(request):
   if request.method != 'POST':
@@ -142,6 +141,35 @@ def create_wizard_create(request):
   return JsonResponse(response)
 
 
+def design_schema(request, index):
+  if request.method == 'POST':
+    pass # TODO: Support POST for update?
+
+  result = {'status': -1, 'message': ''}
+
+  try:
+    searcher = IndexController(request.user)
+    unique_key, fields = searcher.get_index_schema(index)
+
+    result['status'] = 0
+    formatted_fields = []
+    for field in fields:
+      formatted_fields.append({
+        'name': field,
+        'type': fields[field]['type'],
+        'required': fields[field].get('required', None),
+        'indexed': fields[field].get('indexed', None),
+        'stored': fields[field].get('stored', None),
+        'multivalued': fields[field].get('multivalued', None),
+      })
+    result['fields'] = formatted_fields
+    result['unique_key'] = unique_key
+  except Exception, e:
+    result['message'] = _('Could not get index schema: %s') % e
+
+  return JsonResponse(result)
+
+
 def _read_csv(f):
   content = f.read(1024 * 1024)
 

+ 65 - 48
desktop/libs/indexer/src/indexer/controller2.py

@@ -35,7 +35,7 @@ from search.conf import SOLR_URL, SECURITY_ENABLED
 LOG = logging.getLogger(__name__)
 MAX_UPLOAD_SIZE = 100 * 1024 * 1024 # 100 MB
 ALLOWED_FIELD_ATTRIBUTES = set(['name', 'type', 'indexed', 'stored'])
-FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored')]
+FLAGS = [('I', 'indexed'), ('T', 'tokenized'), ('S', 'stored'), ('M', 'multivalued')]
 ZK_SOLR_CONFIG_NAMESPACE = 'configs'
 
 
@@ -43,7 +43,11 @@ def get_solr_ensemble():
   return '%s/solr' % ENSEMBLE.get()
 
 
-class CollectionController(object):
+class IndexControllerException(Exception):
+  pass
+
+
+class IndexController(object):
   """
   Glue the models to the views.
   """
@@ -51,13 +55,7 @@ class CollectionController(object):
     self.user = user
     self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
 
-#  def _format_flags(self, fields):
-#    for field_name, field in fields.items():
-#      for index in range(0, len(FLAGS)):
-#        flags = FLAGS[index]
-#        field[flags[1]] = field['flags'][index] == FLAGS[index][0]
-#    return fields
-#
+
   def is_solr_cloud_mode(self):
     if not hasattr(self, '_solr_cloud_mode'):
       try:
@@ -96,67 +94,63 @@ class CollectionController(object):
 
     return indexes
 
-  def create_collection(self, name, fields, unique_key_field='id', df='text'):
+
+  def create_index(self, name, fields, unique_key_field='id', df='text'):
     """
     Create solr collection or core and instance dir.
     Create schema.xml file so that we can set UniqueKey field.
     """
     if self.is_solr_cloud_mode():
-      # Need to remove path afterwards
       tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
 
-      zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
-      root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
-      config_root_path = '%s/%s' % (solr_config_path, 'conf')
       try:
+        zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
+        root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
+        config_root_path = '%s/%s' % (solr_config_path, 'conf')
         zc.copy_path(root_node, config_root_path)
-      except Exception, e:
-        zc.delete_path(root_node)
-        raise PopupException(_('Error in copying Solr configurations.'), detail=e)
-
-      # Don't want directories laying around
-      shutil.rmtree(tmp_path)
 
-      if not self.api.create_collection(name):
-        # Delete instance directory if we couldn't create a collection.
-        try:
+        if not self.api.create_collection(name):
+          raise Exception('Failed to create collection: %s' % name)
+      except Exception, e:
+        if zc.path_exists(root_node):
+          # Remove the root node from Zookeeper
           zc.delete_path(root_node)
-        except Exception, e:
-          raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
+        raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
+      finally:
+        # Remove tmp config directory
+        shutil.rmtree(tmp_path)
+
     else:  # Non-solrcloud mode
       # Create instance directory locally.
       instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
+
       if os.path.exists(instancedir):
         raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
-      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
-      shutil.move(solr_config_path, instancedir)
-      shutil.rmtree(tmp_path)
 
-      if not self.api.create_core(name, instancedir):
-        # Delete instance directory if we couldn't create a collection.
+      try:
+        tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
+        shutil.move(solr_config_path, instancedir)
+        shutil.rmtree(tmp_path)
+
+        if not self.api.create_core(name, instancedir):
+          raise Exception('Failed to create core: %s' % name)
+      except Exception, e:
+        raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
+      finally:
+        # Delete instance directory if we couldn't create the core.
         shutil.rmtree(instancedir)
-        raise PopupException(_('Could not create collection. Check error logs for more info.'))
 
     return name
-
-#  def get_fields(self, collection_or_core_name):
-#    try:
-#      field_data = self.api.fields(collection_or_core_name)
-#      fields = self._format_flags(field_data['schema']['fields'])
-#    except:
-#      LOG.exception(_('Could not fetch fields for collection %s.') % collection_or_core_name)
-#      raise PopupException(_('Could not fetch fields for collection %s. See logs for more info.') % collection_or_core_name)
-#
-#    try:
-#      uniquekey = self.api.uniquekey(collection_or_core_name)
-#    except:
-#      LOG.exception(_('Could not fetch unique key for collection %s.') % collection_or_core_name)
-#      raise PopupException(_('Could not fetch unique key for collection %s. See logs for more info.') % collection_or_core_name)
-#
-#    return uniquekey, fields
 #
 
-  def delete_collection(self, name):
+  def delete_index(self, name):
+    """
+    Delete solr collection/core and instance dir
+    """
+    # TODO: Implement deletion of local Solr cores
+    if not self.is_solr_cloud_mode():
+      raise PopupException(_('Cannot remove non-Solr cloud cores.'))
+
     if self.api.remove_collection(name):
       # Delete instance directory.
       try:
@@ -170,5 +164,28 @@ class CollectionController(object):
     else:
       raise PopupException(_('Could not remove collection. Check error logs for more info.'))
 
+
+  def get_index_schema(self, index_name):
+    """
+    Returns a tuple of the unique key and schema fields for a given index
+    """
+    try:
+      field_data = self.api.fields(index_name)
+      fields = self._format_flags(field_data['schema']['fields'])
+      uniquekey = self.api.uniquekey(index_name)
+      return uniquekey, fields
+    except Exception, e:
+      LOG.exception(e.message)
+      raise IndexControllerException(_("Error in getting schema information for index '%s'" % index_name))
+
+
   def delete_alias(self, name):
     return self.api.delete_alias(name)
+
+
+  def _format_flags(self, fields):
+    for field_name, field in fields.items():
+      for index in range(0, len(FLAGS)):
+        flags = FLAGS[index]
+        field[flags[1]] = field['flags'][index] == FLAGS[index][0]
+    return fields

+ 30 - 23
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -41,11 +41,11 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
     </%def>
 
     <%def name="creation()">
-      <a href="javascript:void(0)" class="btn" data-bind="click: function() { collection.showCreateModal(true) }">
-        <i class="fa fa-plus-circle"></i> ${ _('Create collection') }
+      <a href="javascript:void(0)" class="btn" data-bind="click: function() { index.showCreateModal(true) }">
+        <i class="fa fa-plus-circle"></i> ${ _('Create index') }
       </a>
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { createWizard.show(true) }">
-        <i class="fa fa-plus-circle"></i> ${ _('Create collection from a file') }
+        <i class="fa fa-plus-circle"></i> ${ _('Create index from a file') }
       </a>      
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
         <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
@@ -61,13 +61,14 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
         <th>${ _('Name') }</th>
         <th>${ _('Type') }</th>
         <th>${ _('Collections') }</th>
+        <th>${ _('Schema') }</th>
       </tr>
     </thead>
     <tbody data-bind="foreach: { data: indexes }">
       <tr>
         <td data-bind="click: $root.handleSelect" class="center" style="cursor: default" data-row-selector-exclude="true">
           <div data-bind="css: { 'hueCheckbox': true, 'fa': true, 'fa-check': isSelected }" data-row-selector-exclude="true"></div>
-          ## <a data-bind="attr: { 'href': '${ url('spark:index') }?index=' + id() }" data-row-selector="true"></a>
+          ## <a data-bind="attr: { 'href': '${ url('search:index') }?index=' + id() }" data-row-selector="true"></a>
         </td>
         <td data-bind="text: name"></td>
         <td data-bind="text: type"></td>
@@ -77,6 +78,11 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
             <i class="fa fa-pencil"></i> ${ _('Edit') }
           </a>
         </td>
+        <td>
+          <a data-bind="attr: { 'href': '/indexer/api/indexes/' + name() + '/schema/' }, visible: type() == 'collection'">
+            <i class="fa fa-pencil"></i> ${ _('Edit') }
+          </a>
+        </td>
       </tr>
     </tbody>
   </table>
@@ -84,17 +90,17 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
   </div>
 </div>
 
-<!-- ko template: 'create-collection' --><!-- /ko -->
+<!-- ko template: 'create-index' --><!-- /ko -->
 
-<script type="text/html" id="create-collection">
-  <div class="snippet-settings" data-bind="visible: collection.showCreateModal">
+<script type="text/html" id="create-index">
+  <div class="snippet-settings" data-bind="visible: index.showCreateModal">
 
-    <input data-bind="value: collection.name"></input>
+    <input data-bind="value: index.name"></input>
 
-    <a href="javascript:void(0)" class="btn" data-bind="click: collection.create">
-      <i class="fa fa-plus-circle"></i> ${ _('Create collection') }
+    <a href="javascript:void(0)" class="btn" data-bind="click: index.create">
+      <i class="fa fa-plus-circle"></i> ${ _('Create index') }
     </a>
-    <a href="javascript:void(0)" class="btn" data-bind="click: function() { collection.showCreateModal(false) }">
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { index.showCreateModal(false) }">
       <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
     </a>
   </div>
@@ -119,9 +125,9 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 </script>
 
 
-<!-- ko template: 'create-collection-wizard' --><!-- /ko -->
+<!-- ko template: 'create-index-wizard' --><!-- /ko -->
 
-<script type="text/html" id="create-collection-wizard">
+<script type="text/html" id="create-index-wizard">
   <div class="snippet-settings" data-bind="visible: createWizard.show">
 
     ${ _('Name') } <input data-bind="value: createWizard.name"></input>
@@ -129,8 +135,8 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
     <!-- ko if: createWizard.name() -->
     <select data-bind="options: createWizard.availableWizards, value: createWizard.wizard, optionsText: 'name'" size="5"></select>
 
-    <span data-bind="template: { name: 'create-collection-from-file', data: createWizard.wizard }"></span>
-    <span data-bind="template: { name: 'create-collection-from-hive', data: createWizard.wizard }"></span>
+    <span data-bind="template: { name: 'create-index-from-file', data: createWizard.wizard }"></span>
+    <span data-bind="template: { name: 'create-index-from-hive', data: createWizard.wizard }"></span>
     
     <ul data-bind="foreach: createWizard.wizard().sample">
       <li>
@@ -161,7 +167,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 </script>
 
 
-<script type="text/html" id="create-collection-from-file">
+<script type="text/html" id="create-index-from-file">
   <!-- ko if: name() == 'file' -->
     <div class="snippet-settings" data-bind="visible: show">
 
@@ -173,7 +179,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 </script>
 
 
-<script type="text/html" id="create-collection-from-hive">
+<script type="text/html" id="create-index-from-hive">
   <!-- ko if: name() == 'hive' -->
     <div class="snippet-settings" data-bind="visible: show">
 
@@ -216,7 +222,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
 
 <script type="text/javascript" charset="utf-8">
-  var Collection = function () {
+  var Index = function () {
     var self = this;
 
     self.showCreateModal = ko.observable(false);
@@ -224,7 +230,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
     self.name = ko.observable('');
 
     self.create = function() {
-      $.post("${ url('indexer:create_collection') }", {
+      $.post("${ url('indexer:create_index') }", {
         "name": self.name
       }, function() {
         window.location.reload();
@@ -335,20 +341,20 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
     self.indexes = ko.mapping.fromJS(${ indexes_json | n });
 
-    self.collection = new Collection(self);
+    self.index = new Index(self);
     self.alias = new Alias(self);
     self.createWizard = new CreateWizard(self);
 
-    self.selectedJobs = ko.computed(function() {
+    self.selectedIndexes = ko.computed(function() {
       return $.grep(self.indexes(), function(index) { return index.isSelected(); });
     });
     self.isLoading = ko.observable(false);
 
     self.oneSelected = ko.computed(function() {
-      return self.selectedJobs().length == 1;
+      return self.selectedIndexes().length == 1;
     });
     self.atLeastOneSelected = ko.computed(function() {
-      return self.selectedJobs().length >= 1;
+      return self.selectedIndexes().length >= 1;
     });
     self.allSelected = ko.observable(false);
 
@@ -393,6 +399,7 @@ ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
         null,
         null,
         null,
+        { "bSortable":false },
       ],
       "aaSorting":[
         [1, 'asc' ]

+ 4 - 3
desktop/libs/indexer/src/indexer/urls.py

@@ -39,9 +39,10 @@ urlpatterns += patterns('indexer.api',
 
 urlpatterns += patterns('indexer.api2',
   # V2
-  url(r'^api/v2/collections/create/$', 'create_collection', name='create_collection'),
-  url(r'^api/alias/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
+  url(r'^api/aliases/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
+  url(r'^api/indexes/create/$', 'create_index', name='create_index'),
   url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes'),
   url(r'^api/indexes/create_wizard_get_sample/$', 'create_wizard_get_sample', name='create_wizard_get_sample'),
-  url(r'^api/indexes/create_wizard_create/$', 'create_wizard_create', name='create_wizard_create')
+  url(r'^api/indexes/create_wizard_create/$', 'create_wizard_create', name='create_wizard_create'),
+  url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
 )

+ 2 - 2
desktop/libs/indexer/src/indexer/views.py

@@ -22,7 +22,7 @@ from django.utils.translation import ugettext as _
 
 from desktop.lib.django_util import JsonResponse, render
 
-from indexer.controller2 import CollectionController
+from indexer.controller2 import IndexController
 from indexer.management.commands import indexer_setup
 
 
@@ -34,7 +34,7 @@ def collections(request, is_redirect=False):
 
 
 def indexes(request):
-  searcher = CollectionController(request.user)
+  searcher = IndexController(request.user)
   indexes = searcher.get_indexes()
   
   for index in indexes:

+ 10 - 0
desktop/libs/libzookeeper/src/libzookeeper/models.py

@@ -71,6 +71,16 @@ class ZookeeperClient(object):
     return children_data
 
 
+  def path_exists(self, namespace):
+    try:
+      self.zk.start()
+
+      return self.zk.exists(namespace) is not None
+    finally:
+      self.zk.stop()
+    return False
+
+
   def copy_path(self, namespace, filepath):
     if self.read_only:
       raise ReadOnlyClientException('Cannot execute copy_path when read_only is set to True.')

+ 15 - 0
desktop/libs/libzookeeper/src/libzookeeper/tests.py

@@ -95,6 +95,21 @@ class TestWithZooKeeper:
     db = client.get_children_data(namespace='')
     assert_true(len(db) > 0)
 
+  def test_path_exists(self):
+    try:
+      root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists')
+      client = ZookeeperClient(hosts=zkensemble(), read_only=False)
+
+      # Delete the root_node first just in case it wasn't cleaned up in previous run
+      client.zk.start()
+      client.zk.create(root_node, value='test_path_exists')
+      client.zk.stop()
+
+      assert_true(client.path_exists(namespace=root_node))
+      assert_false(client.path_exists(namespace='bogus_path'))
+    finally:
+      client.delete_path(root_node)
+
   def test_copy_and_delete_path(self):
     root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_copy_and_delete_path')
     client = ZookeeperClient(hosts=zkensemble(), read_only=False)