Browse Source

[indexer] Add ability to quick-create empty collection

Jenny Kim 10 năm trước cách đây
mục cha
commit
3bce9ef

+ 32 - 1
desktop/libs/indexer/src/indexer/api.py

@@ -30,7 +30,8 @@ from search.models import Collection
 
 from indexer.controller import CollectionManagerController
 from indexer.controller2 import CollectionController
-from indexer.utils import fields_from_log, field_values_from_separated_file, get_type_from_morphline_type, get_field_types
+from indexer.utils import fields_from_log, field_values_from_separated_file, get_type_from_morphline_type, \
+  get_field_types, get_default_fields
 
 
 LOG = logging.getLogger(__name__)
@@ -278,6 +279,36 @@ def collections_data(request, collection):
   return JsonResponse(response)
 
 
+# V2 API
+
+def create_collection(request):
+  if request.method != 'POST':
+    raise PopupException(_('POST request required.'))
+
+  response = {'status': -1}
+
+  name = request.POST.get('name')
+
+  if name:
+    searcher = CollectionController(request.user)
+
+    try:
+      collection = searcher.create_collection(name,
+                                              request.POST.get('fields', get_default_fields()),
+                                              request.POST.get('uniqueKeyField', 'id'),
+                                              request.POST.get('df', 'text'))
+
+      response['status'] = 0
+      response['collection'] = collection
+      response['message'] = _('Collection created!')
+    except Exception, e:
+      response['message'] = _('Collection could not be created: %s') % e
+  else:
+    response['message'] = _('Collection requires a name field.')
+
+  return JsonResponse(response)
+
+
 def create_or_edit_alias(request):
   if request.method != 'POST':
     raise PopupException(_('POST request required.'))

+ 48 - 46
desktop/libs/indexer/src/indexer/controller2.py

@@ -16,19 +16,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import logging
+import os
+import shutil
 
 from django.utils.translation import ugettext as _
 
 from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.i18n import smart_str
+from indexer.conf import CORE_INSTANCE_DIR
+from indexer.utils import copy_configs
 from libsolr.api import SolrApi
 from libzookeeper.conf import ENSEMBLE
 from libzookeeper.models import ZookeeperClient
 from search.conf import SOLR_URL, SECURITY_ENABLED
 
-from desktop.lib.i18n import smart_str
-
 
 LOG = logging.getLogger(__name__)
 MAX_UPLOAD_SIZE = 100 * 1024 * 1024 # 100 MB
@@ -94,6 +96,49 @@ class CollectionController(object):
 
     return indexes
 
+  def create_collection(self, name, fields, unique_key_field='id', df='text'):
+    """
+    Create solr collection or core and instance dir.
+    Create schema.xml file so that we can set UniqueKey field.
+    """
+    if self.is_solr_cloud_mode():
+      # Need to remove path afterwards
+      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
+
+      zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
+      root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
+      config_root_path = '%s/%s' % (solr_config_path, 'conf')
+      try:
+        zc.copy_path(root_node, config_root_path)
+      except Exception, e:
+        zc.delete_path(root_node)
+        raise PopupException(_('Error in copying Solr configurations.'), detail=e)
+
+      # Don't want directories laying around
+      shutil.rmtree(tmp_path)
+
+      if not self.api.create_collection(name):
+        # Delete instance directory if we couldn't create a collection.
+        try:
+          zc.delete_path(root_node)
+        except Exception, e:
+          raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
+    else:  # Non-solrcloud mode
+      # Create instance directory locally.
+      instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
+      if os.path.exists(instancedir):
+        raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
+      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
+      shutil.move(solr_config_path, instancedir)
+      shutil.rmtree(tmp_path)
+
+      if not self.api.create_core(name, instancedir):
+        # Delete instance directory if we couldn't create a collection.
+        shutil.rmtree(instancedir)
+        raise PopupException(_('Could not create collection. Check error logs for more info.'))
+
+    return name
+
 #  def get_fields(self, collection_or_core_name):
 #    try:
 #      field_data = self.api.fields(collection_or_core_name)
@@ -110,49 +155,6 @@ class CollectionController(object):
 #
 #    return uniquekey, fields
 #
-#  def create_collection(self, name, fields, unique_key_field='id', df='text'):
-#    """
-#    Create solr collection or core and instance dir.
-#    Create schema.xml file so that we can set UniqueKey field.
-#    """
-#    if self.is_solr_cloud_mode():
-#      # solrcloud mode
-#
-#      # Need to remove path afterwards
-#      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
-#
-#      zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
-#      root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
-#      config_root_path = '%s/%s' % (solr_config_path, 'conf')
-#      try:
-#        zc.copy_path(root_node, config_root_path)
-#      except Exception, e:
-#        zc.delete_path(root_node)
-#        raise PopupException(_('Error in copying Solr configurations.'), detail=e)
-#
-#      # Don't want directories laying around
-#      shutil.rmtree(tmp_path)
-#
-#      if not self.api.create_collection(name):
-#        # Delete instance directory if we couldn't create a collection.
-#        try:
-#          zc.delete_path(root_node)
-#        except Exception, e:
-#          raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
-#    else:
-#      # Non-solrcloud mode
-#      # Create instance directory locally.
-#      instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
-#      if os.path.exists(instancedir):
-#        raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
-#      tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
-#      shutil.move(solr_config_path, instancedir)
-#      shutil.rmtree(tmp_path)
-#
-#      if not self.api.create_core(name, instancedir):
-#        # Delete instance directory if we couldn't create a collection.
-#        shutil.rmtree(instancedir)
-#        raise PopupException(_('Could not create collection. Check error logs for more info.'))
 
   def delete_collection(self, name):
     if self.api.remove_collection(name):

+ 36 - 13
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -20,7 +20,7 @@
 %>
 <%namespace name="actionbar" file="actionbar.mako" />
 
-${ commonheader(_("Solr Indexes"), "spark", user, "60px") | n,unicode }
+${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
 
 
 <div class="container-fluid">
@@ -41,7 +41,7 @@ ${ commonheader(_("Solr Indexes"), "spark", user, "60px") | n,unicode }
     </%def>
 
     <%def name="creation()">
-      <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
+      <a href="javascript:void(0)" class="btn" data-bind="click: function() { collection.showCreateModal(true) }">
         <i class="fa fa-plus-circle"></i> ${ _('Create collection') }
       </a>
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
@@ -81,28 +81,32 @@ ${ commonheader(_("Solr Indexes"), "spark", user, "60px") | n,unicode }
   </div>
 </div>
 
-<!-- ko template: 'create-alias' --><!-- /ko -->
+<!-- ko template: 'create-collection' --><!-- /ko -->
 
-<script type="text/html" id="create-alias">
-  <div class="snippet-settings" data-bind="visible: alias.showCreateModal">
+<script type="text/html" id="create-collection">
+  <div class="snippet-settings" data-bind="visible: collection.showCreateModal">
 
-    <input data-bind="value: alias.name"></input>
-    <select data-bind="options: alias.availableCollections, selectedOptions: alias.chosenCollections, optionsText: 'name', optionsValue: 'name'" size="5" multiple="true"></select>
+    <input data-bind="value: collection.name"></input>
 
-    <a href="javascript:void(0)" class="btn" data-bind="click: alias.create, visible: alias.chosenCollections().length > 0">
-      <i class="fa fa-plus-circle"></i> ${ _('Create or edit') }
+    <a href="javascript:void(0)" class="btn" data-bind="click: collection.create">
+      <i class="fa fa-plus-circle"></i> ${ _('Create collection') }
     </a>
-    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(false) }">
+    <a href="javascript:void(0)" class="btn" data-bind="click: function() { collection.showCreateModal(false) }">
       <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
     </a>
   </div>
 </script>
 
-<script type="text/html" id="create-collection">
+<!-- ko template: 'create-alias' --><!-- /ko -->
+
+<script type="text/html" id="create-alias">
   <div class="snippet-settings" data-bind="visible: alias.showCreateModal">
 
-    <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
-      <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
+    <input data-bind="value: alias.name"></input>
+    <select data-bind="options: alias.availableCollections, selectedOptions: alias.chosenCollections, optionsText: 'name', optionsValue: 'name'" size="5" multiple="true"></select>
+
+    <a href="javascript:void(0)" class="btn" data-bind="click: alias.create, visible: alias.chosenCollections().length > 0">
+      <i class="fa fa-plus-circle"></i> ${ _('Create or edit') }
     </a>
     <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(false) }">
       <i class="fa fa-plus-circle"></i> ${ _('Cancel') }
@@ -154,6 +158,24 @@ ${ commonheader(_("Solr Indexes"), "spark", user, "60px") | n,unicode }
 
 
 <script type="text/javascript" charset="utf-8">
+  var Collection = function () {
+    var self = this;
+
+    self.showCreateModal = ko.observable(false);
+
+    self.name = ko.observable('');
+
+    self.create = function() {
+      $.post("${ url('indexer:create_collection') }", {
+        "name": self.name
+      }, function() {
+        window.location.reload();
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+      });
+    }
+  };
+
   var Alias = function (vm) {
     var self = this;
 
@@ -189,6 +211,7 @@ ${ commonheader(_("Solr Indexes"), "spark", user, "60px") | n,unicode }
 
     self.indexes = ko.mapping.fromJS(${ indexes_json | n });
 
+    self.collection = new Collection(self);
     self.alias = new Alias(self);
 
     self.selectedJobs = ko.computed(function() {

+ 1 - 0
desktop/libs/indexer/src/indexer/urls.py

@@ -36,6 +36,7 @@ urlpatterns += patterns('indexer.api',
   url(r'^api/collections/(?P<collection>[^/]+)/data/$', 'collections_data', name='api_collections_data'),
 
   # V2
+  url(r'^api/v2/collections/create/$', 'create_collection', name='create_collection'),
   url(r'^api/alias/create_or_edit/$', 'create_or_edit_alias', name='create_or_edit_alias'),
   url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes')
 )

+ 10 - 0
desktop/libs/indexer/src/indexer/utils.py

@@ -359,3 +359,13 @@ def fields_from_log(fh):
   fields.append(('message', 'text_general'))
 
   return fields
+
+
+def get_default_fields():
+  """
+  Returns a list of default fields for the Solr schema.xml
+  :return:
+  """
+  default_field = DEFAULT_FIELD
+  default_field.update({'name': 'id', 'type': 'string', 'multiValued': 'false'})
+  return [default_field]