Эх сурвалжийг харах

HUE-5304 [search] Asynchronously load the indexes

Romain Rigaux 8 жил өмнө
parent
commit
c81368d

+ 3 - 0
desktop/libs/indexer/src/indexer/api3.py

@@ -33,6 +33,7 @@ from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
 from indexer.smart_indexer import Indexer
+from libsolr.api import SolrApi
 
 
 LOG = logging.getLogger(__name__)
@@ -398,6 +399,8 @@ def _index(request, file_format, collection_name, query=None):
   if is_unique_generated:
     schema_fields += [{"name": unique_field, "type": "string"}]
 
+  api = SolrApi(user=request.user)
+   
   collection_manager = CollectionManagerController(request.user)
   if not collection_manager.collection_exists(collection_name):
     collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)

+ 3 - 3
desktop/libs/indexer/src/indexer/solr_api.py

@@ -51,12 +51,12 @@ def api_error_handler(func):
 
 @require_POST
 @api_error_handler
-def list_collections(request):
+def list_indexes(request):
   response = {'status': -1}
 
-  api = SolrApi(user=request.user)
+  client = SolrClient(user=request.user)
 
-  response['collections'] = [{'isCoreOnly': False, 'isAlias': False, 'collections': [], 'name': name} for name in api.collections2()]
+  response['collections'] = client.get_indexes()
   response['status'] = 0
 
   return JsonResponse(response)

+ 5 - 5
desktop/libs/indexer/src/indexer/solr_client.py

@@ -91,11 +91,6 @@ class SolrClient(object):
         for name in collections:
           indexes.append({'name': name, 'type': 'collection', 'collections': []})
 
-      if not self.is_solr_cloud_mode():
-        solr_cores = self.api.cores()
-        for name in solr_cores:
-          indexes.append({'name': name, 'type': 'core', 'collections': []})
-
       if self.is_solr_cloud_mode():
         try:
           solr_aliases = self.api.aliases()
@@ -105,6 +100,11 @@ class SolrClient(object):
         except Exception:
           LOG.exception('Aliases could not be retrieved')
 
+      if not self.is_solr_cloud_mode():
+        solr_cores = self.api.cores()
+        for name in solr_cores:
+          indexes.append({'name': name, 'type': 'core', 'collections': []})
+
     except Exception, e:
       msg = _('Solr server could not be contacted properly: %s') % e
       LOG.warn(msg)

+ 21 - 6
desktop/libs/indexer/src/indexer/templates/indexes.mako

@@ -44,7 +44,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     <%def name="creation()">
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { index.showCreateModal(true) }">
         <i class="fa fa-plus-circle"></i> ${ _('Create index') }
-      </a>      
+      </a>
       <a href="javascript:void(0)" class="btn" data-bind="click: function() { alias.showCreateModal(true) }">
         <i class="fa fa-plus-circle"></i> ${ _('Create alias') }
       </a>
@@ -191,7 +191,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
         $(document).trigger("error", xhr.responseText);
       });
     }
-    
+
     self.edit = function(alias) {
       self.name(alias.name());
       self.chosenCollections(alias.collections());
@@ -227,9 +227,9 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
 
     self.show = ko.observable(false);
     self.showCreate = ko.observable(false);
-    
+
     self.fileWizard = new FileWizard(vm);
-    self.hiveWizard = new HiveWizard(vm);    
+    self.hiveWizard = new HiveWizard(vm);
 
     self.name = ko.observable('');
     self.wizard = ko.observable();
@@ -247,7 +247,7 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
   var Editor = function () {
     var self = this;
 
-    self.indexes = ko.mapping.fromJS(${ indexes_json | n });
+    self.indexes = ko.observable([]);
 
     self.index = new Index(self);
     self.alias = new Alias(self);
@@ -279,6 +279,20 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
 
     self.datatable = null;
 
+    self.fetchIndexes = function() {
+      $.post("${ url('indexer:list_indexes') }", {
+      }, function(data) {
+        var indexes = []
+        data.collections.forEach(function(index) {
+          index.isSelected = false;
+          indexes.push(ko.mapping.fromJS(index));
+        });
+        self.indexes(indexes);
+      }).fail(function (xhr, textStatus, errorThrown) {
+        $(document).trigger("error", xhr.responseText);
+      });
+    };
+
     self.deleteIndexes = function() {
       $.post("${ url('indexer:delete_indexes') }", {
         "indexes": ko.mapping.toJSON(self.selectedIndexes)
@@ -345,8 +359,9 @@ ${ commonheader(_("Index Browser"), "search", user, request, "60px") | n,unicode
     });
 
     $("a[data-row-selector='true']").jHueRowSelector();
+
+    viewModel.fetchIndexes();
   });
 </script>
 
-
 ${ commonfooter(request, messages) | n,unicode }

+ 1 - 1
desktop/libs/indexer/src/indexer/urls.py

@@ -47,13 +47,13 @@ urlpatterns += patterns('indexer.api',
 urlpatterns += patterns('indexer.solr_api',
   # V2
   url(r'^api/aliases/create/$', 'create_alias', name='create_alias'),
+  url(r'^api/indexes/list/$', 'list_indexes', name='list_indexes'),
   url(r'^api/indexes/create/$', 'create_index', name='create_index'),
   url(r'^api/indexes/delete/$', 'delete_indexes', name='delete_indexes'),
   url(r'^api/indexes/(?P<index>\w+)/schema/$', 'design_schema', name='design_schema')
 )
 
 urlpatterns += patterns('indexer.solr_api',
-  url(r'^api/collections/list/$', 'list_collections', name='list_collections'),
   url(r'^api/collections/delete/$', 'delete_collections', name='delete_collections'),
 )
 

+ 1 - 9
desktop/libs/indexer/src/indexer/views.py

@@ -47,15 +47,7 @@ def indexes(request):
   if not request.user.has_hue_permission(action="access", app='search'):
     raise PopupException(_('Missing permission.'), error_code=403)
 
-  searcher = SolrClient(request.user)
-  indexes = searcher.get_indexes()
-
-  for index in indexes:
-    index['isSelected'] = False
-
-  return render('indexes.mako', request, {
-      'indexes_json': json.dumps(indexes)
-  })
+  return render('indexes.mako', request, {})
 
 
 def indexer(request):

+ 70 - 49
desktop/libs/libsolr/src/libsolr/api.py

@@ -365,27 +365,6 @@ class SolrApi(object):
     except Exception, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
-  # Deprecated
-  def create_collection(self, name, shards=1, replication=1):
-    try:
-      params = self._get_params() + (
-        ('action', 'CREATE'),
-        ('name', name),
-        ('numShards', shards),
-        ('replicationFactor', replication),
-        ('collection.configName', name),
-        ('wt', 'json')
-      )
-
-      response = self._root.post('admin/collections', params=params, contenttype='application/json')
-      if 'success' in response:
-        return True
-      else:
-        LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
-        return False
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
 
   def create_collection2(self, name, config_name=None, shards=1, replication=1, **kwargs):
     try:
@@ -446,6 +425,7 @@ class SolrApi(object):
       else:
         raise PopupException(e, title=_('Error while accessing Solr'))
 
+
   def create_alias(self, name, collections):
     try:
       params = self._get_params() + (
@@ -480,24 +460,6 @@ class SolrApi(object):
     except RestException, e:
         raise PopupException(e, title=_('Error while accessing Solr'))
 
-  # Deprecated
-  def remove_collection(self, name):
-    try:
-      params = self._get_params() + (
-        ('action', 'DELETE'),
-        ('name', name),
-        ('wt', 'json')
-      )
-
-      response = self._root.post('admin/collections', params=params, contenttype='application/json')
-      if 'success' in response:
-        return True
-      else:
-        LOG.error("Could not remove collection. Check response:\n%s" % json.dumps(response, indent=2))
-        return False
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
-
 
   def delete_collection(self, name):
     response = {'status': -1, 'message': ''}
@@ -509,12 +471,11 @@ class SolrApi(object):
         ('wt', 'json')
       )
 
-      result = self._root.post('admin/collections', params=params, contenttype='application/json')
-      if 'success' in result:
+      data = self._root.post('admin/collections', params=params, contenttype='application/json')
+      if 'success' in data['responseHeader']['status'] == 0:
         response['status'] = 0
       else:
-        LOG.error("Could not remove collection: %s" % result)
-        response['message'] = result.get('failure')
+        response['message'] = "Could not remove collection: %s" % data
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
     return response
@@ -538,12 +499,12 @@ class SolrApi(object):
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
-  def add_fields(self, collection, fields):
-    try:
-      params = self._get_params()
-      return self._root.post('%s/schema/fields' % collection, params=params, data=json.dumps(fields), contenttype='application/json')
-    except RestException, e:
-      raise PopupException(e, title=_('Error while accessing Solr'))
+#   def add_fields(self, collection, fields):
+#     try:
+#       params = self._get_params()
+#       return self._root.post('%s/schema/fields' % collection, params=params, data=json.dumps(fields), contenttype='application/json')
+#     except RestException, e:
+#       raise PopupException(e, title=_('Error while accessing Solr'))
 
   def cores(self):
     try:
@@ -645,6 +606,7 @@ class SolrApi(object):
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
+
   def info_system(self):
     try:
       params = self._get_params() + (
@@ -656,6 +618,7 @@ class SolrApi(object):
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
+
   def sql(self, collection, statement):
     try:
       if 'limit' not in statement.lower(): # rows is not supported
@@ -686,6 +649,64 @@ class SolrApi(object):
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
+
+  def export(self, name, query, fl, sort, rows=100):
+    # /solr/demo6/export?user.name=hue&doAs=romain&q=code_s:[0%20TO%20400]&wt=json&rows=25&start=0&fl=code_s&sort=code_s+desc
+    try:
+      params = self._get_params() + (
+          ('q', query),
+          ('fl', fl),
+          ('sort', sort),
+          ('rows', rows),
+          ('wt', 'json'),
+      )
+      response = self._root.get('%(name)s/export' % {'name': name}, params=params)
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  # Deprecated
+  def create_collection(self, name, shards=1, replication=1):
+    try:
+      params = self._get_params() + (
+        ('action', 'CREATE'),
+        ('name', name),
+        ('numShards', shards),
+        ('replicationFactor', replication),
+        ('collection.configName', name),
+        ('wt', 'json')
+      )
+
+      response = self._root.post('admin/collections', params=params, contenttype='application/json')
+      if 'success' in response:
+        return True
+      else:
+        LOG.error("Could not create collection. Check response:\n%s" % json.dumps(response, indent=2))
+        return False
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
+  # Deprecated
+  def remove_collection(self, name):
+    try:
+      params = self._get_params() + (
+        ('action', 'DELETE'),
+        ('name', name),
+        ('wt', 'json')
+      )
+
+      response = self._root.post('admin/collections', params=params, contenttype='application/json')
+      if 'success' in response:
+        return True
+      else:
+        LOG.error("Could not remove collection. Check response:\n%s" % json.dumps(response, indent=2))
+        return False
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
+
   def _get_params(self):
     if self.security_enabled:
       return (('doAs', self._user ),)