Kaynağa Gözat

HUE-2218 [search] Backend for getting the stats and terms of a field

Romain Rigaux 11 yıl önce
ebeveyn
işleme
1e20670

+ 28 - 2
apps/search/src/search/templates/search.mako

@@ -343,6 +343,7 @@ ${ dashboard.layout_skeleton() }
           <div style="margin-bottom: 3px">
             <input type="checkbox" data-bind="checkedValue: name, checked: $root.collection.template.fieldsSelected" style="margin: 0" />
             <div data-bind="text: name, css:{'field-selector': true, 'hoverable': $root.collection.template.fieldsSelected.indexOf(name()) > -1}, click: highlightColumn"></div>
+            <i class="fa fa-question-circle pull-right" data-bind="click: function() { $root.fieldAnalysesName(name()); $root.showFieldAnalysis(); }"></i>
           </div>
         </div>
         <div data-bind="visible: $root.collection.template.filteredAttributeFields().length == 0" style="padding-left: 4px; padding-top: 5px; font-size: 40px; color: #CCC">
@@ -563,7 +564,7 @@ ${ dashboard.layout_skeleton() }
 <script type="text/html" id="resultset-pagination">
 <div style="text-align: center; margin-top: 4px">
   <a href="javascript: void(0)" title="${ _('Previous') }">
-    <span data-bind="text: name, click: $root.collection.toggleSortColumnGridLayout"></span>
+    <span data-bind="click: $root.collection.toggleSortColumnGridLayout"></span>
     <i class="fa fa-arrow-left" data-bind="
         visible: $data.response.start * 1.0 >= $root.collection.template.rows() * 1.0,
         click: function() { $root.query.paginate('prev') }">
@@ -587,7 +588,7 @@ ${ dashboard.layout_skeleton() }
   </span>
 
   <a href="javascript: void(0)" title="${ _('Next') }">
-    <span data-bind="text: name, click: $root.collection.toggleSortColumnGridLayout"></span>
+    <span data-bind="click: $root.collection.toggleSortColumnGridLayout"></span>
     <i class="fa fa-arrow-right" data-bind="
         visible: ($root.collection.template.rows() * 1.0 + $data.response.start * 1.0) < $data.response.numFound,
         click: function() { $root.query.paginate('next') }">
@@ -802,6 +803,31 @@ ${ dashboard.layout_skeleton() }
   </div>
 </script>
 
+<script type="text/html" id="analysis-window">
+  <!-- ko if: $root.fieldAnalysesName() -->
+  <div data-bind="with: $root.getFieldAnalysis()">
+    <span data-bind="text: name"></span>
+    <ul class="nav nav-tabs" role="tablist">
+      <li class="active"><a href="#analysis-terms" role="tab" data-toggle="tab" data-bind="click: function() { section('terms'); }">${ _('Terms') }</a></li>
+      <li><a href="#analysis-stats" role="tab" data-toggle="tab" data-bind="click: function() { section('stats'); }">${ _('Stats') }</a></li>
+    </ul>
+    <div class="tab-content">
+      <div class="tab-pane active" id="analysis-terms" data-bind="with: terms">
+        <input type="text" data-bind="value: prefix, valueUpdate:'afterkeydown'" />
+        <span data-bind="text: ko.mapping.toJSON($data)"></span>
+      </div>
+      <div class="tab-pane" id="analysis-stats" data-bind="with: stats">
+        <input type="text" data-bind="value: facet" />
+        <span data-bind="text: ko.mapping.toJSON($data)"></span>
+      </div>
+    </div>
+  </div>
+  <!-- /ko -->
+</script>
+
+<div data-bind="template: { name: 'analysis-window' }"></div>
+
+
 <div id="shareModal" class="modal hide" data-backdrop="true">
   <div class="modal-header">
     <a href="javascript: void(0)" data-dismiss="modal" class="pull-right"><i class="fa fa-times"></i></a>

+ 2 - 0
apps/search/src/search/urls.py

@@ -37,6 +37,8 @@ urlpatterns = patterns('search.views',
   url(r'^get_timeline$', 'get_timeline', name='get_timeline'),
   url(r'^get_collection$', 'get_collection', name='get_collection'),
   url(r'^get_collections$', 'get_collections', name='get_collections'),
+  url(r'^get_stats$', 'get_stats', name='get_stats'),
+  url(r'^get_terms$', 'get_terms', name='get_terms'),
 
   # Admin
   url(r'^admin/collection_delete$', 'admin_collection_delete', name='admin_collection_delete'),

+ 56 - 1
apps/search/src/search/views.py

@@ -37,7 +37,8 @@ from search.conf import SOLR_URL
 from search.data_export import download as export_download
 from search.decorators import allow_admin_only
 from search.management.commands import search_setup
-from search.models import Collection, augment_solr_response, augment_solr_exception
+from search.models import Collection, augment_solr_response, augment_solr_exception,\
+  pairwise2
 from search.search_controller import SearchController
 
 
@@ -319,6 +320,60 @@ def get_document(request):
   return HttpResponse(json.dumps(result), mimetype="application/json")
 
 
+def get_stats(request):
+  result = {'status': -1, 'message': 'Error'}
+
+  try:
+    collection = json.loads(request.POST.get('collection', '{}'))
+    query = json.loads(request.POST.get('query', '{}'))
+    analysis = json.loads(request.POST.get('analysis', '{}'))
+
+    field = analysis['name']
+    facet = analysis['stats']['facet']
+
+    result['stats'] = SolrApi(SOLR_URL.get(), request.user).stats(collection['name'], [field], query, facet)
+    result['status'] = 0
+    result['message'] = ''
+
+  except Exception, e:
+    result['message'] = unicode(str(e), "utf8")
+    if 'not currently supported' in result['message']:
+      result['status'] = 1
+      result['message'] = _('This field does not support stats')
+
+  return HttpResponse(json.dumps(result), mimetype="application/json")
+
+
+def get_terms(request):
+  result = {'status': -1, 'message': 'Error'}
+
+  try:
+    collection = json.loads(request.POST.get('collection', '{}'))
+    analysis = json.loads(request.POST.get('analysis', '{}'))
+
+    field = analysis['name']
+    properties = {
+      'terms.prefix': analysis['terms']['prefix']
+      # lower
+      # limit
+      # mincount
+      # maxcount
+    }
+
+    result['terms'] = SolrApi(SOLR_URL.get(), request.user).terms(collection['name'], field, properties)
+    result['terms'] = pairwise2(field, [], result['terms']['terms'][field])
+    result['status'] = 0
+    result['message'] = ''
+
+  except Exception, e:
+    result['message'] = unicode(str(e), "utf8")
+    if 'not currently supported' in result['message']:
+      result['status'] = 1
+      result['message'] = _('This field does not support stats')
+
+  return HttpResponse(json.dumps(result), mimetype="application/json")
+
+
 def get_timeline(request):
   result = {'status': -1, 'message': 'Error'}
 

+ 115 - 6
apps/search/static/js/search.ko.js

@@ -104,7 +104,7 @@ var Query = function (vm, query) {
   };
 
   self.selectedMultiq.subscribe(function () { // To keep below the computed objects!
-	  vm.search();
+    vm.search();
   });
 
   self.toggleFacet = function (data) {
@@ -197,6 +197,86 @@ var Query = function (vm, query) {
 };
 
 
+var FieldAnalysis = function (vm, field_name) {
+  var self = this;
+
+  self.name = ko.observable(field_name);
+
+  self.section = ko.observable('terms');
+  self.section.subscribe(function () {
+    self.update();
+  });
+  self.terms = ko.mapping.fromJS({'prefix': '', 'data': []});
+  self.terms.prefix.subscribe(function () {
+  self.getTerms();
+  });
+  self.terms.prefix.extend({rateLimit: {timeout: 2000, method: "notifyWhenChangesStop"}});
+  self.stats = ko.mapping.fromJS({'facet': '', 'data': []});
+  self.stats.facet.subscribe(function () {
+    self.getStats();
+  });
+
+  self.update = function() {
+    if (self.section() == 'stats') {
+      if (self.stats.data().length == 0) {
+        self.getStats();
+      }
+    } else {
+      if (self.terms.data().length == 0) {
+         self.getTerms();
+      }
+    }
+  }
+
+  self.getTerms = function () {
+    self.terms.data.removeAll();
+
+    $.post("/search/get_terms", {
+      collection: ko.mapping.toJSON(vm.collection),
+      analysis: ko.mapping.toJSON(self)
+    }, function (data) {
+      if (data.status == 0) {  
+        $.each(data.terms, function(key, val) {
+          self.terms.data.push({'key': key, 'val': val});
+        });
+      }
+      else if (data.status == 1) {
+        self.terms.data.push({'key': 'Error', 'val': data.message});
+      }
+      else {
+        $(document).trigger("error", data.message);
+      }
+    }).fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+
+  self.getStats = function () {
+    self.stats.data.removeAll();
+
+    $.post("/search/get_stats", {
+      collection: ko.mapping.toJSON(vm.collection),
+      query: ko.mapping.toJSON(vm.query),
+      analysis: ko.mapping.toJSON(self)
+    }, function (data) {
+      if (data.status == 0) {  
+        $.each(data.stats.stats.stats_fields[self.name()], function(key, val) {
+          self.stats.data.push({'key': key, 'val': val});
+        });
+      }
+      else if (data.status == 1) {
+        self.stats.data.push({'key': 'Error', 'val': data.message});
+      }
+      else {
+        $(document).trigger("error", data.message);
+      }
+    }).fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+}
+
+
 var Collection = function (vm, collection) {
   var self = this;
 
@@ -354,13 +434,14 @@ var Collection = function (vm, collection) {
   self.template.fieldsModalFilter = ko.observable(""); // For UI
   self.template.fieldsModalType = ko.observable(""); // For UI
   self.template.fieldsAttributesFilter = ko.observable(""); // For UI
+
   self.template.filteredModalFields = ko.observableArray();
   self.template.filteredAttributeFieldsAll = ko.observable(true);
   self.template.filteredAttributeFields = ko.computed(function() {
     var _fields = [];
 
     var _iterable = self.template.fieldsAttributes();
-    if (!self.template.filteredAttributeFieldsAll()){
+    if (! self.template.filteredAttributeFieldsAll()){
       _iterable = self.template.fields();
     }
 
@@ -448,7 +529,7 @@ var Collection = function (vm, collection) {
     self.template.fieldsSelected.removeAll(_toDelete);
     var bulk = $.grep(currentObservable(), function(field) {
       return (_toDelete.indexOf(field.name()) != -1)
-    });     	
+    });  
     currentObservable.removeAll(bulk);
 
     // New fields
@@ -492,6 +573,7 @@ var Collection = function (vm, collection) {
     } else {
       template_field.sort.direction(null);
     }
+
     $(document).trigger("setResultsHeight");
     vm.search();
   };
@@ -520,7 +602,7 @@ var Collection = function (vm, collection) {
     vm.search();
   };
 
-  self.selectTimelineFacet = function (data) { // alert(ko.mapping.toJSON(data));
+  self.selectTimelineFacet = function (data) {
     var facet = self.getFacetById(data.widget_id);
 
     facet.properties.start(data.from);
@@ -697,6 +779,8 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
   self.disableGridlayoutResultChevron = function() {
     self.toggledGridlayoutResultChevron(false);
   };
+  self.fieldAnalyses = ko.observableArray([]);
+  self.fieldAnalysesName = ko.observableArray("");
 
   self.previewColumns = ko.observable("");
   self.columns = ko.observable([]);
@@ -704,7 +788,7 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
 
   self.isEditing = ko.observable(false);
   self.toggleEditing = function () {
-    self.isEditing(!self.isEditing());
+    self.isEditing(! self.isEditing());
   };
   self.isRetrievingResults = ko.observable(false);
 
@@ -898,7 +982,6 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
     });
   }
 
-
   self.getDocument = function (doc) {
     $.post("/search/get_document", {
       collection: ko.mapping.toJSON(self.collection),
@@ -927,6 +1010,32 @@ var SearchViewModel = function (collection_json, query_json, initial_json) {
     });
   };
 
+  self.showFieldAnalysis = function() {
+    if (self.fieldAnalysesName()) {
+      var analyse = self.getFieldAnalysis();
+
+      if (analyse == null) {
+        analyse = new FieldAnalysis(self, self.fieldAnalysesName());
+        self.fieldAnalyses.push(analyse);
+      }
+
+      analyse.update();
+    }
+  }
+
+  self.getFieldAnalysis = function() {
+    var field_name = self.fieldAnalysesName();
+    var _analyse = null;
+
+    $.each(self.fieldAnalyses(), function (index, analyse) {
+      if (analyse.name() == field_name) {
+        _analyse = analyse;
+        return false;
+      }
+    });
+
+    return _analyse;
+  }
 
   self.save = function () {
     $.post("/search/save", {

+ 28 - 5
desktop/libs/libsolr/src/libsolr/api.py

@@ -51,12 +51,18 @@ class SolrApi(object):
       self._client.set_kerberos_auth()
     self._root = resource.Resource(self._client)
 
+
   def _get_params(self):
     if self.security_enabled:
       return (('doAs', self._user ),)
     return (('user.name', DEFAULT_USER), ('doAs', self._user),)
 
 
+  def _get_q(self, query):
+    q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
+    return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
+
+
   def query(self, collection, query):
     solr_query = {}
 
@@ -72,10 +78,8 @@ class SolrApi(object):
     solr_query['rows'] = min(solr_query['rows'], 1000)
     solr_query['start'] = min(solr_query['start'], 10000)
 
-    q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
-
     params = self._get_params() + (
-        ('q', 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')),
+        ('q', self._get_q(query)),
         ('wt', 'json'),
         ('rows', solr_query['rows']),
         ('start', solr_query['start']),
@@ -335,20 +339,39 @@ class SolrApi(object):
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
-  def stats(self, core, fields):
+  def stats(self, core, fields, query=None, facet=''):
     try:
       params = self._get_params() + (
-          ('q', EMPTY_QUERY.get()),
+          ('q', self._get_q(query) if query is not None else EMPTY_QUERY.get()),
           ('wt', 'json'),
           ('rows', 0),
           ('stats', 'true'),
       )
+      if facet:
+        params += (('stats.facet', facet),)
+
       params += tuple([('stats.field', field) for field in fields])
       response = self._root.get('%(core)s/select' % {'core': core}, params=params)
       return self._get_json(response)
     except RestException, e:
       raise PopupException(e, title=_('Error while accessing Solr'))
 
+  def terms(self, core, field, properties=None):
+    try:
+      params = self._get_params() + (
+          ('wt', 'json'),
+          ('rows', 0),
+          ('terms.fl', field),
+      )
+      if properties:
+        for key, val in properties.iteritems():
+          params += ((key, val),)
+
+      response = self._root.get('%(core)s/terms' % {'core': core}, params=params)
+      return self._get_json(response)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Solr'))
+
   def get(self, core, doc_id):
     try:
       params = self._get_params() + (