فهرست منبع

[search] Add query autocomplete

Fix some oozie tests too.
Romain Rigaux 12 سال پیش
والد
کامیت
e351f0f0db

+ 1 - 1
apps/search/src/search/api.py

@@ -86,7 +86,7 @@ class SolrApi(object):
           ('q', solr_query['q']),
           ('wt', 'json'),
       )
-      response = self._root.get('%(core)s/suggest' % solr_query, params)
+      response = self._root.get('%(collection)s/suggest' % solr_query, params)
       if type(response) != dict:
         response = json.loads(response)
       return response

+ 24 - 2
apps/search/src/search/models.py

@@ -252,7 +252,7 @@ em {
   <div class="row-fluid">
     <div class="span12">%s</div>
   </div>
-  <br/>  
+  <br/>
 </div>""" % ' '.join(['{{%s}}' % field for field in collection.fields(user)])
 
       result.update_from_post({'template': json.dumps(template)})
@@ -270,7 +270,8 @@ class Collection(models.Model):
   cores = models.TextField(default=json.dumps({}), verbose_name=_t('Collection with cores data'), help_text=_t('Solr json'))
   properties = models.TextField(
       default=json.dumps({}), verbose_name=_t('Properties'),
-      help_text=_t('Hue properties (e.g. results by pages number)'))
+      help_text=_t('Hue properties (e.g. results by pages number)')
+  )
 
   facets = models.ForeignKey(Facet)
   result = models.ForeignKey(Result)
@@ -294,6 +295,27 @@ class Collection(models.Model):
     return sorted([{'name': field.get('name'), 'type': field.get('type')}
                    for fields in schema.iter('fields') for field in fields.iter('field')])
 
+  @property
+  def properties_dict(self):
+    if not self.properties:
+      self.data = json.dumps({})
+    properties_python = json.loads(self.properties)
+    # Backward compatibility
+    if 'autocomplete' not in properties_python:
+      properties_python['autocomplete'] = False
+    return properties_python
+
+  @property
+  def autocomplete(self):
+    return self.properties_dict['autocomplete']
+
+  @autocomplete.setter
+  def autocomplete(self, autocomplete):
+    properties_ = self.properties_dict
+    properties_['autocomplete'] = autocomplete
+    self.properties = json.dumps(properties_)
+
+
 def get_facet_field_format(field, type, facets):
   format = ""
   try:

+ 38 - 9
apps/search/src/search/templates/admin_collection_properties.mako

@@ -48,7 +48,7 @@ ${ commonheader(_('Search'), "search", user, "29px") | n,unicode }
   </%def>
 
   <%def name="content()">
-  <form method="POST">
+  <form id="collectionProperties" method="POST">
     <ul class="nav nav-tabs" style="margin-bottom:0; margin-top:10px">
       <li class="active">
         <a href="#index" data-toggle="tab">${_('Collection')}</a>
@@ -67,11 +67,13 @@ ${ commonheader(_('Search'), "search", user, "29px") | n,unicode }
           ${ utils.render_field(collection_form['enabled']) }
           ${ utils.render_field(collection_form['label']) }
           ${ utils.render_field(collection_form['name']) }
+
+          ${ _('Autocomplete and suggest queries') } <br/> <input type="checkbox" data-bind="checked: autocomplete" />
         </div>
 
-	    <div class="form-actions">
-	      <button type="submit" class="btn btn-primary" id="save-sorting">${_('Save')}</button>
-	    </div>
+      <div class="form-actions">
+        <a class="btn btn-primary" id="saveBtn">${_('Save')}</a>
+      </div>
       </div>
 
       <div class="tab-pane" id="schema">
@@ -92,16 +94,43 @@ ${ commonheader(_('Search'), "search", user, "29px") | n,unicode }
 <link rel="stylesheet" href="/static/ext/css/codemirror.css">
 <script src="/static/ext/js/codemirror-xml.js"></script>
 
+
+<script src="/static/ext/js/knockout.mapping-2.3.2.js" type="text/javascript" charset="utf-8"></script>
+
 <script type="text/javascript" charset="utf-8">
   $(document).ready(function(){
     var schemaViewer = $("#schema_field")[0];
 
     var codeMirror = CodeMirror(function (elt) {
-      schemaViewer.parentNode.replaceChild(elt, schemaViewer);
-    }, {
-      value: schemaViewer.value,
-      readOnly: true,
-      lineNumbers: true
+        schemaViewer.parentNode.replaceChild(elt, schemaViewer);
+      }, {
+        value: schemaViewer.value,
+        readOnly: true,
+        lineNumbers: true
+    });
+
+    var ViewModel = function() {
+        var self = this;
+        self.autocomplete = ko.observable(${ collection_properties | n }.autocomplete);
+
+        self.submit = function(form) {
+          var form = $("#collectionProperties");
+
+          $("<input>").attr("type", "hidden")
+                  .attr("name", "autocomplete")
+                  .attr("value", ko.utils.stringifyJson(self.autocomplete))
+                  .appendTo(form);
+
+          form.submit();
+        };
+      };
+
+    window.viewModel = new ViewModel();
+    ko.applyBindings(window.viewModel, $('#collectionProperties')[0]);
+
+
+    $("#saveBtn").click(function () {
+      window.viewModel.submit();
     });
 
     codeMirror.setSize("100%", $(document).height() - 150 - $(".form-actions").outerHeight());

+ 22 - 0
apps/search/src/search/templates/search.mako

@@ -524,6 +524,28 @@ ${ commonheader(_('Search'), "search", user, "90px") | n,unicode }
     });
     % endif
 
+    % if hue_collection.autocomplete:
+    $("#id_query").attr("autocomplete", "off");
+
+    var autocompleteTimeout = -1;
+    $("#id_query").on("keyup", function () {
+      window.clearTimeout(autocompleteTimeout);
+      autocompleteTimeout = window.setTimeout(function() {
+        var query = $("#id_query").val();
+        if (query) {
+          $.ajax("${ url('search:query_suggest', collection_id=hue_collection.id) }" + query, {
+            type: 'GET',
+            success: function (data) {
+              if (data.message.spellcheck && ! jQuery.isEmptyObject(data.message.spellcheck.suggestions)) {
+                $('#id_query').typeahead({source: data.message.spellcheck.suggestions[1].suggestion});
+              }
+            }
+          });
+        }
+      }, 300);
+    });
+    % endif
+
     function getFq(existing, currentField, currentValue) {
       if (existing.indexOf(currentField) > -1) {
         var _pieces = existing.split("|");

+ 4 - 2
apps/search/src/search/views.py

@@ -224,10 +224,11 @@ def admin_collection_properties(request, collection_id):
 
   if request.method == 'POST':
     collection_form = CollectionForm(request.POST, instance=hue_collection, user=request.user)
-    if collection_form.is_valid():
+    if collection_form.is_valid(): # Check for autocomplete in data?
       searcher = SearchController(request.user)
       hue_collection = collection_form.save(commit=False)
       hue_collection.is_core_only = not searcher.is_collection(hue_collection.name)
+      hue_collection.autocomplete = json.loads(request.POST.get('autocomplete'))
       hue_collection.save()
       return redirect(reverse('search:admin_collection_properties', kwargs={'collection_id': hue_collection.id}))
     else:
@@ -239,6 +240,7 @@ def admin_collection_properties(request, collection_id):
     'solr_collection': solr_collection,
     'hue_collection': hue_collection,
     'collection_form': collection_form,
+    'collection_properties': json.dumps(hue_collection.properties_dict)
   })
 
 
@@ -349,7 +351,7 @@ def query_suggest(request, collection_id, query=""):
   result = {'status': -1, 'message': 'Error'}
 
   solr_query = {}
-  solr_query['collection'] = collection
+  solr_query['collection'] = hue_collection.name
   solr_query['q'] = query
 
   try:

+ 1 - 1
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -180,7 +180,7 @@ class Submission(object):
     if self.jt and self.jt.logical_name:
       jobtracker_addr = self.jt.logical_name
 
-    if self.fs and self.fs.logical_name:
+    if self.fs.logical_name:
       fs_defaultfs = self.fs.logical_name
     else:
       fs_defaultfs = self.fs.fs_defaultfs

+ 110 - 1
desktop/libs/liboozie/src/liboozie/submittions_test.py

@@ -19,6 +19,9 @@ import logging
 
 from nose.tools import assert_equal, assert_true
 
+from hadoop import cluster
+from hadoop.conf import HDFS_CLUSTERS, MR_CLUSTERS, YARN_CLUSTERS
+
 from liboozie.submittion import Submission
 from oozie.tests import OozieMockBase
 
@@ -53,6 +56,7 @@ class TestSubmission(OozieMockBase):
         'nameNode': 'hdfs://curacao:8020'
       }, submission.properties)
 
+
   def test_get_logical_properties(self):
     submission = Submission(self.user, fs=MockFs(logical_name='fsname'), jt=MockJt(logical_name='jtname'))
 
@@ -63,4 +67,109 @@ class TestSubmission(OozieMockBase):
     assert_equal({
         'jobTracker': 'jtname',
         'nameNode': 'fsname'
-      }, submission.properties)
+      }, submission.properties)
+
+  def test_update_properties(self):
+    finish = []
+    finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
+    finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
+    try:
+      properties = {
+        'user.name': 'hue',
+        'test.1': 'http://localhost/test?test1=test&test2=test',
+        'nameNode': 'hdfs://curacao:8020',
+        'jobTracker': 'jtaddress'
+      }
+
+      final_properties = properties.copy()
+      submission = Submission(None, properties=properties, oozie_id='test', fs=MockFs())
+      assert_equal(properties, submission.properties)
+      submission._update_properties('jtaddress', 'deployment-directory')
+      assert_equal(final_properties, submission.properties)
+
+      cluster.clear_caches()
+      fs = cluster.get_hdfs()
+      jt = cluster.get_next_ha_mrcluster()[1]
+      final_properties = properties.copy()
+      final_properties.update({
+        'jobTracker': 'jtaddress',
+        'nameNode': fs.fs_defaultfs
+      })
+      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
+      assert_equal(properties, submission.properties)
+      submission._update_properties('jtaddress', 'deployment-directory')
+      assert_equal(final_properties, submission.properties)
+
+      finish.append(HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('namenode'))
+      finish.append(MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker'))
+      cluster.clear_caches()
+      fs = cluster.get_hdfs()
+      jt = cluster.get_next_ha_mrcluster()[1]
+      final_properties = properties.copy()
+      final_properties.update({
+        'jobTracker': 'jobtracker',
+        'nameNode': 'namenode'
+      })
+      submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
+      assert_equal(properties, submission.properties)
+      submission._update_properties('jtaddress', 'deployment-directory')
+      assert_equal(final_properties, submission.properties)
+    finally:
+      cluster.clear_caches()
+      for reset in finish:
+        reset()
+
+
+  def test_get_external_parameters(self):
+    xml = """
+<workflow-app name="Pig" xmlns="uri:oozie:workflow:0.4">
+    <start to="Pig"/>
+    <action name="Pig">
+        <pig>
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <prepare>
+                  <delete path="${output}"/>
+            </prepare>
+            <script>aggregate.pig</script>
+              <argument>-param</argument>
+              <argument>INPUT=${input}</argument>
+              <argument>-param</argument>
+              <argument>OUTPUT=${output}</argument>
+        </pig>
+        <ok to="end"/>
+        <error to="kill"/>
+    </action>
+    <kill name="kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+    <end name="end"/>
+</workflow-app>
+    """
+
+    properties = """
+#
+# Licensed to the Hue
+#
+
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+examplesRoot=examples
+
+oozie.use.system.libpath=true
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
+    """
+    parameters = Submission(self.user)._get_external_parameters(xml, properties)
+
+    assert_equal({'oozie.use.system.libpath': 'true',
+                   'input': '',
+                   'jobTracker': 'localhost:8021',
+                   'oozie.wf.application.path': '${nameNode}/user/${user.name}/${examplesRoot}/apps/pig',
+                   'examplesRoot': 'examples',
+                   'output': '',
+                   'nameNode': 'hdfs://localhost:8020',
+                   'queueName': 'default'
+                  },
+                 parameters)

+ 0 - 114
desktop/libs/liboozie/src/liboozie/tests.py

@@ -20,13 +20,9 @@ import logging
 from django.contrib.auth.models import User
 from nose.tools import assert_equal
 
-from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import reformat_xml
 
-from hadoop import cluster
-from hadoop.conf import HDFS_CLUSTERS, MR_CLUSTERS, YARN_CLUSTERS
 from liboozie.types import WorkflowAction, Coordinator
-from liboozie.submittion import Submission
 from liboozie.utils import config_gen
 from oozie.tests import MockOozieApi
 
@@ -61,113 +57,3 @@ def test_config_gen():
   <value><![CDATA[hue]]></value>
 </property>
 </configuration>"""), reformat_xml(config_gen(properties)))
-
-
-def test_update_properties():
-  finish = []
-  finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
-  finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
-  try:
-    properties = {
-      'user.name': 'hue',
-      'test.1': 'http://localhost/test?test1=test&test2=test'
-    }
-
-    final_properties = properties.copy()
-    submission = Submission(None, properties=properties, oozie_id='test')
-    assert_equal(properties, submission.properties)
-    submission._update_properties('jtaddress', 'deployment-directory')
-    assert_equal(final_properties, submission.properties)
-
-    cluster.clear_caches()
-    fs = cluster.get_hdfs()
-    jt = cluster.get_next_ha_mrcluster()[1]
-    final_properties = properties.copy()
-    final_properties.update({
-      'jobTracker': 'jtaddress',
-      'nameNode': fs.fs_defaultfs
-    })
-    submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
-    assert_equal(properties, submission.properties)
-    submission._update_properties('jtaddress', 'deployment-directory')
-    assert_equal(final_properties, submission.properties)
-
-    finish.append(HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('namenode'))
-    finish.append(MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker'))
-    cluster.clear_caches()
-    fs = cluster.get_hdfs()
-    jt = cluster.get_next_ha_mrcluster()[1]
-    final_properties = properties.copy()
-    final_properties.update({
-      'jobTracker': 'jobtracker',
-      'nameNode': 'namenode'
-    })
-    submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
-    assert_equal(properties, submission.properties)
-    submission._update_properties('jtaddress', 'deployment-directory')
-    assert_equal(final_properties, submission.properties)
-  finally:
-    cluster.clear_caches()
-    for reset in finish:
-      reset()
-
-
-class TestSubmission():
-
-  def setUp(self):
-    self.c = make_logged_in_client(is_superuser=False)
-    self.user = User.objects.get(username='test')
-
-  def test_get_external_parameters(self):
-    xml = """
-<workflow-app name="Pig" xmlns="uri:oozie:workflow:0.4">
-    <start to="Pig"/>
-    <action name="Pig">
-        <pig>
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <prepare>
-                  <delete path="${output}"/>
-            </prepare>
-            <script>aggregate.pig</script>
-              <argument>-param</argument>
-              <argument>INPUT=${input}</argument>
-              <argument>-param</argument>
-              <argument>OUTPUT=${output}</argument>
-        </pig>
-        <ok to="end"/>
-        <error to="kill"/>
-    </action>
-    <kill name="kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-    <end name="end"/>
-</workflow-app>
-    """
-
-    properties = """
-#
-# Licensed to the Hue
-#
-
-nameNode=hdfs://localhost:8020
-jobTracker=localhost:8021
-queueName=default
-examplesRoot=examples
-
-oozie.use.system.libpath=true
-
-oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
-    """
-    parameters = Submission(self.user)._get_external_parameters(xml, properties)
-
-    assert_equal({'oozie.use.system.libpath': 'true',
-                   'input': '',
-                   'jobTracker': 'localhost:8021',
-                   'oozie.wf.application.path': '${nameNode}/user/${user.name}/${examplesRoot}/apps/pig',
-                   'examplesRoot': 'examples',
-                   'output': '',
-                   'nameNode': 'hdfs://localhost:8020',
-                   'queueName': 'default'
-                  },
-                 parameters)