Browse Source

HUE-3065 [desktop] Allow searching of raw SQL in saved queries

Jenny Kim 9 years ago
parent
commit
c9b17a21f7

+ 7 - 4
desktop/core/src/desktop/models.py

@@ -864,7 +864,7 @@ class Document2QueryMixin(object):
       trashed_ids = [doc.id for doc in docs if Document2.TRASH_DIR in doc.path]
       trashed_ids = [doc.id for doc in docs if Document2.TRASH_DIR in doc.path]
       docs = docs.exclude(id__in=trashed_ids)
       docs = docs.exclude(id__in=trashed_ids)
 
 
-    return docs.defer('description', 'data', 'extra').distinct().order_by('-last_modified')
+    return docs.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
 
 
 
 
   def search_documents(self, types=None, search_text=None, order_by=None):
   def search_documents(self, types=None, search_text=None, order_by=None):
@@ -880,7 +880,8 @@ class Document2QueryMixin(object):
       documents = documents.filter(type__in=types)
       documents = documents.filter(type__in=types)
 
 
     if search_text:
     if search_text:
-      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text))
+      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) |
+                                   Q(search__icontains=search_text))
 
 
     if order_by:  # TODO: Validate that order_by is a valid sort parameter
     if order_by:  # TODO: Validate that order_by is a valid sort parameter
       documents = documents.order_by(order_by)
       documents = documents.order_by(order_by)
@@ -1001,6 +1002,7 @@ class Document2(models.Model):
 
 
   data = models.TextField(default='{}')
   data = models.TextField(default='{}')
   extra = models.TextField(default='')
   extra = models.TextField(default='')
+  search = models.TextField(blank=True, null=True, help_text=_t('Searchable text for the document.'))
   # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access
   # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access
 
 
   last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
   last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
@@ -1272,7 +1274,7 @@ class Document2(models.Model):
     information like personally identifiable information, that information could be leaked into the Hue database and
     information like personally identifiable information, that information could be leaked into the Hue database and
     logfiles.
     logfiles.
     """
     """
-    if global_redaction_engine.is_enabled() and self.type == 'notebook':
+    if global_redaction_engine.is_enabled() and (self.type == 'notebook' or self.type.startswith('query')):
       data_dict = self.data_dict
       data_dict = self.data_dict
       snippets = data_dict.get('snippets', [])
       snippets = data_dict.get('snippets', [])
       for snippet in snippets:
       for snippet in snippets:
@@ -1283,6 +1285,7 @@ class Document2(models.Model):
             snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
             snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
             snippet['is_redacted'] = True
             snippet['is_redacted'] = True
       self.data = json.dumps(data_dict)
       self.data = json.dumps(data_dict)
+      self.search = global_redaction_engine.redact(self.search)
 
 
   def _contains_cycle(self):
   def _contains_cycle(self):
     """
     """
@@ -1345,7 +1348,7 @@ class Directory(Document2):
 
 
     documents = documents.exclude(is_history=True)
     documents = documents.exclude(is_history=True)
 
 
-    return documents.defer('description', 'data', 'extra').distinct().order_by('-last_modified')
+    return documents.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
 
 
 
 
   def save(self, *args, **kwargs):
   def save(self, *args, **kwargs):

+ 8 - 5
desktop/core/src/desktop/tests_doc2.py

@@ -176,8 +176,8 @@ class TestDocument2(object):
     # Creates 2 directories and 2 queries and saves to home directory
     # Creates 2 directories and 2 queries and saves to home directory
     dir1 = Directory.objects.create(name='test_dir1', owner=self.user)
     dir1 = Directory.objects.create(name='test_dir1', owner=self.user)
     dir2 = Directory.objects.create(name='test_dir2', owner=self.user)
     dir2 = Directory.objects.create(name='test_dir2', owner=self.user)
-    query1 = Document2.objects.create(name='query1.sql', type='query-hive', owner=self.user, data={})
-    query2 = Document2.objects.create(name='query2.sql', type='query-hive', owner=self.user, data={})
+    query1 = Document2.objects.create(name='query1.sql', type='query-hive', owner=self.user, data={}, search='foobar')
+    query2 = Document2.objects.create(name='query2.sql', type='query-hive', owner=self.user, data={}, search='barfoo')
     children = [dir1, dir2, query1, query2]
     children = [dir1, dir2, query1, query2]
 
 
     self.home_dir.children.add(*children)
     self.home_dir.children.add(*children)
@@ -196,11 +196,14 @@ class TestDocument2(object):
     assert_true(all(doc['type'] == 'directory' for doc in data['children']))
     assert_true(all(doc['type'] == 'directory' for doc in data['children']))
 
 
     # Test search text
     # Test search text
-    response = self.client.get('/desktop/api2/doc', {'path': '/', 'text': 'query'})
+    response = self.client.get('/desktop/api2/doc', {'path': '/', 'text': 'foo'})
     data = json.loads(response.content)
     data = json.loads(response.content)
-    assert_equal('query', data['text'])
+    assert_equal('foo', data['text'])
     assert_equal(2, data['count'])
     assert_equal(2, data['count'])
-    assert_true(all('query' in doc['name'] for doc in data['children']))
+
+    response = self.client.get('/desktop/api2/doc', {'path': '/', 'text': 'foobar'})
+    data = json.loads(response.content)
+    assert_equal(1, data['count'])
 
 
     # Test pagination with limit
     # Test pagination with limit
     response = self.client.get('/desktop/api2/doc', {'path': '/', 'page': 2, 'limit': 2})
     response = self.client.get('/desktop/api2/doc', {'path': '/', 'page': 2, 'limit': 2})

+ 22 - 6
desktop/libs/notebook/src/notebook/api.py

@@ -310,6 +310,7 @@ def save_notebook(request):
   notebook['id'] = notebook_doc.id
   notebook['id'] = notebook_doc.id
   notebook_doc1 = notebook_doc.doc.get()
   notebook_doc1 = notebook_doc.doc.get()
   notebook_doc.update_data(notebook)
   notebook_doc.update_data(notebook)
+  notebook_doc.search = _get_statement(notebook)
   notebook_doc.name = notebook_doc1.name = notebook['name']
   notebook_doc.name = notebook_doc1.name = notebook['name']
   notebook_doc.description = notebook_doc1.description = notebook['description']
   notebook_doc.description = notebook_doc1.description = notebook['description']
   notebook_doc.save()
   notebook_doc.save()
@@ -350,11 +351,31 @@ def _historify(notebook, user):
 
 
   notebook['uuid'] = history_doc.uuid
   notebook['uuid'] = history_doc.uuid
   history_doc.update_data(notebook)
   history_doc.update_data(notebook)
+  history_doc.search = _get_statement(notebook)
   history_doc.save()
   history_doc.save()
 
 
   return history_doc
   return history_doc
 
 
 
 
+def _set_search_field(notebook_doc):
+  notebook = Notebook(document=notebook_doc).get_data()
+  statement = _get_statement(notebook)
+  notebook_doc.search = statement
+  return notebook_doc
+
+
+def _get_statement(notebook):
+  statement = ''
+  if notebook['snippets'] and len(notebook['snippets']) > 0:
+    try:
+      statement = notebook['snippets'][0]['result']['handle']['statement']
+      if type(statement) == dict:  # Old format
+        statement = notebook['snippets'][0]['statement_raw']
+    except KeyError:  # Old format
+      statement = notebook['snippets'][0]['statement_raw']
+  return statement
+
+
 @require_GET
 @require_GET
 @api_error_handler
 @api_error_handler
 @check_document_access_permission()
 @check_document_access_permission()
@@ -374,12 +395,7 @@ def get_history(request):
   for doc in docs.order_by('-last_modified')[:limit]:
   for doc in docs.order_by('-last_modified')[:limit]:
     notebook = Notebook(document=doc).get_data()
     notebook = Notebook(document=doc).get_data()
     if 'snippets' in notebook:
     if 'snippets' in notebook:
-      try:
-        statement = notebook['snippets'][0]['result']['handle']['statement']
-        if type(statement) == dict: # Old format
-          statement = notebook['snippets'][0]['statement_raw']
-      except KeyError: # Old format
-        statement = notebook['snippets'][0]['statement_raw']
+      statement = _get_statement(notebook)
       history.append({
       history.append({
         'name': doc.name,
         'name': doc.name,
         'id': doc.id,
         'id': doc.id,

+ 4 - 1
desktop/libs/notebook/src/notebook/tests.py

@@ -106,7 +106,7 @@ class TestNotebookApi(object):
         ],
         ],
         "type": "query-hive",
         "type": "query-hive",
         "id": null,
         "id": null,
-        "snippets": [],
+        "snippets": [{"id":"2b7d1f46-17a0-30af-efeb-33d4c29b1055","type":"hive","status":"running","statement":"select * from web_logs","properties":{"settings":[],"files":[],"functions":[]},"result":{"id":"b424befa-f4f5-8799-a0b4-79753f2552b1","type":"table","handle":{"log_context":null,"statements_count":1,"end":{"column":21,"row":0},"statement_id":0,"has_more_statements":false,"start":{"column":0,"row":0},"secret":"rVRWw7YPRGqPT7LZ/TeFaA==an","has_result_set":true,"statement":"select * from web_logs","operation_type":0,"modified_row_count":null,"guid":"7xm6+epkRx6dyvYvGNYePA==an"}},"lastExecuted": 1462554843817,"database":"default"}],
         "uuid": "d9efdee1-ef25-4d43-b8f9-1a170f69a05a"
         "uuid": "d9efdee1-ef25-4d43-b8f9-1a170f69a05a"
     }
     }
     """
     """
@@ -118,6 +118,9 @@ class TestNotebookApi(object):
     doc = Document2.objects.get(pk=data['id'])
     doc = Document2.objects.get(pk=data['id'])
     assert_equal(Document2.objects.get_home_directory(self.user).uuid, doc.parent_directory.uuid)
     assert_equal(Document2.objects.get_home_directory(self.user).uuid, doc.parent_directory.uuid)
 
 
+    # Test that saving a notebook will save the search field to the first statement text
+    assert_equal(doc.search, "select * from web_logs")
+
 
 
   def test_historify(self):
   def test_historify(self):
     # Starts with no history
     # Starts with no history