Quellcode durchsuchen

HUE-3065 [desktop] Allow searching of raw SQL in saved queries

Jenny Kim vor 9 Jahren
Ursprung
Commit
c9b17a2

+ 7 - 4
desktop/core/src/desktop/models.py

@@ -864,7 +864,7 @@ class Document2QueryMixin(object):
       trashed_ids = [doc.id for doc in docs if Document2.TRASH_DIR in doc.path]
       docs = docs.exclude(id__in=trashed_ids)
 
-    return docs.defer('description', 'data', 'extra').distinct().order_by('-last_modified')
+    return docs.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
 
 
   def search_documents(self, types=None, search_text=None, order_by=None):
@@ -880,7 +880,8 @@ class Document2QueryMixin(object):
       documents = documents.filter(type__in=types)
 
     if search_text:
-      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text))
+      documents = documents.filter(Q(name__icontains=search_text) | Q(description__icontains=search_text) |
+                                   Q(search__icontains=search_text))
 
     if order_by:  # TODO: Validate that order_by is a valid sort parameter
       documents = documents.order_by(order_by)
@@ -1001,6 +1002,7 @@ class Document2(models.Model):
 
   data = models.TextField(default='{}')
   extra = models.TextField(default='')
+  search = models.TextField(blank=True, null=True, help_text=_t('Searchable text for the document.'))
   # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access
 
   last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
@@ -1272,7 +1274,7 @@ class Document2(models.Model):
     information like personally identifiable information, that information could be leaked into the Hue database and
     logfiles.
     """
-    if global_redaction_engine.is_enabled() and self.type == 'notebook':
+    if global_redaction_engine.is_enabled() and (self.type == 'notebook' or self.type.startswith('query')):
       data_dict = self.data_dict
       snippets = data_dict.get('snippets', [])
       for snippet in snippets:
@@ -1283,6 +1285,7 @@ class Document2(models.Model):
             snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
             snippet['is_redacted'] = True
       self.data = json.dumps(data_dict)
+      self.search = global_redaction_engine.redact(self.search)
 
   def _contains_cycle(self):
     """
@@ -1345,7 +1348,7 @@ class Directory(Document2):
 
     documents = documents.exclude(is_history=True)
 
-    return documents.defer('description', 'data', 'extra').distinct().order_by('-last_modified')
+    return documents.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')
 
 
   def save(self, *args, **kwargs):

+ 8 - 5
desktop/core/src/desktop/tests_doc2.py

@@ -176,8 +176,8 @@ class TestDocument2(object):
     # Creates 2 directories and 2 queries and saves to home directory
     dir1 = Directory.objects.create(name='test_dir1', owner=self.user)
     dir2 = Directory.objects.create(name='test_dir2', owner=self.user)
-    query1 = Document2.objects.create(name='query1.sql', type='query-hive', owner=self.user, data={})
-    query2 = Document2.objects.create(name='query2.sql', type='query-hive', owner=self.user, data={})
+    query1 = Document2.objects.create(name='query1.sql', type='query-hive', owner=self.user, data={}, search='foobar')
+    query2 = Document2.objects.create(name='query2.sql', type='query-hive', owner=self.user, data={}, search='barfoo')
     children = [dir1, dir2, query1, query2]
 
     self.home_dir.children.add(*children)
@@ -196,11 +196,14 @@ class TestDocument2(object):
     assert_true(all(doc['type'] == 'directory' for doc in data['children']))
 
     # Test search text
-    response = self.client.get('/desktop/api2/doc', {'path': '/', 'text': 'query'})
+    response = self.client.get('/desktop/api2/doc', {'path': '/', 'text': 'foo'})
     data = json.loads(response.content)
-    assert_equal('query', data['text'])
+    assert_equal('foo', data['text'])
     assert_equal(2, data['count'])
-    assert_true(all('query' in doc['name'] for doc in data['children']))
+
+    response = self.client.get('/desktop/api2/doc', {'path': '/', 'text': 'foobar'})
+    data = json.loads(response.content)
+    assert_equal(1, data['count'])
 
     # Test pagination with limit
     response = self.client.get('/desktop/api2/doc', {'path': '/', 'page': 2, 'limit': 2})

+ 22 - 6
desktop/libs/notebook/src/notebook/api.py

@@ -310,6 +310,7 @@ def save_notebook(request):
   notebook['id'] = notebook_doc.id
   notebook_doc1 = notebook_doc.doc.get()
   notebook_doc.update_data(notebook)
+  notebook_doc.search = _get_statement(notebook)
   notebook_doc.name = notebook_doc1.name = notebook['name']
   notebook_doc.description = notebook_doc1.description = notebook['description']
   notebook_doc.save()
@@ -350,11 +351,31 @@ def _historify(notebook, user):
 
   notebook['uuid'] = history_doc.uuid
   history_doc.update_data(notebook)
+  history_doc.search = _get_statement(notebook)
   history_doc.save()
 
   return history_doc
 
 
+def _set_search_field(notebook_doc):
+  notebook = Notebook(document=notebook_doc).get_data()
+  statement = _get_statement(notebook)
+  notebook_doc.search = statement
+  return notebook_doc
+
+
+def _get_statement(notebook):
+  statement = ''
+  if notebook['snippets'] and len(notebook['snippets']) > 0:
+    try:
+      statement = notebook['snippets'][0]['result']['handle']['statement']
+      if type(statement) == dict:  # Old format
+        statement = notebook['snippets'][0]['statement_raw']
+    except KeyError:  # Old format
+      statement = notebook['snippets'][0]['statement_raw']
+  return statement
+
+
 @require_GET
 @api_error_handler
 @check_document_access_permission()
@@ -374,12 +395,7 @@ def get_history(request):
   for doc in docs.order_by('-last_modified')[:limit]:
     notebook = Notebook(document=doc).get_data()
     if 'snippets' in notebook:
-      try:
-        statement = notebook['snippets'][0]['result']['handle']['statement']
-        if type(statement) == dict: # Old format
-          statement = notebook['snippets'][0]['statement_raw']
-      except KeyError: # Old format
-        statement = notebook['snippets'][0]['statement_raw']
+      statement = _get_statement(notebook)
       history.append({
         'name': doc.name,
         'id': doc.id,

+ 4 - 1
desktop/libs/notebook/src/notebook/tests.py

@@ -106,7 +106,7 @@ class TestNotebookApi(object):
         ],
         "type": "query-hive",
         "id": null,
-        "snippets": [],
+        "snippets": [{"id":"2b7d1f46-17a0-30af-efeb-33d4c29b1055","type":"hive","status":"running","statement":"select * from web_logs","properties":{"settings":[],"files":[],"functions":[]},"result":{"id":"b424befa-f4f5-8799-a0b4-79753f2552b1","type":"table","handle":{"log_context":null,"statements_count":1,"end":{"column":21,"row":0},"statement_id":0,"has_more_statements":false,"start":{"column":0,"row":0},"secret":"rVRWw7YPRGqPT7LZ/TeFaA==an","has_result_set":true,"statement":"select * from web_logs","operation_type":0,"modified_row_count":null,"guid":"7xm6+epkRx6dyvYvGNYePA==an"}},"lastExecuted": 1462554843817,"database":"default"}],
         "uuid": "d9efdee1-ef25-4d43-b8f9-1a170f69a05a"
     }
     """
@@ -118,6 +118,9 @@ class TestNotebookApi(object):
     doc = Document2.objects.get(pk=data['id'])
     assert_equal(Document2.objects.get_home_directory(self.user).uuid, doc.parent_directory.uuid)
 
+    # Test that saving a notebook will save the search field to the first statement text
+    assert_equal(doc.search, "select * from web_logs")
+
 
   def test_historify(self):
     # Starts with no history