فهرست منبع

HUE-7877 [dashboard] Plug-in subquery metadata and submission logic

Romain Rigaux 7 سال پیش
والد
کامیت
ef977286b5

+ 5 - 4
apps/beeswax/src/beeswax/api.py

@@ -98,7 +98,7 @@ def autocomplete(request, database=None, table=None, column=None, nested=None):
   return JsonResponse(response)
 
 
-def _autocomplete(db, database=None, table=None, column=None, nested=None):
+def _autocomplete(db, database=None, table=None, column=None, nested=None, query=None):
   response = {}
 
   try:
@@ -108,11 +108,12 @@ def _autocomplete(db, database=None, table=None, column=None, nested=None):
       tables_meta = db.get_tables_meta(database=database)
       response['tables_meta'] = tables_meta
     elif column is None:
-      if False:
+      print 'aaaaaaaaaaaaaaaaaaaaaaaaaa ', query
+      if query is not None:
         class SubQueryTable():
           def __init__(self, db, query):
             self.query = query
-            # Table Properties below            
+            # Table Properties            
             self.name = 'Test'
             # TODO Replace 't.', type different too?
             self.cols =  db.get_query_metadata(query).data_table.cols()
@@ -122,7 +123,7 @@ def _autocomplete(db, database=None, table=None, column=None, nested=None):
             self.is_view = False
             self.partition_keys = []
             self.properties = {}
-        table = SubQueryTable(db, 'SELECT app, bytes, concat(\'http://\', url) as url2 FROM web_logs')
+        table = SubQueryTable(db, query)
       else:
         table = db.get_table(database, table)
       response['hdfs_link'] = table.hdfs_link

+ 1 - 1
desktop/core/src/desktop/templates/common_notebook_ko_components.mako

@@ -410,7 +410,7 @@ except ImportError, e:
             if (resp.status == 0) {
               if (IS_HUE_4) {
                 $(".modal-backdrop").remove();
-                if (self.saveTarget() == 'hdfs-file' || (self.saveTarget() == 'search-index' && typeof resp.rowcount !== 'undefined')) {
+                if (self.saveTarget() == 'hdfs-file' || self.saveTarget() == 'search-index') {
                   $("#saveResultsModal").modal("hide");
                   huePubSub.publish('open.link', resp.watch_url);
                 } else {

+ 4 - 1
desktop/libs/dashboard/src/dashboard/api.py

@@ -549,8 +549,11 @@ def get_collection(request):
   try:
     name = request.POST['name']
     engine = request.POST['engine']
+    source = request.POST.get('source')
 
-    collection = Collection2(request.user, name=name, engine=engine)
+    source = 'query'
+
+    collection = Collection2(request.user, name=name, engine=engine, source=source)
     collection_json = collection.get_json(request.user)
 
     result['collection'] = json.loads(collection_json)

+ 3 - 1
desktop/libs/dashboard/src/dashboard/dashboard_api.py

@@ -22,7 +22,9 @@ LOG = logging.getLogger(__name__)
 
 
 def get_engine(user, engine='solr', facet=None, source='data'):
-  if isinstance(engine, dict):
+  if isinstance(engine, dict):    
+    if source == 'data':
+      source = engine.get('source')
     engine = engine.get('engine', 'solr')
 
   if engine == 'report' and facet:

+ 1 - 1
desktop/libs/dashboard/src/dashboard/models.py

@@ -171,7 +171,7 @@ class Collection2(object):
     return props
 
   def get_default(self, user, name, engine='solr', source='data'):
-    fields = self.fields_data(user, name, engine)
+    fields = self.fields_data(user, name, engine, source=source)
     id_field = [field['name'] for field in fields if field.get('isId')]
 
     if id_field:

+ 1 - 0
desktop/libs/dashboard/src/dashboard/static/dashboard/js/search.ko.js

@@ -515,6 +515,7 @@ var Collection = function (vm, collection) {
   self.engine.subscribe(function() {
     self.name(null);
   });
+  self.source = ko.observable(typeof collection.source != "undefined" && collection.source != null ? collection.source : "data");
   self.async = ko.computed(function() {
     return ['impala', 'hive'].indexOf(self.engine()) != -1;
   });

+ 1 - 2
desktop/libs/dashboard/src/dashboard/views.py

@@ -157,8 +157,7 @@ def new_search(request):
 def browse(request, name, is_mobile=False):
   engine = request.GET.get('engine', 'solr')
   source = request.GET.get('source', 'data')
-  # get engine
-  # get name
+  engine = 'hive'
   collections = get_engine(request.user, engine, source=source).datasets()
   if not collections and engine == 'solr':
     return no_collections(request)

+ 5 - 2
desktop/libs/notebook/src/notebook/api.py

@@ -624,7 +624,7 @@ def format(request):
 @check_document_access_permission()
 @api_error_handler
 def export_result(request):
-  response = {'status': -1, 'message': _('Exporting result failed.')}
+  response = {'status': -1, 'message': _('Success')}
 
   # Passed by check_document_access_permission but unused by APIs
   notebook = json.loads(request.POST.get('notebook', '{}'))
@@ -708,7 +708,7 @@ def export_result(request):
     if is_embedded:
       if destination == '__hue__':
         notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
-        response['watch_url'] = reverse('search:browse', kwargs={'name': notebook_id, 'source': 'query'})
+        response['watch_url'] = reverse('dashboard:browse', kwargs={'name': notebook_id}) + '?source=query'
         response['status'] = 0        
         return JsonResponse(response)
 
@@ -747,6 +747,9 @@ def export_result(request):
       response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=index_query&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
       response['status'] = 0
 
+    if response.get('status') != 0:
+      response['message'] =  _('Exporting result failed.')
+
     request.audit = {
       'operation': 'EXPORT',
       'operationText': 'User %s exported to Search index: %s' % (request.user.username, destination),

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -444,7 +444,7 @@ class HS2Api(Api):
   @query_error_handler
   def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
     db = self._get_db(snippet)
-    return _autocomplete(db, database, table, column, nested)
+    return _autocomplete(db, database, table, column, nested, query=snippet.get('query'))
 
 
   @query_error_handler

+ 23 - 11
desktop/libs/notebook/src/notebook/dashboard_api.py

@@ -93,7 +93,7 @@ class SQLDashboardApi(DashboardApi):
             mincount_field_name = 'count__' + '_'.join(mincount_fields_name)
             mincount_fields_operation.append('COUNT(*) OVER (PARTITION BY %s) AS %s' % (', '.join(mincount_fields_name), mincount_field_name) )
             mincount_where.append('%s >= %s' % (mincount_field_name, str(f['mincount'])))
-          sql_mincount = '''(SELECT * FROM (SELECT *, %(fields)s
+          sql_from = '''(SELECT * FROM (SELECT *, %(fields)s
           FROM %(database)s.%(table)s) default
           WHERE %(where)s) default''' % {
             'fields': ', '.join(mincount_fields_operation),
@@ -102,7 +102,7 @@ class SQLDashboardApi(DashboardApi):
             'where': ' AND '.join(mincount_where)
           }
         else:
-          sql_mincount = '%(database)s.%(table)s' % {
+          sql_from = '%(database)s.%(table)s' % {
             'database': database,
             'table': table
           }
@@ -110,14 +110,12 @@ class SQLDashboardApi(DashboardApi):
         order_by = ', '.join([self._get_dimension_field(f)['order_by'] for f in reversed(facet['properties']['facets']) if f['sort'] != 'default'])
 
         sql = '''SELECT %(fields)s
-        FROM %(sql_mincount)s
+        FROM %(sql_from)s
         %(filters)s
         GROUP BY %(fields_dimensions)s
         %(order_by)s
         LIMIT %(limit)s''' % {
-            'sql_mincount': sql_mincount,
-            'database': database,
-            'table': table,
+            'sql_from': sql_from,
             'fields': ', '.join(fields),
             'fields_dimensions': ', '.join(fields_dimensions),
             'order_by': 'ORDER BY %s' % order_by if order_by else '',
@@ -150,8 +148,6 @@ class SQLDashboardApi(DashboardApi):
         sql = '''SELECT %(fields)s
         FROM %(sql_from)s
         %(filters)s''' % {
-            'database': database,
-            'table': table,
             'sql_from': sql_from,
             'fields': self._get_aggregate_function(facet['properties']['facets'][0]),
             'filters': self._convert_filters_to_where(filters),
@@ -159,10 +155,20 @@ class SQLDashboardApi(DashboardApi):
       elif facet['type'] == 'statement':
         sql = facet['properties']['statement']
     else:
+      print '==================================================================== ', self.source
+      print '===================================================================='
       fields = Collection2.get_field_list(dashboard)
-      sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
+      if self.source == 'query':
+        # Open snippet and get statement
+        sql_from = '(select app from web_logs) t'
+        database, table = '', ''
+      else:
+        sql_from = '`%(database)s`.`%(table)s`' % {
           'database': database,
-          'table': table,
+          'table': table
+        }
+      sql = "SELECT %(fields)s FROM %(sql_from)s" % {
+          'sql_from': sql_from,
           'fields': ', '.join(['`%s`' % f if f != '*' else '*' for f in fields])
       }
       if filters:
@@ -217,9 +223,15 @@ class SQLDashboardApi(DashboardApi):
   def fields(self, name):
     # self.source
     # get query if source == 'query' + default DB
-    database, table = self._get_database_table_names(name)
     snippet = {'type': self.engine}
 
+    if self.source == 'query':
+      # Open snippet and get statement
+      snippet['query'] = 'select app from web_logs'
+      database, table = '', ''
+    else:
+      database, table = self._get_database_table_names(name)
+
     table_metadata = get_api(MockRequest(self.user), snippet).autocomplete(snippet, database, table)
 
     return {