Переглянути джерело

HUE-8610 [core] Always send the full cluster instead of id to the APIs

Johan Ahlen 7 роки тому
батько
коміт
a537d64

+ 3 - 2
apps/beeswax/src/beeswax/api.py

@@ -671,7 +671,7 @@ def _get_sample_data(db, database, table, column, async=False, cluster=None, ope
       notebook = make_notebook(
           name=_('Table sample for `%(database)s`.`%(table)s`.`%(column)s`') % {'database': database, 'table': table, 'column': column},
           editor_type=_get_servername(db),
-          compute=cluster.get('id') if cluster else None,
+          compute=cluster,
           statement=sample_data,
           status='ready-execute',
           skip_historify=True,
@@ -776,7 +776,8 @@ def analyze_table(request, database, table, columns=None):
 @error_handler
 def get_table_stats(request, database, table, column=None):
   app_name = get_app_name(request)
-  query_server = get_query_server_config(app_name)
+  cluster = json.loads(request.GET.get('cluster', '{}'))
+  query_server = get_query_server_config(app_name, cluster=cluster)
   db = dbms.get(request.user, query_server)
 
   response = {'status': -1, 'message': '', 'redirect': ''}

+ 2 - 2
apps/impala/src/impala/api.py

@@ -18,6 +18,7 @@
 ## Main views are inherited from Beeswax.
 
 import logging
+import json
 
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_POST
@@ -37,10 +38,9 @@ LOG = logging.getLogger(__name__)
 @require_POST
 @error_handler
 def invalidate(request):
-  cluster = {'id': request.POST.get('cluster', 'default')}
+  cluster = json.loads(request.POST.get('cluster', '{}'))
 
   # TODO: create cluster_config for get_query_server_config
-
   query_server = dbms.get_query_server_config()
   db = beeswax_dbms.get(request.user, query_server=query_server, cluster=cluster)
 

+ 2 - 1
apps/metastore/src/metastore/static/metastore/js/metastore.model.js

@@ -949,7 +949,8 @@ var MetastoreTable = (function () {
       $.post('/tables/drop/' + self.database.catalogEntry.name, {
         table_selection: ko.mapping.toJSON([self.name]),
         skip_trash: 'off',
-        is_embeddable: true
+        is_embeddable: true,
+        cluster: JSON.stringify(self.database.catalogEntry.compute)
       }, function(resp) {
         if (resp.history_uuid) {
           huePubSub.publish('notebook.task.submitted', resp.history_uuid);

+ 4 - 4
apps/metastore/src/metastore/templates/metastore.mako

@@ -544,7 +544,7 @@ ${ components.menubar(is_embeddable) }
         <input type="hidden" name="start_time" value=""/>
         <input type="hidden" name="source_type" data-bind="value: $root.source().type"/>
         <input type="hidden" name="namespace" data-bind="value: catalogEntry.namespace.id"/>
-        <input type="hidden" name="compute" data-bind="value: catalogEntry.compute.id"/>
+        <input type="hidden" name="cluster" data-bind="value: JSON.stringify(catalogEntry.compute)"/>
     % else:
       <form data-bind="attr: { 'action': '/metastore/tables/drop/' + catalogEntry.name }" method="POST">
     % endif
@@ -1126,12 +1126,12 @@ ${ components.menubar(is_embeddable) }
     });
   }
 
-  function queryAndWatchUrl(url, sourceType, namespaceId, computeId) {
+  function queryAndWatchUrl(url, sourceType, namespaceId, compute) {
     $.post(url, {
       format: "json",
       sourceType: sourceType,
       namespace: namespaceId,
-      compute: computeId
+      cluster: compute
     },function(resp) {
       if (resp.history_uuid) {
         huePubSub.publish('open.editor.query', resp.history_uuid);
@@ -1148,7 +1148,7 @@ ${ components.menubar(is_embeddable) }
       location.href = '/notebook/browse/' + catalogEntry.path.join('/')
     } else {
       queryAndWatchUrl('/notebook/browse/' + catalogEntry.path.join('/') + '/', catalogEntry.getSourceType(),
-              catalogEntry.namespace && catalogEntry.namespace.id, catalogEntry.compute && catalogEntry.compute.id)
+              catalogEntry.namespace && catalogEntry.namespace.id, catalogEntry.compute)
     }
   }
 

+ 5 - 4
apps/metastore/src/metastore/views.py

@@ -161,7 +161,8 @@ def alter_database(request, database):
 def get_database_metadata(request, database):
   response = {'status': -1, 'data': ''}
   source_type = request.POST.get('source_type', 'hive')
-  cluster = {'id': request.GET.get('cluster', 'default')}
+  cluster = json.loads(request.POST.get('cluster', '{}'))
+
   db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
 
   try:
@@ -268,7 +269,7 @@ def get_table_metadata(request, database, table):
 
 def describe_table(request, database, table):
   app_name = get_app_name(request)
-  cluster = {'id': request.GET.get('cluster', 'default')}
+  cluster = json.loads(request.GET.get('cluster', '{}'))
 
   db = _get_db(user=request.user, cluster=cluster)
 
@@ -405,7 +406,7 @@ def drop_table(request, database):
       tables = request.POST.getlist('table_selection')
       tables_objects = [db.get_table(database, table) for table in tables]
       skip_trash = request.POST.get('skip_trash') == 'on'
-      compute = request.POST.get('compute')
+      cluster = json.loads(request.POST.get('cluster', '{}'))
       namespace = request.POST.get('namespace')
 
       if request.POST.get('is_embeddable'):
@@ -418,7 +419,7 @@ def drop_table(request, database):
             status='ready',
             database=database,
             namespace=namespace,
-            compute=compute,
+            compute=cluster,
             on_success_url='assist.db.refresh',
             is_task=True,
             last_executed=last_executed

+ 9 - 11
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -1275,7 +1275,7 @@ var ApiHelper = (function () {
     if (options.sourceType === 'impala' && (options.invalidate === 'invalidate' || options.invalidate === 'invalidateAndFlush')) {
       var data = {
         flush_all: options.invalidate === 'invalidateAndFlush',
-        cluster: options.compute && options.compute.id
+        cluster: JSON.stringify(options.compute)
       };
 
       if (options.path && options.path.length > 0) {
@@ -1393,8 +1393,8 @@ var ApiHelper = (function () {
    *
    * @param {Object} options
    * @param {boolean} [options.silenceErrors]
-   * @param {ContextCompute} [options.compute]
    *
+   * @param {ContextCompute} options.compute
    * @param {string[]} options.path
    *
    * @return {CancellablePromise}
@@ -1419,16 +1419,13 @@ var ApiHelper = (function () {
       }
     }
 
-    var params = {
-      'format' : 'json'
+    var data = {
+      format: 'json',
+      cluster: JSON.stringify(options.compute)
       //'source_type': options.sourceType // TODO: Blows up server-side with 'impala'
     };
 
-    if (options.compute && options.compute.id) {
-      params['cluster'] = options.compute.id;
-    }
-
-    var request = self.simpleGet(url, params, {
+    var request = self[options.path.length === 1 ? 'simplePost' : 'simpleGet'](url, data, {
       silenceErrors: options.silenceErrors,
       successCallback: function (response) {
         if (options.path.length === 1) {
@@ -1507,6 +1504,7 @@ var ApiHelper = (function () {
    * @param {boolean} [options.silenceErrors]
    *
    * @param {string} options.sourceType
+   * @param {ContextCompute} options.compute
    * @param {string[]} options.path
    *
    * @return {CancellablePromise}
@@ -1984,7 +1982,7 @@ var ApiHelper = (function () {
   /**
    * @param {Object} options
    * @param {boolean} [options.silenceErrors]
-   * @param {string} options.computeId
+   * @param {ContextCompute} options.compute
    * @param {string} options.queryId
    * @return {CancellablePromise}
    */
@@ -2007,7 +2005,7 @@ var ApiHelper = (function () {
       tries++;
       cancellablePromises.pop(); // Remove the last one
       cancellablePromises.push(deferred, self.simplePost(url, {
-        'cluster_id': '"' + options.computeId + '"',
+        'cluster': JSON.stringify(options.compute),
         'query_id': '"' + options.queryId + '"'
       }, options).done(function (response) {
         if (response && response.data) {

+ 1 - 1
desktop/core/src/desktop/templates/assist.mako

@@ -3220,7 +3220,7 @@ from desktop.views import _ko
           self.loadingExecutionAnalysis(true);
           lastExecutionAnalysisPromise = ApiHelper.getInstance().fetchQueryExecutionAnalysis({
             silenceErrors: true,
-            computeId: details.computeId,
+            compute: details.compute,
             queryId: details.queryId
           }).done(function (response) {
             self.executionAnalysis(response.query)

+ 2 - 2
desktop/libs/metadata/src/metadata/workload_analytics_api.py

@@ -49,11 +49,11 @@ def error_handler(view_fn):
 def get_impala_query(request):
   response = {'status': -1}
 
-  cluster_id = json.loads(request.POST.get('cluster_id'))
+  cluster = json.loads(request.POST.get('cluster'))
   query_id = json.loads(request.POST.get('query_id'))
 
   client = WorkfloadAnalyticsClient(request.user)
-  data = client.get_impala_query(cluster_id=cluster_id, query_id=query_id)
+  data = client.get_impala_query(cluster=cluster, query_id=query_id)
 
   if data:
     response['status'] = 0

+ 4 - 4
desktop/libs/metadata/src/metadata/workload_analytics_client.py

@@ -30,8 +30,8 @@ class WorkfloadAnalyticsClient():
   def __init__(self, user):
     self.user = user
 
-  def get_impala_query(self, cluster_id, query_id):
-    return WorkloadAnalytics(self.user).get_impala_query(cluster_id=cluster_id, query_id=query_id)
+  def get_impala_query(self, cluster, query_id):
+    return WorkloadAnalytics(self.user).get_impala_query(cluster=cluster, query_id=query_id)
 
   def list_uploads(self):
     return WorkloadAnalytics(self.user).list_uploads()
@@ -51,8 +51,8 @@ class WorkloadAnalytics():
 
   def __init__(self, user): pass
 
-  def get_impala_query(self, cluster_id, query_id):
-    parameters = {'clusterId': cluster_id, 'queryId': query_id}
+  def get_impala_query(self, cluster, query_id):
+    parameters = {'clusterId': cluster.get('id'), 'queryId': query_id}
 
     return _exec('wa', 'getImpalaQuery', parameters=parameters)
 

+ 2 - 4
desktop/libs/notebook/src/notebook/models.py

@@ -59,7 +59,7 @@ def escape_rows(rows, nulls_only=False, encoding=None):
 def make_notebook(name='Browse', description='', editor_type='hive', statement='', status='ready',
                   files=None, functions=None, settings=None, is_saved=False, database='default', snippet_properties=None, batch_submit=False,
                   on_success_url=None, skip_historify=False, is_task=False, last_executed=-1, is_notebook=False, pub_sub_url=None, result_properties={},
-                  namespace='default', compute='default'):
+                  namespace='default', compute={'id':'default'}):
   '''
   skip_historify: do not add the task to the query history. e.g. SQL Dashboard
   isManaged: true when being a managed by Hue operation (include_managed=True in document), e.g. exporting query result, dropping some tables
@@ -131,9 +131,7 @@ def make_notebook(name='Browse', description='', editor_type='hive', statement='
          'namespace': {
             'id': namespace
          },
-         'compute': {
-            'id': compute
-         },
+         'compute': compute,
          'result': {'handle':{}},
          'variables': []
       }

+ 3 - 3
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -1860,11 +1860,11 @@ var EditorViewModel = (function() {
                 self.checkStatusTimeout = setTimeout(self.checkStatus, delay);
               }
             } else if (self.status() === 'available') {
-              if (self.type() === 'impala') {
+              if (self.type() === 'impala' && self.compute() && self.compute().crn && self.compute().crn.indexOf('altus') !== -1) {
 
-                // TODO: Use real compute and query ID
+                // TODO: Use real query ID
                 huePubSub.publish('assist.update.execution.analysis', {
-                  computeId: '6bfa86a8-55a2-4466-9003-2b222a9be137', // TODO: Compute ID ?
+                  compute: self.compute(),
                   queryId: '56433486cd84d475:3a86f97000000000'
                 });
 

+ 1 - 1
desktop/libs/notebook/src/notebook/views.py

@@ -155,7 +155,7 @@ def browse(request, database, table, partition_spec=None):
   statement = get_api(request, snippet).get_browse_query(snippet, database, table, partition_spec)
   editor_type = snippet['type']
   namespace = request.POST.get('namespace', 'default')
-  compute = request.POST.get('compute', 'default')
+  compute = json.loads(request.POST.get('cluster', '{}'))
 
   if request.method == 'POST':
     notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=statement, status='ready-execute',