Sfoglia il codice sorgente

HUE-9249 [browser] API to download a query profile

Note: need to see how to avoid closing the notebook query on download click.
Romain 5 anni fa
parent
commit
857bd707cd

+ 6 - 2
apps/beeswax/src/beeswax/data_export.py

@@ -163,11 +163,15 @@ class DataAdapter(object):
       for row in results.rows():
         num_bytes = self._getsizeofascii(row)
         if self.limit_rows and self.row_counter + 1 > self.max_rows:
-          LOG.warn('The query results exceeded the maximum row limit of %d and has been truncated to first %d rows.' % (self.max_rows, self.row_counter))
+          LOG.warn('The query results exceeded the maximum row limit of %d and has been truncated to first %d rows.' % (
+              self.max_rows, self.row_counter)
+          )
           self.is_truncated = True
           break
         if self.limit_bytes and self.bytes_counter + num_bytes > self.max_bytes:
-          LOG.warn('The query results exceeded the maximum bytes limit of %d and has been truncated to first %d rows.' % (self.max_bytes, self.row_counter))
+          LOG.warn('The query results exceeded the maximum bytes limit of %d and has been truncated to first %d rows.' % (
+              self.max_bytes, self.row_counter)
+          )
           self.is_truncated = True
           break
         self.row_counter += 1

+ 9 - 3
apps/jobbrowser/src/jobbrowser/api2.py

@@ -18,6 +18,8 @@
 import json
 import logging
 
+from django.http import HttpResponse
+
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.django_util import JsonResponse
 from django.utils.translation import ugettext as _
@@ -135,7 +137,11 @@ def profile(request):
   api = get_api(request.user, interface, cluster=cluster)
   api._set_request(request) # For YARN
 
-  response[app_property] = api.profile(app_id, app_type, app_property, app_filters)
-  response['status'] = 0
+  resp = api.profile(app_id, app_type, app_property, app_filters)
 
-  return JsonResponse(response)
+  if isinstance(resp, HttpResponse):
+    return resp
+  else:
+    response[app_property] = resp
+    response['status'] = 0
+    return JsonResponse(response)

+ 6 - 2
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -25,10 +25,12 @@ from datetime import datetime
 
 from django.utils.translation import ugettext as _
 
-from jobbrowser.apis.base_api import Api
+from desktop.lib import export_csvxls
 from libanalyze import analyze as analyzer, rules
 from notebook.conf import ENABLE_QUERY_ANALYSIS
 
+from jobbrowser.apis.base_api import Api
+
 ANALYZER = rules.TopDownAnalysis() # We need to parse some files so save as global
 LOG = logging.getLogger(__name__)
 
@@ -157,7 +159,6 @@ class QueryApi(Api):
 
     return message;
 
-
   def logs(self, appid, app_type, log_name=None, is_embeddable=False):
     return {'logs': ''}
 
@@ -166,6 +167,8 @@ class QueryApi(Api):
       return self._memory(appid, app_type, app_property, app_filters)
     elif app_property == 'profile':
       return self._query_profile(appid)
+    elif app_property == 'download-profile':
+      return export_csvxls.make_response([self._query_profile(appid)['profile']], 'txt', 'query-profile_%s' % appid)
     elif app_property == 'backends':
       return self._query_backends(appid)
     elif app_property == 'finstances':
@@ -173,6 +176,7 @@ class QueryApi(Api):
     else:
       return self._query(appid)
 
+
   def profile_encoded(self, appid):
     return self.api.get_query_profile_encoded(query_id=appid)
 

+ 62 - 0
apps/jobbrowser/src/jobbrowser/apis/query_api_tests.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import sys
+
+from django.urls import reverse
+from nose.plugins.skip import SkipTest
+from nose.tools import assert_equal, assert_true
+
+from desktop.auth.backend import rewrite_user
+from desktop.lib.django_test_util import make_logged_in_client
+from useradmin.models import User
+
+from jobbrowser.apis.query_api import QueryApi
+
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock
+else:
+  from mock import patch, Mock
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TestApi():
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
+    self.user = rewrite_user(User.objects.get(username="test"))
+
+
+  def test_download_profile(self):
+    with patch('jobbrowser.apis.query_api._get_api') as _get_api:
+      with patch('jobbrowser.apis.query_api.QueryApi._query_profile') as _query_profile:
+        _query_profile.return_value = {'profile': 'Query (id=d94d2fb4815a05c4:b1ccec1500000000):\n  Summary:...'}
+
+        appid = '00001'
+        app_type = Mock()
+        app_filters = []
+
+        resp = QueryApi(self.user).profile(appid, app_type, 'download-profile', app_filters)
+
+        assert_equal(resp.status_code, 200)
+        assert_equal(resp['Content-Disposition'], 'attachment; filename="query-profile_00001.txt"')
+        assert_equal(resp.content, b'Query (id=d94d2fb4815a05c4:b1ccec1500000000):\n  Summary:...')

+ 44 - 6
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -1637,14 +1637,19 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           <pre data-bind="text: (properties.plan && properties.plan().summary) || _('The selected tab has no data')"/>
         </div>
         <div class="tab-pane" id="queries-page-profile${ SUFFIX }" data-profile="profile">
-          <button class="btn" type="button" data-clipboard-target="#query-impala-profile" data-bind="
+          <button class="btn" type="button" data-clipboard-target="#query-impala-profile" style="float: right;" data-bind="
+              visible: properties.profile && properties.profile().profile,
               clipboard: { onSuccess: function() { $.jHueNotify.info('${ _("Profile copied to clipboard!") }'); } }">
             <i class="fa fa-fw fa-clipboard"></i> ${ _('Clipboard') }
           </button>
-          <a class="btn" href="/desktop/download_logs" download>
+          <button class="btn" type="button" style="float: right;" data-bind="
+              click: function(){ submitQueryProfileDownloadForm('download-profile'); },
+              visible: properties.profile && properties.profile().profile">
             <i class="fa fa-fw fa-download"></i> ${ _('Download') }
-          </a>
-          <pre id="query-impala-profile" data-bind="text: (properties.profile && properties.profile().profile) || _('The selected tab has no data')"/>
+          </button>
+          <div id="downloadProgressModal"></div>
+          <pre id="query-impala-profile" style="float: left; margin-top: 8px" data-bind="
+              text: (properties.profile && properties.profile().profile) || _('The selected tab has no data')"/>
         </div>
         <div class="tab-pane" id="queries-page-memory${ SUFFIX }" data-profile="mem_usage">
           <pre data-bind="text: (properties.memory && properties.memory().mem_usage) || _('The selected tab has no data')"/>
@@ -1756,7 +1761,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         </ul>
       </div>
     </div>
-    <div data-bind="css:{'span10': !$root.isMini(), 'span12 no-margin': $root.isMini() }">
+    <div data-bind="css:{ 'span10': !$root.isMini(), 'span12 no-margin': $root.isMini() }">
 
       <ul class="nav nav-pills margin-top-20">
         <li>
@@ -3009,6 +3014,39 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         return lastFetchLogsRequest;
       };
 
+      self.submitQueryProfileDownloadForm = function(name) {
+        var $downloadForm = $(
+          '<form method="POST" class="download-form" style="display: inline" action="' +
+            window.HUE_BASE_URL +
+            '/jobbrowser/api/job/profile"></form>'
+        );
+
+        $('<input type="hidden" name="csrfmiddlewaretoken" />')
+          .val(window.CSRF_TOKEN)
+          .appendTo($downloadForm);
+        $('<input type="hidden" name="cluster" />')
+          .val(ko.mapping.toJSON(vm.compute))
+          .appendTo($downloadForm);
+        $('<input type="hidden" name="app_id" />')
+          .val(ko.mapping.toJSON(self.id))
+          .appendTo($downloadForm);
+        $('<input type="hidden" name="interface" />')
+          .val(ko.mapping.toJSON(vm.interface))
+          .appendTo($downloadForm);
+        $('<input type="hidden" name="app_type" />')
+          .val(ko.mapping.toJSON(self.type))
+          .appendTo($downloadForm);
+        $('<input type="hidden" name="app_property" />')
+          .val(ko.mapping.toJSON(name))
+          .appendTo($downloadForm);
+        $('<input type="hidden" name="app_filters" />')
+          .val(ko.mapping.toJSON(self.filters))
+          .appendTo($downloadForm);
+
+        $('#downloadProgressModal').append($downloadForm);
+        $downloadForm.submit();
+      }
+
       self.fetchProfile = function (name, callback) {
         vm.apiHelper.cancelActiveRequest(lastFetchProfileRequest);
         lastFetchProfileRequest = $.post("/jobbrowser/api/job/profile", {
@@ -3642,7 +3680,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           return '${ is_mini }' == 'False' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('pyspark') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
         };
         var queryInterfaceCondition = function () {
-          return true || '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('impala') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
+          return '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('impala') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
         };
         var queryHiveInterfaceCondition = function () {
           return '${ ENABLE_HIVE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('hive') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);

+ 1 - 1
desktop/core/src/desktop/js/apps/notebook2/app.js

@@ -359,7 +359,7 @@ export const initNotebook2 = () => {
       // Close the notebook snippets when leaving the page
       window.onbeforeunload = function(e) {
         if (!viewModel.selectedNotebook().avoidClosing) {
-          viewModel.selectedNotebook().close();
+          //viewModel.selectedNotebook().close(); // TODO
         }
       };
       $(window).data('beforeunload', window.onbeforeunload);

+ 9 - 4
desktop/core/src/desktop/lib/export_csvxls.py

@@ -20,8 +20,7 @@ Common library to export either CSV or XLS.
 """
 from future import standard_library
 standard_library.install_aliases()
-from builtins import next
-from builtins import object
+from builtins import next, object
 import gc
 import logging
 import numbers
@@ -41,11 +40,17 @@ if sys.version_info[0] > 2:
 else:
   from StringIO import StringIO as string_io
 
+
 LOG = logging.getLogger(__name__)
 
 DOWNLOAD_CHUNK_SIZE = 1 * 1024 * 1024 # 1MB
 ILLEGAL_CHARS = r'[\000-\010]|[\013-\014]|[\016-\037]'
-FORMAT_TO_CONTENT_TYPE = {'csv': 'application/csv', 'xls': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'json': 'application/json'}
+FORMAT_TO_CONTENT_TYPE = {
+    'csv': 'application/csv',
+    'xls': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    'json': 'application/json'
+}
+
 
 def nullify(cell):
   return cell if cell is not None else "NULL"
@@ -147,7 +152,7 @@ def make_response(generator, format, name, encoding=None, user_agent=None): #TOD
   elif format == 'xls':
     format = 'xlsx'
     resp = HttpResponse(next(generator), content_type=content_type)
-  elif format == 'json':
+  elif format == 'json' or format == 'txt':
     resp = HttpResponse(generator, content_type=content_type)
   else:
     raise Exception("Unknown format: %s" % format)

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -497,7 +497,7 @@ class HS2Api(Api):
           }
           for job in queries_with_state
         ]
-    elif snippet['dialect'] == 'impala' and has_query_browser or True:
+    elif snippet['dialect'] == 'impala' and has_query_browser:
       guid = snippet['result']['handle']['guid']
       if isinstance(guid, str):
         guid = guid.encode('utf-8')

+ 1 - 1
package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "gethue",
-  "version": "4.6.3",
+  "version": "4.7.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {