Bläddra i källkod

HUE-6296 [optimizer] Upload past query history as example

Romain Rigaux 8 år sedan
förälder
incheckning
4e23f53ae0

+ 12 - 1
apps/about/src/about/templates/admin_wizard.mako

@@ -15,10 +15,12 @@
 ## limitations under the License.
 
 <%!
-from desktop.views import commonheader, commonfooter
 from django.core.urlresolvers import reverse
 from django.utils.encoding import smart_unicode
 from django.utils.translation import ugettext as _
+
+from desktop.views import commonheader, commonfooter
+from metadata.conf import OPTIMIZER
 %>
 
 <%namespace name="layout" file="/about_layout.mako" />
@@ -26,6 +28,7 @@ from django.utils.translation import ugettext as _
 %if not is_embeddable:
 ${ commonheader(_('Quick Start'), "quickstart", user, request, "70px") | n,unicode }
 %endif
+
 ${ layout.menubar(section='quick_start') }
 
 <div class="container-fluid">
@@ -87,6 +90,14 @@ ${ layout.menubar(section='quick_start') }
                     </a>
                   </li>
               % endif
+              % if OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get() > 0:
+                  <li>
+                    <a href="javascript:void(0)" class="installBtn" data-loading-text="${ _('Uploading...') }"
+                       data-sample-url="${ url('metadata:upload_history') }" title="${ _('Send and analyze past %s executed queries to provide smarter SQL recommendations') % OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get() }">
+                      <i class="fa fa-upload"></i> ${ _('SQL Query history') }
+                    </a>
+                  </li>
+              % endif
               % if 'search' in app_names:
                   <li>
                     <a href="javascript:void(0)" class="installBtn" data-loading-text="${ _('Installing...') }"

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1644,6 +1644,9 @@
     # Automatically upload queries after their execution in order to improve recommendations.
     ## auto_upload_queries=true
 
+    # Allow admins to upload the last N executed queries in the quick start wizard. Use 0 to disable.
+    ## query_history_upload_limit=10000
+
     # In secure mode (HTTPS), if Optimizer SSL certificates have to be verified against certificate authority.
     ## ssl_cert_ca_verify=True
 

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1648,6 +1648,9 @@
     # Automatically upload queries after their execution in order to improve recommendations.
     ## auto_upload_queries=true
 
+    # Allow admins to upload the last N executed queries in the quick start wizard. Use 0 to disable.
+    ## query_history_upload_limit=10000
+
     # In secure mode (HTTPS), if Optimizer SSL certificates have to be verified against certificate authority.
     ## ssl_cert_ca_verify=True
 

+ 6 - 0
desktop/libs/metadata/src/metadata/conf.py

@@ -123,6 +123,12 @@ OPTIMIZER = ConfigSection(
       default=True,
       type=coerce_bool
     ),
+    QUERY_HISTORY_UPLOAD_LIMIT = Config(
+      key="query_history_upload_limit",
+      help=_t("Allow admins to upload the last N executed queries in the quick start wizard. Use 0 to disable."),
+      default=10000,
+      type=int
+    ),
 
     EMAIL=Config(
       key="email",

+ 20 - 9
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -320,19 +320,30 @@ def _convert_queries(queries_data):
 def upload_history(request):
   response = {'status': -1}
 
-  n = request.POST.get('n')
-  source_platform = request.POST.get('sourcePlatform', 'hive')
+  if request.user.is_superuser:
+    api = OptimizerApi(request.user)
+    histories = []
+    upload_stats = {}
 
-  history = Document2.objects.get_history(doc_type='query-%s' % source_platform, user=request.user)
-  if n:
-    history = history[:n]
+    if request.POST.get('sourcePlatform'):
+      n = min(request.POST.get('n', OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get()))
+      source_platform = request.POST.get('sourcePlatform', 'hive')
+      histories = [(source_platform, Document2.objects.get_history(doc_type='query-%s' % source_platform, user=request.user)[:n])]
 
-  queries = _convert_queries([Notebook(document=doc).get_data() for doc in history])
+    elif OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get() > 0:
+      histories = [
+        (source_platform, Document2.objects.filter(type='query-%s' % source_platform, is_history=True, is_managed=False, is_trashed=False).order_by('-last_modified')[:OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get()])
+            for source_platform in ['hive', 'impala']
+      ]
 
-  api = OptimizerApi(request.user)
+    for source_platform, history in histories:
+      queries = _convert_queries([Notebook(document=doc).get_data() for doc in history])
+      upload_stats[source_platform] = api.upload(data=queries, data_type='queries', source_platform=source_platform)
 
-  response['upload_history'] = api.upload(data=queries, data_type='queries', source_platform=source_platform)
-  response['status'] = 0
+    response['upload_history'] = upload_stats
+    response['status'] = 0
+  else:
+    response['message'] = _('Query history upload requires Admin privileges or feature is disabled.')
 
   return JsonResponse(response)
 

+ 2 - 2
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -183,11 +183,11 @@ class OptimizerApi(object):
 
           for row in queries_csv:
             f_queries.write(row)
-            LOG.debug(row)
+            LOG.debug(row[:1000])
         else:
           # Table, column stats
           f_queries.write(json.dumps(data))
-          LOG.debug(json.dumps(data))
+          LOG.debug(json.dumps(data[:10]))
 
       finally:
         f_queries.close()

+ 2 - 2
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -1627,8 +1627,8 @@ var EditorViewModel = (function() {
         sourcePlatform: self.type()
       }, function(data) {
         if (data.status == 0) {
-          $(document).trigger("info", data.upload_history.count + " queries uploaded successfully. Processing them...");
-          self.watchUploadStatus(data.upload_history.status.workloadId);
+          $(document).trigger("info", data.upload_history[self.type()].count + " queries uploaded successfully. Processing them...");
+          self.watchUploadStatus(data.upload_history[self.type()].status.workloadId);
         } else {
           $(document).trigger("error", data.message);
         }

+ 7 - 0
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1721,6 +1721,13 @@ from notebook.conf import ENABLE_QUERY_BUILDER, ENABLE_QUERY_SCHEDULING, ENABLE_
             <i class="fa fa-fw fa-random"></i> ${_('Check compatibility')}
           </a>
         </li>
+        % if conf.DJANGO_DEBUG_MODE.get() and user.is_superuser:
+        <li>
+          <a href="javascript:void(0)" data-bind="click: function() { huePubSub.publish('editor.upload.history'); }" title="${ _('Load recent queries in order to improve recommendations') }">
+            <i class="fa fa-fw fa-cloud-upload"></i> ${_('Upload history')}
+          </a>
+        </li>
+        % endif
         <!-- /ko -->
       </ul>
     </div>