Explorar el Código

HUE-9266 [jb] Adding skeleton of Query History as Job Browser type

Romain hace 6 años
padre
commit
79a2af292c

+ 3 - 0
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -34,6 +34,7 @@ def get_api(user, interface, cluster=None):
   from jobbrowser.apis.data_eng_api import DataEngClusterApi, DataEngJobApi
   from jobbrowser.apis.clusters import ClusterApi
   from jobbrowser.apis.data_warehouse import DataWarehouseClusterApi
+  from jobbrowser.apis.history import HistoryApi
   from jobbrowser.apis.livy_api import LivySessionsApi, LivyJobApi
   from jobbrowser.apis.job_api import JobApi
   from jobbrowser.apis.query_api import QueryApi
@@ -56,6 +57,8 @@ def get_api(user, interface, cluster=None):
   elif interface == 'celery-beat':
     from jobbrowser.apis.beat_api import BeatApi
     return BeatApi(user)
+  elif interface == 'history':
+    return HistoryApi(user)
   elif interface == 'engines':
     return ClusterApi(user)
   elif interface == 'dataeng-clusters':

+ 122 - 0
apps/jobbrowser/src/jobbrowser/apis/history.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from datetime import datetime
+from dateutil import parser
+
+from django.utils.translation import ugettext as _
+
+from desktop.models import Document2
+from notebook.api import _get_statement
+from notebook.models import Notebook
+
+from jobbrowser.apis.base_api import Api
+
+
+LOG = logging.getLogger(__name__)
+
+
+class HistoryApi(Api):
+
+  def apps(self, filters):
+    tasks = Document2.objects.get_history(doc_type='query-hive', user=self.user)
+    apps = []
+
+    for app in tasks:
+      # Copied, Document class should have a get_history method (via method or inheritance)
+      notebook = Notebook(document=app).get_data()
+      is_notification_manager = False # Supposed SQL Editor query only right now
+      if 'snippets' in notebook:
+        statement = notebook['description'] if is_notification_manager else _get_statement(notebook)
+        history = {
+          'name': app.name,
+          'id': app.id,
+          'uuid': app.uuid,
+          'type': app.type,
+          'data': {
+              'statement': statement[:1001] if statement else '',
+              'lastExecuted': notebook['snippets'][0].get('lastExecuted', -1),
+              'status':  notebook['snippets'][0]['status'],
+              'parentSavedQueryUuid': notebook.get('parentSavedQueryUuid', '')
+          } if notebook['snippets'] else {},
+          'absoluteUrl': app.get_absolute_url(),
+      }
+      apps.append({
+          'id': 'history-%(id)s' % history,
+          'name': history['data']['statement'],
+          'status': history['data']['status'],
+          'apiStatus': self._api_status(history),
+          'type': 'history-%s' % history['type'],
+          'user': self.user.username,
+          'progress': 50,
+          'queue': '',
+          'canWrite': True,
+          'duration': 1,
+          'submitted': 1
+        })
+
+    return {
+      'apps': apps,
+      'total': len(tasks)
+    }
+
+
+  def app(self, appid):
+    appid = appid.rsplit('-')[-1]
+
+    app = Document2.objects.document(user=self.user, doc_id=appid)
+
+    return {
+      'id': 'history-%s' % app.id,
+      'name': app.name,
+      'status': 'ready',
+      'apiStatus': 'RUNNING',
+      'type': 'history',
+      'user': app.description,
+      'progress': 50,
+      'queue': '',
+      'duration': 1,
+      'canWrite': True,
+      'submitted': 1,
+      'properties': {
+      }
+    }
+
+
+  def action(self, app_ids, operation):
+    # Notebook API
+    pass
+
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
+    return {'logs': ''}
+
+
+  def profile(self, appid, app_type, app_property, app_filters):
+    appid = appid.rsplit('-')[-1]
+
+    return {}
+
+
+  def _api_status(self, task):
+    if task['data']['status'] in ('expired', 'failed'):
+      return 'FAILED'
+    elif task['data']['status'] == 'available':
+      return 'SUCCEEDED'
+    else:
+      return 'RUNNING'

+ 10 - 1
apps/jobbrowser/src/jobbrowser/conf.py

@@ -42,6 +42,7 @@ LOG_OFFSET = Config(
   help=_('Offset in bytes where a negative offset will fetch the last N bytes for the given log file (default 1MB).')
 )
 
+# Deprecated
 ENABLE_V2 = Config(
   key="enable_v2",
   help=_("Show the version 2 of app which unifies all the past browsers into one."),
@@ -56,6 +57,7 @@ MAX_JOB_FETCH = Config(
   help=_('Maximum number of jobs to fetch and display when pagination is not supported for the type.')
 )
 
+# Deprecated
 ENABLE_QUERY_BROWSER = Config(
   key="enable_query_browser",
   help=_("Show the Impala query section for listing and showing more troubleshooting information."),
@@ -68,4 +70,11 @@ ENABLE_HIVE_QUERY_BROWSER = Config(
   help=_("# Show the Hive query section for listing and showing more troubleshooting information."),
   type=coerce_bool,
   default=False
-) 
+) 
+
+ENABLE_HISTORY_V2 = Config(
+  key="enable_history_v2",
+  help=_("Show the version 2 of job/query History which unifies the all into one."),
+  type=coerce_bool,
+  default=False
+)

+ 96 - 9
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -20,7 +20,7 @@ from desktop.conf import CUSTOM, IS_K8S_ONLY
 from desktop.views import commonheader, commonfooter, _ko
 from metadata.conf import PROMETHEUS
 
-from jobbrowser.conf import DISABLE_KILLING_JOBS, MAX_JOB_FETCH, ENABLE_QUERY_BROWSER, ENABLE_HIVE_QUERY_BROWSER
+from jobbrowser.conf import DISABLE_KILLING_JOBS, MAX_JOB_FETCH, ENABLE_QUERY_BROWSER, ENABLE_HIVE_QUERY_BROWSER, ENABLE_HISTORY_V2
 %>
 
 <%
@@ -403,6 +403,10 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
               <!-- ko if: $root.job() -->
               <!-- ko with: $root.job() -->
+                <!-- ko if: mainType() == 'history' -->
+                  <div class="jb-panel" data-bind="template: { name: 'history-page${ SUFFIX }' }"></div>
+                <!-- /ko -->
+
                 <!-- ko if: mainType() == 'jobs' -->
                   <div class="jb-panel" data-bind="template: { name: 'job-page${ SUFFIX }' }"></div>
                 <!-- /ko -->
@@ -691,9 +695,88 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
   <!-- ko if: type() == 'SPARK_EXECUTOR' -->
     <div data-bind="template: { name: 'job-spark-executor-page${ SUFFIX }', data: $root.job() }"></div>
   <!-- /ko -->
+</script>
 
+
+<script type="text/html" id="history-page${ SUFFIX }">
+  <div class="row-fluid">
+    <div data-bind="css: {'span2': !$root.isMini(), 'span12': $root.isMini() }">
+      <div class="sidebar-nav">
+        <ul class="nav nav-list">
+          <li class="nav-header">${ _('Id') }</li>
+          <li class="break-word"><span data-bind="text: id"></span></li>
+          <!-- ko if: doc_url -->
+          <li class="nav-header">${ _('Document') }</li>
+          <li>
+            <a data-bind="hueLink: doc_url" href="javascript: void(0);" title="${ _('Open in editor') }">
+              <span data-bind="text: name"></span>
+            </a>
+          </li>
+          <!-- /ko -->
+          <!-- ko ifnot: doc_url -->
+          <li class="nav-header">${ _('Name') }</li>
+          <li><span data-bind="text: name"></span></li>
+          <!-- /ko -->
+          <li class="nav-header">${ _('Status') }</li>
+          <li><span data-bind="text: status"></span></li>
+          <li class="nav-header">${ _('User') }</li>
+          <li><span data-bind="text: user"></span></li>
+          <li class="nav-header">${ _('Progress') }</li>
+          <li><span data-bind="text: progress"></span>%</li>
+          <li>
+            <div class="progress-job progress" style="background-color: #FFF; width: 100%" data-bind="css: {'progress-danger': apiStatus() === 'FAILED', 'progress-warning': apiStatus() === 'RUNNING', 'progress-success': apiStatus() === 'SUCCEEDED' }">
+              <div class="bar" data-bind="style: {'width': progress() + '%'}"></div>
+            </div>
+          </li>
+          <li class="nav-header">${ _('Duration') }</li>
+          <li><span data-bind="text: duration().toHHMMSS()"></span></li>
+          <li class="nav-header">${ _('Submitted') }</li>
+          <li><span data-bind="moment: {data: submitted, format: 'LLL'}"></span></li>
+        </ul>
+      </div>
+    </div>
+    <div data-bind="css: {'span10': !$root.isMini(), 'span12 no-margin': $root.isMini() }">
+
+      <ul class="nav nav-pills margin-top-20">
+        <li>
+          <a href="#history-page-statements${ SUFFIX }" data-bind="click: function(){ fetchProfile('properties'); $('a[href=\'#history-page-statements${ SUFFIX }\']').tab('show'); }">
+            ${ _('Properties') }
+          </a>
+        </li>
+        <li class="pull-right" data-bind="template: { name: 'job-actions${ SUFFIX }' }"></li>
+      </ul>
+
+      <div class="clearfix"></div>
+
+      <div class="tab-content">
+        <div class="tab-pane active" id="history-page-statements${ SUFFIX }">
+          <table id="actionsTable" class="datatables table table-condensed">
+            <thead>
+            <tr>
+              <th>${_('Id')}</th>
+              <th>${_('State')}</th>
+              <th>${_('Output')}</th>
+            </tr>
+            </thead>
+            <tbody data-bind="foreach: properties['statements']">
+              <tr data-bind="click: function() {  $root.job().id(id); $root.job().fetchJob(); }" class="pointer">
+                <td>
+                  <a data-bind="hueLink: '/jobbrowser/jobs/' + id(), clickBubble: false">
+                    <i class="fa fa-tasks"></i>
+                  </a>
+                </td>
+                <td data-bind="text: state"></td>
+                <td data-bind="text: output"></td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  </div>
 </script>
 
+
 <script type="text/html" id="job-yarn-page${ SUFFIX }">
   <div class="row-fluid">
     <div data-bind="css:{'span2': !$root.isMini(), 'span12': $root.isMini() }">
@@ -1302,7 +1385,6 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
 
 <script type="text/html" id="dataware-clusters-page${ SUFFIX }">
-
   <div class="row-fluid">
     <div data-bind="css:{'span2': !$root.isMini(), 'span12': $root.isMini() }">
       <div class="sidebar-nav">
@@ -1334,7 +1416,6 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
   <button class="btn" title="${ _('Troubleshoot') }" data-bind="click: troubleshoot">
     <i class="fa fa-tachometer"></i> ${ _('Troubleshoot') }
   </button>
-
 </script>
 
 
@@ -2743,7 +2824,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
       self.hasKill = ko.pureComputed(function() {
         return self.type() && (
-          ['MAPREDUCE', 'SPARK', 'workflow', 'schedule', 'bundle', 'QUERY', 'TEZ', 'YarnV2', 'DDL', 'celery-beat'].indexOf(self.type()) != -1 ||
+          ['MAPREDUCE', 'SPARK', 'workflow', 'schedule', 'bundle', 'QUERY', 'TEZ', 'YarnV2', 'DDL', 'celery-beat', 'history'].indexOf(self.type()) != -1 ||
           self.type().indexOf('Data Warehouse') != -1 ||
           self.type().indexOf('Altus') != -1
         );
@@ -2754,7 +2835,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       });
 
       self.hasResume = ko.pureComputed(function() {
-        return ['workflow', 'schedule', 'bundle', 'celery-beat'].indexOf(self.type()) != -1;
+        return ['workflow', 'schedule', 'bundle', 'celery-beat', 'history'].indexOf(self.type()) != -1;
       });
       self.resumeEnabled = ko.pureComputed(function() {
         return self.hasResume() && self.canWrite() && self.apiStatus() == 'PAUSED';
@@ -2768,7 +2849,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       });
 
       self.hasPause = ko.pureComputed(function() {
-        return ['workflow', 'schedule', 'bundle', 'celery-beat'].indexOf(self.type()) != -1;
+        return ['workflow', 'schedule', 'bundle', 'celery-beat', 'history'].indexOf(self.type()) != -1;
       });
       self.pauseEnabled = ko.pureComputed(function() {
         return self.hasPause() && self.canWrite() && self.apiStatus() == 'RUNNING';
@@ -3321,7 +3402,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           'dataeng-clusters',
           'dataware-clusters',
           'dataware2-clusters',
-          'celery-beat'
+          'celery-beat',
+          'history'
         ].indexOf(vm.interface()) != -1 && !self.isCoordinator();
       });
       self.killEnabled = ko.pureComputed(function() {
@@ -3331,7 +3413,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       });
 
       self.hasResume = ko.pureComputed(function() {
-        return ['workflows', 'schedules', 'bundles', 'dataware2-clusters', 'celery-beat'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
+        return ['workflows', 'schedules', 'bundles', 'dataware2-clusters', 'celery-beat', 'history'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
       });
       self.resumeEnabled = ko.pureComputed(function() {
         return self.hasResume() && self.selectedJobs().length > 0 && $.grep(self.selectedJobs(), function(job) {
@@ -3349,7 +3431,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       });
 
       self.hasPause = ko.pureComputed(function() {
-        return ['workflows', 'schedules', 'bundles', 'dataware2-clusters', 'celery-beat'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
+        return ['workflows', 'schedules', 'bundles', 'dataware2-clusters', 'celery-beat', 'history'].indexOf(vm.interface()) != -1 && !self.isCoordinator();
       });
       self.pauseEnabled = ko.pureComputed(function() {
         return self.hasPause() && self.selectedJobs().length > 0 && $.grep(self.selectedJobs(), function(job) {
@@ -3659,6 +3741,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       });
 
       self.availableInterfaces = ko.pureComputed(function () {
+        var historyInterfaceCondition = function () {
+          return '${ ENABLE_HISTORY_V2.get() }' == 'True';
+        };
         var jobsInterfaceCondition = function () {
           return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('yarn') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
         };
@@ -3703,6 +3788,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           {'interface': 'queries-impala', 'label': '${ _ko('Impala') }', 'condition': queryInterfaceCondition},
           {'interface': 'queries-hive', 'label': '${ _ko('Hive') }', 'condition': queryHiveInterfaceCondition},
           {'interface': 'celery-beat', 'label': '${ _ko('Scheduled Tasks') }', 'condition': schedulerBeatInterfaceCondition},
+          {'interface': 'history', 'label': '${ _ko('History') }', 'condition': historyInterfaceCondition},
           {'interface': 'workflows', 'label': '${ _ko('Workflows') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'schedules', 'label': '${ _ko('Schedules') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'bundles', 'label': '${ _ko('Bundles') }', 'condition': schedulerExtraInterfaceCondition},
@@ -3893,6 +3979,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           case 'queries-impala':
           case 'queries-hive':
           case 'celery-beat':
+          case 'history':
           case 'workflows':
           case 'schedules':
           case 'bundles':

+ 1 - 1
desktop/libs/notebook/src/notebook/api.py

@@ -509,7 +509,7 @@ def _clear_sessions(notebook):
 def _historify(notebook, user):
   query_type = notebook['type']
   name = notebook['name'] if (notebook['name'] and notebook['name'].strip() != '') else DEFAULT_HISTORY_NAME
-  is_managed = notebook.get('isManaged') == True  # Prevents None
+  is_managed = notebook.get('isManaged') == True # Prevents None
 
   if is_managed and Document2.objects.filter(uuid=notebook['uuid']).exists():
     history_doc = Document2.objects.get(uuid=notebook['uuid'])