Pārlūkot izejas kodu

HUE-3797 [browser] Add listing of past Hive schedule executions

Romain 5 gadi atpakaļ
vecāks
revīzija
f82d5937f1

+ 12 - 4
apps/jobbrowser/src/jobbrowser/apis/schedule_hive.py

@@ -75,6 +75,8 @@ class HiveScheduleApi(Api):
         'canWrite': self.user.username == app['user'],
         'submitted': app['enabled'],
         'properties': {
+            'query': app['query'],
+            'tasks': []
         }
     }
 
@@ -83,7 +85,7 @@ class HiveScheduleApi(Api):
     api = HiveSchedulerApi(user=self.user)
 
     operations = []
-    actual_app_ids = [app_id.replace('schedule-hive-', '') for app_id in app_ids]
+    actual_app_ids = [app_id.rsplit('-')[-1] for app_id in app_ids]
 
     for app_id in actual_app_ids:
       try:
@@ -106,10 +108,16 @@ class HiveScheduleApi(Api):
   def profile(self, appid, app_type, app_property, app_filters):
     appid = appid.rsplit('-')[-1]
 
-    if app_property == 'properties':
-      api = get_api(self.user)
+    if app_property == 'tasks':
+      api = HiveSchedulerApi(user=self.user)
 
-      return api.get_statements(appid)
+      return [
+        {
+          'status': task['state'],
+          'title': task['executor_query_id'],
+        }
+        for task in api.list_executed_tasks(appid)
+      ]
     else:
       return {}
 

+ 29 - 16
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -2011,38 +2011,51 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
     </div>
     <div data-bind="css: {'span10': !$root.isMini(), 'span12 no-margin': $root.isMini() }">
       <ul class="nav nav-pills margin-top-20">
-        <li>
-          <a href="#schedule-hive-page-statements${ SUFFIX }" data-bind="click: function(){ fetchProfile('properties'); $('a[href=\'#schedule-hive-statements${ SUFFIX }\']').tab('show'); }">
+        <li class="active">
+          <a href="#schedule-hive-page-properties${ SUFFIX }" data-toggle="tab">
             ${ _('Properties') }
           </a>
         </li>
+        <li>
+          <a href="#schedule-hive-page-queries${ SUFFIX }" data-bind="click: function(){ fetchProfile('tasks'); $('a[href=\'#schedule-hive-queries${ SUFFIX }\']').tab('show'); }" data-toggle="tab">
+            ${ _('Queries') }
+          </a>
+        </li>
         <li class="pull-right" data-bind="template: { name: 'job-actions${ SUFFIX }' }"></li>
       </ul>
 
       <div class="clearfix"></div>
 
       <div class="tab-content">
-        <div class="tab-pane active" id="schedule-hive-page-statements${ SUFFIX }">
-          <table id="actionsTable" class="datatables table table-condensed">
+        <div class="tab-pane active" id="schedule-hive-page-properties${ SUFFIX }">
+          <pre data-bind="html: properties['query']"></pre>
+        </div>
+
+        <div class="tab-pane" id="schedule-hive-page-queries${ SUFFIX }">
+          <!-- ko with: coordinatorActions() -->
+          <form class="form-inline">
+            <div data-bind="template: { name: 'job-actions${ SUFFIX }' }" class="pull-right"></div>
+          </form>
+
+          <table id="schedulesHiveTable" class="datatables table table-condensed status-border-container">
             <thead>
             <tr>
-              <th>${_('Id')}</th>
-              <th>${_('State')}</th>
-              <th>${_('Output')}</th>
+              <th width="1%"><div class="select-all hue-checkbox fa" data-bind="hueCheckAll: { allValues: apps, selectedValues: selectedJobs }"></div></th>
+              <th>${_('Status')}</th>
+              <th>${_('Title')}</th>
             </tr>
             </thead>
-            <tbody data-bind="foreach: properties['statements']">
-              <tr data-bind="click: function() {  $root.job().id(id); $root.job().fetchJob(); }" class="pointer">
-                <td>
-                  <a data-bind="hueLink: '/jobbrowser/jobs/' + id(), clickBubble: false">
-                    <i class="fa fa-tasks"></i>
-                  </a>
+            <tbody data-bind="foreach: apps">
+              <tr class="status-border pointer" data-bind="css: {'completed': properties.status() == 'SUCCEEDED', 'running': ['RUNNING', 'FAILED', 'KILLED'].indexOf(properties.status()) != -1, 'failed': properties.status() == 'FAILED' || properties.status() == 'KILLED'}, click: function() {  if (properties.externalId() && properties.externalId() != '-') { $root.job().id(properties.externalId()); $root.job().fetchJob(); } }">
+                <td data-bind="click: function() {}, clickBubble: false">
+                  <div class="hue-checkbox fa" data-bind="click: function() {}, clickBubble: false, multiCheck: '#schedulesHiveTable', value: $data, hueChecked: $parent.selectedJobs"></div>
                 </td>
-                <td data-bind="text: state"></td>
-                <td data-bind="text: output"></td>
+                <td><span class="label job-status-label" data-bind="text: properties.status"></span></td>
+                <td data-bind="text: properties.title"></td>
               </tr>
             </tbody>
           </table>
+          <!-- /ko -->
         </div>
       </div>
     </div>
@@ -2834,7 +2847,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       };
 
       self.coordinatorActions = ko.pureComputed(function() {
-        if (self.mainType() == 'schedules' && self.properties['tasks']) {
+        if (self.mainType().indexOf('schedule') != -1 && self.properties['tasks']) {
           var apps = self.properties['tasks']().map(function (instance) {
             var job = new CoordinatorAction(vm, ko.mapping.toJS(instance), self);
             job.properties = ko.mapping.fromJS(instance);

+ 36 - 0
desktop/core/src/desktop/lib/scheduler/lib/hive.py

@@ -106,6 +106,42 @@ class HiveSchedulerApi(Api):
       self._get_task(row) for row in handle['result']['data']
     ]
 
+  def list_executed_tasks(self, app_id):
+    sql_query = """
+SELECT scheduled_executions.*
+FROM information_schema.scheduled_executions
+JOIN information_schema.scheduled_queries ON scheduled_queries.schedule_name = scheduled_executions.schedule_name
+where scheduled_query_id = %(scheduled_query_id)s
+LIMIT 100""" % {
+      'scheduled_query_id': app_id
+    }
+
+    job = make_notebook(
+        name='List Hive scheduled execution',
+        editor_type='hive',
+        statement=sql_query,
+        status='ready',
+        database='default',
+        is_task=False,
+    )
+    request = MockRequest(self.user)
+
+    handle = job.execute_and_wait(request, include_results=True)
+
+    return [
+        {
+        'scheduled_execution_id': row[0],
+        'schedule_name': row[1],
+        'executor_query_id': row[2],
+        'state': row[3],
+        'start_time': row[4],
+        'end_time': row[5],
+        'elapsed': row[6],
+        'error_message': row[7],
+        'last_update_time': row[8],
+      } for row in handle['result']['data']
+    ]
+
   def list_task(self, task_id):
     task_id = task_id.replace('schedule-hive-', '')
 

+ 4 - 3
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -53,7 +53,8 @@ class AuthenticationRequired(Exception):
     self.message = message
 
 class OperationTimeout(Exception):
-  pass
+  def __str__(self):
+    return 'OperationTimeout'
 
 class OperationNotSupported(Exception):
   pass
@@ -289,7 +290,7 @@ class Notebook(object):
     return _execute_notebook(request, notebook_data, snippet)
 
 
-  def execute_and_wait(self, request, timeout_sec=30.0, sleep_interval=0.5, include_results=False):
+  def execute_and_wait(self, request, timeout_sec=30.0, sleep_interval=1, include_results=False):
     """
     Run query and check status until it finishes or timeouts.
 
@@ -315,7 +316,7 @@ class Notebook(object):
           # TODO: close
         return handle
 
-      status = self.check_status(request, operation_id=operation_id)
+      handle = self.check_status(request, operation_id=operation_id)
       time.sleep(sleep_interval)
       curr = time.time()