Browse Source

HUE-6245 [dataeng] Load a dataeng job in Job Browser

Romain Rigaux 8 years ago
parent
commit
6ba6014

+ 21 - 32
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -34,20 +34,20 @@ class DataEngClusterApi(Api):
 
 
   def apps(self, filters):
   def apps(self, filters):
 #     kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
 #     kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
-# 
+#
 #     text_filters = _extract_query_params(filters)
 #     text_filters = _extract_query_params(filters)
-# 
+#
 #     if not has_dashboard_jobs_access(self.user):
 #     if not has_dashboard_jobs_access(self.user):
 #       kwargs['filters'].append(('user', self.user.username))
 #       kwargs['filters'].append(('user', self.user.username))
 #     elif 'user' in text_filters:
 #     elif 'user' in text_filters:
 #       kwargs['filters'].append(('user', text_filters['username']))
 #       kwargs['filters'].append(('user', text_filters['username']))
-# 
+#
 #     if 'time' in filters:
 #     if 'time' in filters:
 #       kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])
 #       kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])
-# 
+#
 #     if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
 #     if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
 #       kwargs['filters'].extend([('text', text_filters.get('text'))])
 #       kwargs['filters'].extend([('text', text_filters.get('text'))])
-# 
+#
 #     if filters.get('states'):
 #     if filters.get('states'):
 #       states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
 #       states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
 #       for _state in filters.get('states'):
 #       for _state in filters.get('states'):
@@ -57,7 +57,7 @@ class DataEngClusterApi(Api):
     api = DataEng(self.user)
     api = DataEng(self.user)
 
 
     jobs = api.list_clusters()
     jobs = api.list_clusters()
-    
+
     return {
     return {
       'apps': [{
       'apps': [{
         'id': app['clusterName'],
         'id': app['clusterName'],
@@ -72,7 +72,7 @@ class DataEngClusterApi(Api):
       } for app in jobs['clusters']],
       } for app in jobs['clusters']],
       'total': None
       'total': None
     }
     }
-            
+
 
 
 
 
   def app(self, appid):
   def app(self, appid):
@@ -144,26 +144,26 @@ class DataEngJobApi(Api):
 
 
   def apps(self, filters):
   def apps(self, filters):
     kwargs = {}
     kwargs = {}
-# 
+#
 #     text_filters = _extract_query_params(filters)
 #     text_filters = _extract_query_params(filters)
-# 
+#
 #     if not has_dashboard_jobs_access(self.user):
 #     if not has_dashboard_jobs_access(self.user):
 #       kwargs['filters'].append(('user', self.user.username))
 #       kwargs['filters'].append(('user', self.user.username))
 #     elif 'user' in text_filters:
 #     elif 'user' in text_filters:
 #       kwargs['filters'].append(('user', text_filters['username']))
 #       kwargs['filters'].append(('user', text_filters['username']))
-# 
+#
     if 'time' in filters:
     if 'time' in filters:
       if filters['time']['time_unit'] == 'minutes':
       if filters['time']['time_unit'] == 'minutes':
         delta = timedelta(minutes=int(filters['time']['time_value']))
         delta = timedelta(minutes=int(filters['time']['time_value']))
       elif filters['time']['time_unit'] == 'hours':
       elif filters['time']['time_unit'] == 'hours':
         delta = timedelta(hours=int(filters['time']['time_value']))
         delta = timedelta(hours=int(filters['time']['time_value']))
       else:
       else:
-        delta = timedelta(days=int(filters['time']['time_value']))                                                                    
+        delta = timedelta(days=int(filters['time']['time_value']))
       kwargs['creation_date_after'] = (datetime.today() - delta).strftime(DATE_FORMAT)
       kwargs['creation_date_after'] = (datetime.today() - delta).strftime(DATE_FORMAT)
 
 
 #     if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
 #     if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
 #       kwargs['filters'].extend([('text', text_filters.get('text'))])
 #       kwargs['filters'].extend([('text', text_filters.get('text'))])
-# 
+#
 #     if filters.get('states'):
 #     if filters.get('states'):
 #       states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
 #       states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
 #       for _state in filters.get('states'):
 #       for _state in filters.get('states'):
@@ -190,26 +190,18 @@ class DataEngJobApi(Api):
     }
     }
 
 
   def app(self, appid):
   def app(self, appid):
-    oozie_api = get_oozie(self.user)
-    workflow = oozie_api.get_job(jobid=appid)
+    handle = DataEng(self.user).list_jobs(job_ids=[appid])
+    job = handle['jobs'][0]
 
 
     common = {
     common = {
-        'id': workflow.id,
-        'name': workflow.appName,
-        'status': workflow.status,
-        'apiStatus': self._api_status(workflow.status),
-        'progress': workflow.get_progress(),
-        'type': 'workflow',
+        'id': job['jobId'],
+        'name': job['jobId'],
+        'status': job['status'],
+        'apiStatus': self._api_status(job['status']),
+        'progress': 50,
+        'type': 'dataeng-job',
     }
     }
 
 
-    request = MockDjangoRequest(self.user)
-    response = list_oozie_workflow(request, job_id=appid)
-    common['properties'] = json.loads(response.content)
-    common['properties']['xml'] = ''
-    common['properties']['properties'] = ''
-    common['properties']['coordinator_id'] = workflow.get_parent_job_id()
-    common['properties']['bundle_id'] = workflow.conf_dict.get('oozie.bundle.id')
-
     return common
     return common
 
 
 
 
@@ -225,10 +217,7 @@ class DataEngJobApi(Api):
 
 
 
 
   def logs(self, appid, app_type, log_name=None):
   def logs(self, appid, app_type, log_name=None):
-    request = MockDjangoRequest(self.user)
-    data = get_oozie_job_log(request, job_id=appid)
-
-    return {'logs': json.loads(data.content)['log']}
+    return {'logs': ''}
 
 
 
 
   def profile(self, appid, app_type, app_property):
   def profile(self, appid, app_type, app_property):

+ 3 - 0
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -1422,6 +1422,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         else if (/oozie-oozi-B/.test(self.id())) {
         else if (/oozie-oozi-B/.test(self.id())) {
           interface = 'bundles';
           interface = 'bundles';
         }
         }
+        else if (/[a-z0-9]{8}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0-9]{12}/.test(self.id())) {
+          interface = 'dataeng-jobs';
+        }
 
 
         interface = vm.isValidInterface(interface);
         interface = vm.isValidInterface(interface);
         vm.interface(interface);
         vm.interface(interface);

+ 6 - 7
desktop/libs/notebook/src/notebook/connectors/dataeng.py

@@ -44,8 +44,6 @@ def _exec(args):
     raise PopupException(e, title=_('Error accessing'))
     raise PopupException(e, title=_('Error accessing'))
 
 
   response = json.loads(data)
   response = json.loads(data)
-  # Chck data['status'] == 'success'
-  response['status'] = 'success'
 
 
   return response
   return response
 
 
@@ -58,7 +56,7 @@ class DataEngApi(Api):
 
 
   def execute(self, notebook, snippet):
   def execute(self, notebook, snippet):
     statement = snippet['statement']
     statement = snippet['statement']
-    cluster_name = 'romain-cluster'
+    cluster_name = 'rjustice-cluster'
 
 
     handle = DataEng(self.user).submit_hive_job(cluster_name, statement, params=None, job_xml=None)
     handle = DataEng(self.user).submit_hive_job(cluster_name, statement, params=None, job_xml=None)
     job = handle['job']
     job = handle['job']
@@ -79,11 +77,12 @@ class DataEngApi(Api):
     job_id = snippet['result']['handle']['id']
     job_id = snippet['result']['handle']['id']
 
 
     handle = DataEng(self.user).list_jobs(job_ids=[job_id])
     handle = DataEng(self.user).list_jobs(job_ids=[job_id])
+    job = handle['jobs'][0]
 
 
-    if handle['status'] in RUNNING_STATES:
+    if job['status'] in RUNNING_STATES:
       return response
       return response
-    elif handle['status'] in ('failed', 'terminated'):
-      raise QueryError(_('Job was %s') % handle['status'])
+    elif job['status'] in ('failed', 'terminated'):
+      raise QueryError(_('Job was %s') % job['status'])
     else:
     else:
       response['status'] = 'available'
       response['status'] = 'available'
 
 
@@ -120,7 +119,7 @@ class DataEngApi(Api):
     job_id = snippet['result']['handle']['id']
     job_id = snippet['result']['handle']['id']
     return [{
     return [{
         'name': job_id,
         'name': job_id,
-        'url': reverse('jobbrowser.views.apps') + '#' + job_id,
+        'url': reverse('jobbrowser.views.apps') + '#!' + job_id,
         'started': True,
         'started': True,
         'finished': False # Would need call to check_status
         'finished': False # Would need call to check_status
       }
       }