Browse Source

HUE-2645 [dataeng] Update dataeng command to version 0.9

Romain Rigaux 8 years ago
parent
commit
123ab74

+ 8 - 79
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -74,70 +74,26 @@ class DataEngClusterApi(Api):
     }
 
 
-
   def app(self, appid):
-    oozie_api = get_oozie(self.user)
-    workflow = oozie_api.get_job(jobid=appid)
-
-    common = {
-        'id': workflow.id,
-        'name': workflow.appName,
-        'status': workflow.status,
-        'apiStatus': self._api_status(workflow.status),
-        'progress': workflow.get_progress(),
-        'type': 'workflow',
-    }
-
-    request = MockDjangoRequest(self.user)
-    response = list_oozie_workflow(request, job_id=appid)
-    common['properties'] = json.loads(response.content)
-    common['properties']['xml'] = ''
-    common['properties']['properties'] = ''
-    common['properties']['coordinator_id'] = workflow.get_parent_job_id()
-    common['properties']['bundle_id'] = workflow.conf_dict.get('oozie.bundle.id')
-
-    return common
+    return {}
 
 
   def action(self, appid, action):
-    if action == 'change' or action == 'ignore' or ',' not in appid:
-      request = MockDjangoRequest(self.user)
-      response = manage_oozie_jobs(request, appid, action['action'])
-    else:
-      request = MockDjangoRequest(self.user, post={'job_ids': appid, 'action': action['action']})
-      response = bulk_manage_oozie_jobs(request)
-
-    return json.loads(response.content)
+    return {}
 
 
   def logs(self, appid, app_type, log_name=None):
-    request = MockDjangoRequest(self.user)
-    data = get_oozie_job_log(request, job_id=appid)
-
-    return {'logs': json.loads(data.content)['log']}
+    return {'logs': ''}
 
 
   def profile(self, appid, app_type, app_property):
-    if app_property == 'xml':
-      oozie_api = get_oozie(self.user)
-      workflow = oozie_api.get_job(jobid=appid)
-      return {
-        'xml': workflow.definition,
-      }
-    elif app_property == 'properties':
-      oozie_api = get_oozie(self.user)
-      workflow = oozie_api.get_job(jobid=appid)
-      return {
-        'properties': workflow.conf_dict,
-      }
-
     return {}
 
   def _api_status(self, status):
     if status in ['CREATING', 'CREATED', 'TERMINATING']:
       return 'RUNNING'
-    elif status in ['ARCHIVING']:
-      return 'SUCCEEDED'    
+    elif status in ['ARCHIVING', 'COMPLETED']:
+      return 'SUCCEEDED'
     else:
       return 'FAILED' # KILLED and FAILED
 
@@ -196,13 +152,6 @@ class DataEngJobApi(Api):
 
     job = handle['jobs'][0]
 
-# {u'jobs': [{u'status': u'FAILED', u'submitterCrn': u'crn:altus:iam:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:user:/csGD5p16ZWkUateZrvVk9zm10gXQAkjqKvsIPVkQ5U=/d122f235-3a45-46a0-b3a6-e572c76a711c',
-#              u'jobId': u'9c697bc9-175d-4260-8d84-56fffc148810', u'clusterCrn': u'crn:altus:dataeng:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:cluster:praveen-hive-on-mr/2bb313d7-ffbf-42bd-a9d9-5e4f827ea799', 
-#              u'creationDate': u'2017-04-06T00:32:51.673000+00:00', u'crn': u'crn:altus:dataeng:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:job:9c697bc9-175d-4260-8d84-56fffc148810', 
-#              u'hiveJobDetails': {u'params': [], u'script': u'file:///Users/praveen/Code/Cloudera/scripts/queries/simple.hql'},
-#               u'jobType': u'HIVE', u'failureAction': u'INTERRUPT_JOB_QUEUE'}]}
-
-
     common = {
         'id': job['jobId'],
         'name': job['jobId'],
@@ -213,7 +162,7 @@ class DataEngJobApi(Api):
         'submitted': job['creationDate'],
         'type': 'dataeng-job-%s' % job['jobType'],
     }
-    
+
     common['properties'] = {
       'properties': job
     }
@@ -222,14 +171,7 @@ class DataEngJobApi(Api):
 
 
   def action(self, appid, action):
-    if action == 'change' or action == 'ignore' or ',' not in appid:
-      request = MockDjangoRequest(self.user)
-      response = manage_oozie_jobs(request, appid, action['action'])
-    else:
-      request = MockDjangoRequest(self.user, post={'job_ids': appid, 'action': action['action']})
-      response = bulk_manage_oozie_jobs(request)
-
-    return json.loads(response.content)
+    return {}
 
 
   def logs(self, appid, app_type, log_name=None):
@@ -237,19 +179,6 @@ class DataEngJobApi(Api):
 
 
   def profile(self, appid, app_type, app_property):
-    if app_property == 'xml':
-      oozie_api = get_oozie(self.user)
-      workflow = oozie_api.get_job(jobid=appid)
-      return {
-        'xml': workflow.definition,
-      }
-    elif app_property == 'properties':
-      oozie_api = get_oozie(self.user)
-      workflow = oozie_api.get_job(jobid=appid)
-      return {
-        'properties': workflow.conf_dict,
-      }
-
     return {}
 
   def _api_status(self, status):
@@ -258,5 +187,5 @@ class DataEngJobApi(Api):
     elif status in ['COMPLETED']:
       return 'SUCCEEDED'
     else:
-      return 'FAILED' # INTERRUPTED , KILLED and FAILED
+      return 'FAILED' # INTERRUPTED , KILLED, TERMINATED and FAILED
 

+ 0 - 27
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -2121,34 +2121,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
       var loadHash = function () {
         if (window.location.pathname.indexOf('jobbrowser') > -1) {
-<<<<<<< HEAD
           jobBrowserViewModel.load();
-=======
-          var h = window.location.hash;
-
-          h = h.indexOf('#!') === 0 ? h.substr(2) : '';
-          switch (h) {
-            case '':
-              h = 'jobs';
-            case 'slas':
-            case 'oozie-info':
-            case 'jobs':
-            case 'workflows':
-            case 'schedules':
-            case 'bundles':
-            case 'dataeng-clusters':
-            case 'dataeng-jobs':
-              viewModel.selectInterface(h);
-              break;
-            default:
-              if (h.indexOf('id=') === 0 && !viewModel.isMini()){
-                new Job(viewModel, {id: h.substr(3)}).fetchJob();
-              }
-              else {
-                viewModel.selectInterface('reset');
-              }
-          }
->>>>>>> HUE-1176 [jb] Avoid error on SLA page
         }
       };
 

+ 15 - 13
desktop/libs/notebook/src/notebook/connectors/dataeng.py

@@ -52,14 +52,16 @@ RUNNING_STATES = ('QUEUED', 'RUNNING')
 
 
 class DataEngApi(Api):
-
+  
+  def __init__(self, user, cluster_name, interpreter=None, request=None):
+    Api.__init__(self, user, interpreter=interpreter, request=request)
+    self.cluster_name = cluster_name
 
   def execute(self, notebook, snippet):
     statement = snippet['statement']
-    cluster_name = 'rjustice-cluster'
 
-    handle = DataEng(self.user).submit_hive_job(cluster_name, statement, params=None, job_xml=None)
-    job = handle['job']
+    handle = DataEng(self.user).submit_hive_job(self.cluster_name, statement, params=None, job_xml=None)
+    job = handle['jobs'][0]
 
     if job['status'] not in RUNNING_STATES:
       raise QueryError('Submission failure', handle=job['status'])
@@ -199,23 +201,23 @@ class DataEng():
     return _exec(args)
 
   def submit_hive_job(self, cluster_name, script, params=None, job_xml=None):
-    args = ['submit-hive-job', '--cluster-name', cluster_name, '--script', script]
+    job = {'script': script}
 
     if params:
-      args.extend(['--params', params])
+      job['params'] =  params
     if job_xml:
-      args.extend(['--job-xml ', job_xml])
-
-    return _exec(args)
+      job['jobXml'] =  job_xml
 
+    return self.submit_jobs(cluster_name, [{'hiveJob': job}])
+  
   def submit_spark_job(self):
-    return _exec(['submit-spark-job'])
+    return _exec(['submit-jobs'])
   
   def submit_yarn_job(self):
-    return _exec(['submit-yarn-job'])
-  
-  def submit_jobs(self):
     return _exec(['submit-jobs'])
+  
+  def submit_jobs(self, cluster_name, jobs):
+    return _exec(['submit-jobs', '--cluster-name', cluster_name, '--jobs', json.dumps(jobs)])
 
   def terminate_jobs(self, job_ids):
     return _exec(['terminate-jobs', '--job-ids', job_ids])