Browse Source

HUE-8330 [notebook] Refactor Dataeng API

Romain Rigaux 7 years ago
parent
commit
f8ad57bd3b

+ 5 - 4
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -21,7 +21,8 @@ from datetime import datetime,  timedelta
 
 from django.utils.translation import ugettext as _
 
-from notebook.connectors.dataeng import DataEng, DATE_FORMAT
+from notebook.connectors.altus import DATE_FORMAT
+from notebook.connectors.dataeng import DataEngApi
 
 from jobbrowser.apis.base_api import Api
 
@@ -32,7 +33,7 @@ LOG = logging.getLogger(__name__)
 class DataEngClusterApi(Api):
 
   def apps(self, filters):
-    api = DataEng(self.user)
+    api = DataEngApi(self.user)
 
     jobs = api.list_clusters()
 
@@ -92,7 +93,7 @@ class DataEngJobApi(Api):
       kwargs['creation_date_after'] = (datetime.today() - delta).strftime(DATE_FORMAT)
     # Todo: filter on 'cluster_crn'
 
-    api = DataEng(self.user)
+    api = DataEngApi(self.user)
 
     jobs = api.list_jobs(**kwargs)
 
@@ -112,7 +113,7 @@ class DataEngJobApi(Api):
     }
 
   def app(self, appid):
-    handle = DataEng(self.user).describe_job(job_id=appid)
+    handle = DataEngApi(self.user).describe_job(job_id=appid)
 
     job = handle['job']
 

File diff suppressed because it is too large
+ 0 - 0
apps/oozie/src/oozie/static/oozie/js/workflow-editor.ko.js


+ 19 - 13
apps/oozie/src/oozie/templates/editor2/workflow_editor.mako

@@ -413,25 +413,31 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons, is_emb
 
 
   <div class="container-fluid">
-  <div class="row-fluid">
-    %if is_embeddable:
-    <div class="span12 margin-top-20">
-    %else:
-    <div class="span12" data-bind="style:{'marginTop' : $root.isEditing() ? '120px': '50px'}">
-    %endif
-    <div class="object-name" style="text-align: center">
-      <span data-bind="editable: $root.workflow.name, editableOptions: {enabled: $root.isEditing(), placement: 'right'}"></span>
-    </div>
-    <div class="object-description" style="text-align: center; margin-top: 10px">
-      <span data-bind="editable: $root.workflow.properties.description, editableOptions: {enabled: $root.isEditing(), placement: 'right', emptytext: '${_ko('Add a description...')}'}"></span>
-    </div>
+    <span class="pull-right">
+    <!-- ko if: availableComputes().length > 1 -->
+      <select data-bind="selectize: availableComputes, value: compute, optionsValue: 'id', optionsText: 'name'" class="input-medium"></select>
+      ## <div class="margin-left-10" data-bind="component: { name: 'hue-drop-down', params: { icon: 'fa-snowflake-o', value: namespace, entries: availableNamespaces, labelAttribute: 'name', foreachVisible: true, searchable: true, linkTitle: '${ _ko('Namespaces') }' } }" style="display: inline-block"></div>
+    <!-- /ko -->
+    </span>
+      
+    <div class="row-fluid">
+      %if is_embeddable:
+      <div class="span12 margin-top-20">
+      %else:
+      <div class="span12" data-bind="style:{'marginTop' : $root.isEditing() ? '120px': '50px'}">
+      %endif
+      <div class="object-name" style="text-align: center">
+        <span data-bind="editable: $root.workflow.name, editableOptions: {enabled: $root.isEditing(), placement: 'right'}"></span>
+      </div>
+      <div class="object-description" style="text-align: center; margin-top: 10px">
+        <span data-bind="editable: $root.workflow.properties.description, editableOptions: {enabled: $root.isEditing(), placement: 'right', emptytext: '${_ko('Add a description...')}'}"></span>
+      </div>
     </div>
   </div>
 </div>
 
 
 
-
 ${ workflow.render() }
 
 

+ 15 - 5
desktop/core/src/desktop/api2.py

@@ -34,9 +34,9 @@ from django.views.decorators.csrf import ensure_csrf_cookie
 from django.views.decorators.http import require_POST
 
 from metadata.conf import has_navigator
-from metadata.catalog_api import search_entities as metadata_search_entities, _highlight
-from metadata.catalog_api import search_entities_interactive as metadata_search_entities_interactive
-from notebook.connectors.altus import SdxApi, AnalyticDbApi
+from metadata.navigator_api import search_entities as metadata_search_entities, _highlight
+from metadata.navigator_api import search_entities_interactive as metadata_search_entities_interactive
+from notebook.connectors.altus import SdxApi, AnalyticDbApi, DataEngApi
 from notebook.connectors.base import Notebook
 from notebook.views import upgrade_session_properties
 
@@ -122,7 +122,6 @@ def get_context_computes(request, interface):
       } for cluster in clusters
     ])
 
-    # From Altus
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
       computes.extend([{
           'id': cluster.get('crn', 'None'),
@@ -132,7 +131,18 @@ def get_context_computes(request, interface):
           # environmentType
           # secured
           # cdhVersion
-        } for cluster in AnalyticDbApi(request.user).list_clusters()]
+        } for cluster in AnalyticDbApi(request.user).list_clusters()['clusters']]
+      )
+
+  if interface == 'oozie':
+    if [cluster for cluster in clusters if cluster['type'] == 'altus']:
+      computes.extend([{
+          'id': cluster.get('crn'),
+          'name': cluster.get('clusterName'),
+          'status': cluster.get('status'),
+          'environmentType': cluster.get('environmentType'),
+          'serviceType': cluster.get('serviceType'),
+        } for cluster in DataEngApi(request.user).list_clusters()['clusters']]
       )
 
   response[interface] = computes

+ 89 - 1
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -19,6 +19,8 @@ import logging
 import json
 import subprocess
 
+from datetime import datetime,  timedelta
+
 from django.urls import reverse
 from django.utils.translation import ugettext as _
 
@@ -27,6 +29,8 @@ from desktop.lib.exceptions_renderable import PopupException
 
 LOG = logging.getLogger(__name__)
 
+DATE_FORMAT = "%Y-%m-%d"
+
 
 def _exec(command, args):
   try:
@@ -68,6 +72,90 @@ class SdxApi():
     return _exec('sdx', args)['namespaces']
 
 
+class DataEngApi():
+
+  def __init__(self, user): pass
+
+  def list_jobs(self, submitter_crns=None, page_size=None, starting_token=None, job_statuses=None, job_ids=None, job_types=None, creation_date_before=None,
+        creation_date_after=None, cluster_crn=None, order=None):
+    args = ['list-jobs']
+
+    if creation_date_after is None:
+      creation_date_after = (datetime.today() - timedelta(days=7)).strftime(DATE_FORMAT)
+
+    if submitter_crns:
+      args.extend(['--submitter-crns', submitter_crns])
+    if page_size is not None:
+      args.extend(['--page-size', str(page_size)])
+    if starting_token:
+      args.extend(['--starting-token', starting_token])
+    if job_statuses:
+      args.extend(['--job-statuses', job_statuses])
+    if job_ids:
+      args.extend(['--job-ids'] + job_ids)
+    if job_types:
+      args.extend(['--job-types', job_types])
+    if creation_date_before:
+      args.extend(['--creation-date-before', creation_date_before])
+    if creation_date_after:
+      args.extend(['--creation-date-after', creation_date_after])
+    if cluster_crn:
+      args.extend(['--cluster-crn', cluster_crn])
+    if order:
+      args.extend(['--order', order])
+
+    return _exec('dataeng', args)
+
+  def describe_job(self, job_id):
+    args = ['describe-job', '--job-id', job_id]
+
+    return _exec('dataeng', args)
+
+  def submit_hive_job(self, cluster_name, script, params=None, job_xml=None):
+    job = {'script': script}
+
+    if params:
+      job['params'] =  params
+    if job_xml:
+      job['jobXml'] =  job_xml
+
+    return self.submit_jobs(cluster_name, [{'hiveJob': job}])
+
+  def submit_spark_job(self):
+    return _exec('dataeng', ['submit-jobs'])
+
+  def submit_yarn_job(self):
+    return _exec('dataeng', ['submit-jobs'])
+
+  def submit_jobs(self, cluster_name, jobs):
+    return _exec('dataeng', ['submit-jobs', '--cluster-name', cluster_name, '--jobs', json.dumps(jobs)])
+
+  def terminate_job(self, job_id):
+    return _exec('dataeng', ['terminate-job', '--job-id', job_id])
+
+
+  def list_clusters(self, names=None, page_size=None, starting_token=None):
+    args = ['list-clusters']
+
+    if names:
+      args.extend(['--cluster-names', names])
+    if page_size is not None:
+      args.extend(['--page-size', str(page_size)])
+    if starting_token:
+      args.extend(['--starting-token', starting_token])
+
+    return _exec('dataeng', args)
+
+  def create_cluster(self):
+    return _exec('dataeng', ['create-cluster'])
+
+  def delete_cluster(self):
+    return _exec('dataeng', ['delete-cluster'])
+
+  def describe_clusters(self):
+    return _exec('dataeng', ['describe-cluster'])
+
+
 class AnalyticDbApi():
 
   def __init__(self, user): pass
@@ -96,4 +184,4 @@ class AnalyticDbApi():
     """
 
     args = ['list-clusters']
-    return _exec('analyticdb', args)['clusters']
+    return _exec('analyticdb', args)

+ 4 - 108
desktop/libs/notebook/src/notebook/connectors/dataeng.py

@@ -16,40 +16,20 @@
 # limitations under the License.
 
 import logging
-import json
 import re
-import subprocess
-
-from datetime import datetime,  timedelta
 
 from django.urls import reverse
 from django.utils.translation import ugettext as _
 
-from desktop.lib.exceptions_renderable import PopupException
 from metadata.workload_analytics_client import WorkfloadAnalyticsClient
 
+from notebook.connectors.altus import DataEngApi as DataEngClient
 from notebook.connectors.base import Api, QueryError
 
 
 LOG = logging.getLogger(__name__)
 
 
-def _exec(args):
-  try:
-    data = subprocess.check_output([
-        'altus',
-        'dataeng',
-       ] +
-       args
-    )
-  except Exception, e:
-    raise PopupException(e, title=_('Error accessing'))
-
-  response = json.loads(data)
-
-  return response
-
-DATE_FORMAT = "%Y-%m-%d"
 RUNNING_STATES = ('QUEUED', 'RUNNING', 'SUBMITTING')
 
 
@@ -63,7 +43,7 @@ class DataEngApi(Api):
   def execute(self, notebook, snippet):
     statement = snippet['statement']
 
-    handle = DataEng(self.user).submit_hive_job(self.cluster_name, statement, params=None, job_xml=None)
+    handle = DataEngClient(self.user).submit_hive_job(self.cluster_name, statement, params=None, job_xml=None)
     job = handle['jobs'][0]
 
     if job['status'] not in RUNNING_STATES:
@@ -81,7 +61,7 @@ class DataEngApi(Api):
 
     job_id = snippet['result']['handle']['id']
 
-    handle = DataEng(self.user).list_jobs(job_ids=[job_id])
+    handle = DataEngClient(self.user).list_jobs(job_ids=[job_id])
     job = handle['jobs'][0]
 
     if job['status'] in RUNNING_STATES:
@@ -106,7 +86,7 @@ class DataEngApi(Api):
   def cancel(self, notebook, snippet):
     if snippet['result']['handle'].get('id'):
       job_id = snippet['result']['handle']['id']
-      DataEng(self.user).terminate_job(job_id=job_id)
+      DataEngClient(self.user).terminate_job(job_id=job_id)
       response = {'status': 0}
     else:
       response = {'status': -1, 'message': _('Could not cancel because of unsuccessful submition.')}
@@ -145,87 +125,3 @@ class DataEngApi(Api):
 
   def close_session(self, session):
     pass
-
-
-class DataEng():
-
-  def __init__(self, user): pass
-
-  def list_jobs(self, submitter_crns=None, page_size=None, starting_token=None, job_statuses=None, job_ids=None, job_types=None, creation_date_before=None,
-        creation_date_after=None, cluster_crn=None, order=None):
-    args = ['list-jobs']
-
-    if creation_date_after is None:
-      creation_date_after = (datetime.today() - timedelta(days=7)).strftime(DATE_FORMAT)
-
-    if submitter_crns:
-      args.extend(['--submitter-crns', submitter_crns])
-    if page_size is not None:
-      args.extend(['--page-size', str(page_size)])
-    if starting_token:
-      args.extend(['--starting-token', starting_token])
-    if job_statuses:
-      args.extend(['--job-statuses', job_statuses])
-    if job_ids:
-      args.extend(['--job-ids'] + job_ids)
-    if job_types:
-      args.extend(['--job-types', job_types])
-    if creation_date_before:
-      args.extend(['--creation-date-before', creation_date_before])
-    if creation_date_after:
-      args.extend(['--creation-date-after', creation_date_after])
-    if cluster_crn:
-      args.extend(['--cluster-crn', cluster_crn])
-    if order:
-      args.extend(['--order', order])
-
-    return _exec(args)
-
-  def describe_job(self, job_id):
-    args = ['describe-job', '--job-id', job_id]
-
-    return _exec(args)
-
-  def submit_hive_job(self, cluster_name, script, params=None, job_xml=None):
-    job = {'script': script}
-
-    if params:
-      job['params'] =  params
-    if job_xml:
-      job['jobXml'] =  job_xml
-
-    return self.submit_jobs(cluster_name, [{'hiveJob': job}])
-
-  def submit_spark_job(self):
-    return _exec(['submit-jobs'])
-
-  def submit_yarn_job(self):
-    return _exec(['submit-jobs'])
-
-  def submit_jobs(self, cluster_name, jobs):
-    return _exec(['submit-jobs', '--cluster-name', cluster_name, '--jobs', json.dumps(jobs)])
-
-  def terminate_job(self, job_id):
-    return _exec(['terminate-job', '--job-id', job_id])
-
-
-  def list_clusters(self, names=None, page_size=None, starting_token=None):
-    args = ['list-clusters']
-
-    if names:
-      args.extend(['--cluster-names', names])
-    if page_size is not None:
-      args.extend(['--page-size', str(page_size)])
-    if starting_token:
-      args.extend(['--starting-token', starting_token])
-
-    return _exec(args)
-
-  def create_cluster(self):
-    return _exec(['create-cluster'])
-
-  def delete_cluster(self):
-    return _exec(['delete-cluster'])
-
-  def describe_clusters(self):
-    return _exec(['describe-cluster'])

Some files were not shown because too many files changed in this diff