Преглед изворни кода

HUE-5090 [oozie] Initial prefixing of Oozie batch jobs

Romain Rigaux пре 9 година
родитељ
комит
eb12788

+ 13 - 3
apps/jobbrowser/src/jobbrowser/apis/batch_api.py

@@ -19,9 +19,9 @@ import logging
 
 from django.utils.translation import ugettext as _
 
-from liboozie.oozie_api import get_oozie
 from jobbrowser.apis.base_api import Api
-
+from liboozie.oozie_api import get_oozie
+from notebook.connectors.oozie_batch import OozieApi
 
 LOG = logging.getLogger(__name__)
 
@@ -40,7 +40,17 @@ class BatchApi(Api):
     kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
     wf_list = oozie_api.get_workflows(**kwargs)
 
-    return [{'id': app.id, 'status': app.status} for app in wf_list.jobs]
+    return [{
+        'id': app.id,
+        'name': app.appName,
+        'status': app.status,
+        'type': 'workflow',
+        'user': app.user,
+        'progress': 100,
+        'duration': 10 * 3600,
+        'submitted': 10 * 3600
+        
+    } for app in wf_list.jobs if app.appName.startswith(OozieApi.BATCH_JOB_PREFIX)]
 
   def app(self, appid):
     oozie_api = get_oozie(self.user)

+ 3 - 0
apps/jobbrowser/src/jobbrowser/apis/job_api.py

@@ -87,3 +87,6 @@ class YarnAtsApi(Api):
 class ImpalaApi(Api):
   pass
 
+
+class Sqoop2Api(Api):
+  pass

+ 5 - 2
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -68,7 +68,10 @@ class Indexer(object):
   def run_morphline(self, request, collection_name, morphline, input_path, query=None):
     workspace_path = self._upload_workspace(morphline)
 
-    notebook = Notebook(name='Indexer job for %s' % collection_name)
+    notebook = Notebook(
+        name='Indexer job for %s' % collection_name,
+        isManaged=True
+    )
 
     if query:
       q = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=query))
@@ -110,7 +113,7 @@ class Indexer(object):
     notebook_data = notebook.get_data()
     snippet = {'wasBatchExecuted': True, 'type': 'oozie', 'id': notebook_data['snippets'][0]['id'], 'statement': ''}
 
-    job_handle = _execute_notebook(request, notebook_data, snippet) # To set as managed
+    job_handle = _execute_notebook(request, notebook_data, snippet)
 
     return job_handle
 

+ 2 - 1
desktop/libs/notebook/src/notebook/api.py

@@ -370,7 +370,8 @@ def _historify(notebook, user):
     name=name,
     type=query_type,
     owner=user,
-    is_history=True
+    is_history=True,
+    is_managed=notebook.get('isManaged') == True
   )
 
   # Link history of saved query

+ 1 - 0
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -71,6 +71,7 @@ class Notebook(object):
           'description': '',
           'type': 'notebook',
           'isSaved': False,
+          'isManaged': False,
           'sessions': [],
           'snippets': [],
       }

+ 2 - 1
desktop/libs/notebook/src/notebook/connectors/oozie_batch.py

@@ -49,6 +49,7 @@ class OozieApi(Api):
   RESULTS_PATTERN_GENERIC = "(?P<results>>>> Invoking Main class now >>>.+<<< Invocation of Main class completed <<<)"
   RESULTS_PATTERN_MAPREDUCE = "(?P<results>.+)"
   RESULTS_PATTERN_PIG = "(?P<results>>>> Invoking Pig command line now >>>.+<<< Invocation of Pig command completed <<<)"
+  BATCH_JOB_PREFIX = 'Hue_Batch_'
 
   def __init__(self, *args, **kwargs):
     Api.__init__(self, *args, **kwargs)
@@ -64,7 +65,7 @@ class OozieApi(Api):
 
     if notebook['type'] == 'notebook':
       # Convert notebook to workflow
-      workflow_doc = WorkflowBuilder().create_notebook_workflow(notebook=notebook, user=self.user, managed=True, name=_("Batch job for %s") % (notebook['name'] or notebook['type']))
+      workflow_doc = WorkflowBuilder().create_notebook_workflow(notebook=notebook, user=self.user, managed=True, name=_("%s for %s") % (OozieApi.BATCH_JOB_PREFIX, notebook['name'] or notebook['type']))
       workflow = Workflow(document=workflow_doc, user=self.user)
     else:
       notebook_doc = Document2.objects.get_by_uuid(user=self.user, uuid=notebook['uuid'], perm_type='read')