Browse Source

HUE-4112 [oozie] Automatically persist the workflow of a scheduled query

Romain Rigaux 9 years ago
parent
commit
6163384223

+ 21 - 5
apps/oozie/src/oozie/models2.py

@@ -39,13 +39,14 @@ from desktop.models import DefaultConfiguration, Document2, Document
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.exceptions import WebHdfsException
 
+from liboozie.oozie_api import get_oozie
 from liboozie.submission2 import Submission
 from liboozie.submission2 import create_directories
+from notebook.models import Notebook
 
 from oozie.conf import REMOTE_SAMPLE_DIR
 from oozie.utils import utc_datetime_format, UTC_TIME_FORMAT, convert_to_server_timezone
 from oozie.importlib.workflows import generate_v2_graph_nodes, MalformedWfDefException, InvalidTagWithNamespaceException
-from liboozie.oozie_api import get_oozie
 
 
 LOG = logging.getLogger(__name__)
@@ -2795,9 +2796,24 @@ class History(object):
 
 class WorkflowBuilder():
   """
-  Focus on building nodes, not the UI layout (should be graph automatically).
+  Focus on building nodes, not the UI layout (should be graphed automatically in dashboard).
+  Only support Hive document currently, but then will have Pig, PySpark, MapReduce...
   """
-  def create_hive_document_workflow(self, name, parameters, user):
+
+  def create_workflow(self, doc_uuid, user, name=None, managed=False):
+    document = Document2.objects.get_by_uuid(user=user, uuid=doc_uuid)
+    notebook = Notebook(document=document)
+    parameters = find_dollar_braced_variables(notebook.get_str())
+
+    if name is None:
+      name = _('Schedule of ') + document.name
+  
+    workflow_doc = self.create_hive_document_workflow(name, doc_uuid, parameters, user, managed=managed)
+    workflow_doc.dependencies.add(document)
+
+    return workflow_doc
+
+  def create_hive_document_workflow(self, name, doc_uuid, parameters, user, managed=False):
     api = get_oozie(user)
 
     credentials = [HiveDocumentAction.DEFAULT_CREDENTIALS] if api.security_enabled else []
@@ -2861,7 +2877,7 @@ class WorkflowBuilder():
           u'properties': {
               u'files': [],
               u'job_xml': u'',
-              u'uuid': uuid,
+              u'uuid': doc_uuid,
               u'parameters': params,
               u'retry_interval': [],
               u'retry_max': [],
@@ -3128,7 +3144,7 @@ class WorkflowBuilder():
       }]}
     )
 
-    workflow_doc = Document2.objects.create(name=name, type='oozie-workflow2', owner=user, data=data, managed=True)
+    workflow_doc = Document2.objects.create(name=name, type='oozie-workflow2', owner=user, data=data, is_managed=managed)
     Document.objects.link(workflow_doc, owner=workflow_doc.owner, name=workflow_doc.name, description=workflow_doc.description, extra='workflow2')
 
     return workflow_doc

+ 0 - 2
apps/oozie/src/oozie/urls.py

@@ -103,8 +103,6 @@ urlpatterns += patterns(
   url(r'^editor/bundle/save/$', 'save_bundle', name='save_bundle'),
   url(r'^editor/bundle/submit/(?P<doc_id>\d+)$', 'submit_bundle', name='editor_submit_bundle'),
   url(r'^editor/bundle/open_v1/$', 'open_old_bundle', name='open_old_bundle'),
-
-  url(r'^editor/document/schedule/$', 'schedule_document', name='schedule_document'),
 )
 
 

+ 15 - 27
apps/oozie/src/oozie/views/editor2.py

@@ -36,8 +36,6 @@ from liboozie.credentials import Credentials
 from liboozie.oozie_api import get_oozie
 from liboozie.submission2 import Submission
 
-from notebook.models import Notebook
-
 from oozie.decorators import check_document_access_permission, check_document_modify_permission,\
   check_editor_access_permission
 from oozie.forms import ParameterForm
@@ -490,7 +488,21 @@ def edit_coordinator(request):
     coordinator = Coordinator()
     coordinator.set_workspace(request.user)
 
-  workflow_uuid = request.GET.get('workflow')
+  # Automatically create the workflow of a scheduled document
+  document_uuid = request.GET.get('document')
+  if document_uuid:
+    # Has already a workflow managing the query for this user?
+    workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid])
+    if workflows.exists():
+      print workflows
+      workflow_doc = workflows.get()
+    else:
+      print 'create'
+      workflow_doc = WorkflowBuilder().create_workflow(doc_uuid=document_uuid, user=request.user, managed=True)
+    workflow_uuid = workflow_doc.uuid
+  elif request.GET.get('workflow'):
+    workflow_uuid = request.GET.get('workflow')
+
   if workflow_uuid:
     coordinator.data['properties']['workflow'] = workflow_uuid
 
@@ -870,27 +882,3 @@ def _submit_bundle(request, bundle, properties):
   except RestException, ex:
     LOG.exception('Error submitting bundle')
     raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
-
-
-@check_editor_access_permission
-def schedule_document(request):
-  if request.method != 'POST':
-    raise PopupException(_('A POST request is required.'))
-
-  uuid = request.POST.get('uuid')
-
-  document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
-  notebook = Notebook(document=document)
-  parameters = find_dollar_braced_variables(notebook.get_str())
-
-  name = _('Schedule of ') + document.name
-
-  workflow_doc = WorkflowBuilder.create_hive_document_workflow(name, parameters, request.user)
-  workflow_doc.dependencies.add(document)
-
-  response = {
-    'status': 0,
-    'url': reverse('oozie:new_coordinator') + '?workflow=' + workflow_doc.uuid
-  }
-
-  return JsonResponse(response)

+ 3 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -1838,8 +1838,10 @@
 
     self.loadScheduler = function() {
       logGA('schedule/edit');
+      // Create or load existing schedule
       $.get("/oozie/editor/coordinator/new/", {
-        format: 'json'
+        format: 'json',
+        document: self.selectedNotebook().uuid()
       }, function (data) {
         $("#schedulerEditor").html(data.layout);
         var viewModel = new CoordinatorEditorViewModel(data.coordinator, data.credentials, data.workflows, data.can_edit);

+ 5 - 4
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -987,7 +987,7 @@ ${ hueIcons.symbols() }
       <li class="active"><a href="#sessionsTab" data-toggle="tab">${_('Sessions')}</a></li>
       % if ENABLE_QUERY_SCHEDULING.get():
       <li><a href="#scheduleTab" data-toggle="tab">${_('Schedule')}</a></li>
-      <li><a href="#scheduleActionsTab" data-toggle="tab">${_('Executions')}</a></li>
+      <li><a href="#scheduledJobsTab" data-toggle="tab">${_('Jobs')}</a></li>
       % endif
     </ul>
 
@@ -1048,9 +1048,10 @@ ${ hueIcons.symbols() }
       <legend><i class="fa fa-calendar"></i> ${ _('Schedule') }</legend>
 
       <!-- ko if: $root.selectedNotebook() && $root.selectedNotebook().id() -->
-        <a data-bind="click: $root.loadScheduler">Load scheduler</a>
+        <a data-bind="click: $root.loadScheduler">Load scheduler</a></br>
+        Save coord</br>
 
-        [Start / Monitor - Stop - Sync]
+        [Start / Monitor - Stop - Sync]</br>
 
         <div id="schedulerEditor">
         </div>
@@ -1062,7 +1063,7 @@ ${ hueIcons.symbols() }
     </div>
 
     ## To move to 'notification' panel
-    <div class="tab-pane" id="scheduleActionsTab">
+    <div class="tab-pane" id="scheduledJobsTab">
       <input type="text" data-bind="value: viewSchedulerId, click: viewScheduler"></input>
 
       <div id="schedulerViewer">