Bläddra i källkod

HUE-3797 [scheduler] Add minimal celery beat submission API implementation

Romain 6 år sedan
förälder
incheckning
65bc5ddd0b

+ 14 - 16
apps/oozie/src/oozie/templates/editor2/common_scheduler.inc.mako

@@ -13,6 +13,7 @@
 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
+
 <%!
 from desktop.views import _ko
 from django.utils.translation import ugettext as _
@@ -20,13 +21,14 @@ from django.utils.translation import ugettext as _
 
 <%namespace name="utils" file="../utils.inc.mako" />
 
+
 <%def name="import_layout(embedded=False)">
 
 <div class="container-fluid">
   <div class="row-fluid">
     <div class="span12 coordinator">
 
-      %if not embedded:
+      % if not embedded:
       <div class="card card-home">
         <h1 class="card-heading simple" style="border-bottom: none"><span data-bind="editable: $root.coordinator.name, editableOptions: {enabled: $root.isEditing(), placement: 'right'}"></span></h1>
         <div class="card-body muted" style="margin-top: 2px" data-bind="visible: $root.isEditing() || (! $root.isEditing() && $root.coordinator.properties.description)">
@@ -56,7 +58,7 @@ from django.utils.translation import ugettext as _
           <!-- /ko -->
         </div>
       </div>
-      %endif
+      % endif
 
       <div class="card card-home" data-bind="visible: coordinator.properties.workflow() || coordinator.properties.document()" style="margin-top: 20px">
         <h1 class="card-heading simple">${ _('How often?') }
@@ -97,14 +99,13 @@ from django.utils.translation import ugettext as _
                 </div>
              </form>
             </div>
-
           </div>
 
           <div data-bind="visible: coordinator.showAdvancedFrequencyUI">
             <form class="
-            %if not embedded:
+            % if not embedded:
             form-horizontal
-            %endif
+            % endif
             ">
 
               <div class="control-group">
@@ -159,12 +160,8 @@ from django.utils.translation import ugettext as _
                   <span class="help-inline"></span>
                 </div>
               </div>
-
             </form>
-
           </div>
-
-
         </div>
       </div>
 
@@ -382,11 +379,10 @@ from django.utils.translation import ugettext as _
     </div>
   </div>
 </div>
-
 </%def>
 
-<%def name="import_modals()">
 
+<%def name="import_modals()">
 <div id="chooseWorkflowDemiModal" class="${ is_embeddable and 'modal' or 'demi-modal' } fade" data-backdrop="${ is_embeddable and 'true' or 'false' }">
   %if is_embeddable:
   <div class="modal-header">
@@ -394,6 +390,7 @@ from django.utils.translation import ugettext as _
     <h2 id="myModalLabel" class="modal-title">${_('Choose a workflow')}</h2>
   </div>
   %endif
+
   <div class="modal-body">
     %if not is_embeddable:
     <a href="javascript: void(0)" data-dismiss="modal" class="pull-right"><i class="fa fa-times"></i></a>
@@ -423,6 +420,7 @@ from django.utils.translation import ugettext as _
   %endif
 </div>
 
+
 <div id="settingsModal" class="modal hide fade">
   <div class="modal-header">
     <button type="button" class="close" data-dismiss="modal" aria-label="${ _('Close') }"><span aria-hidden="true">&times;</span></button>
@@ -453,9 +451,9 @@ from django.utils.translation import ugettext as _
 
       <h4>${ _('Execution') }</h4>
       <select data-bind="value: coordinator.properties.execution">
-    <option value="FIFO">${ _("FIFO (oldest first)") }</option>
-    <option value="LIFO">${ _("LIFO (newest first)") }</option>
-    <option value="LAST_ONLY">${ _("LAST_ONLY (discards all older materializations)") }</option>
+        <option value="FIFO">${ _("FIFO (oldest first)") }</option>
+        <option value="LIFO">${ _("LIFO (newest first)") }</option>
+        <option value="LAST_ONLY">${ _("LAST_ONLY (discards all older materializations)") }</option>
       </select>
 
       <h4>${ _('Throttle') }</h4>
@@ -495,7 +493,7 @@ from django.utils.translation import ugettext as _
   var datasetTypeaheadSource = ["/data/${'${'}YEAR}/${'${'}MONTH}/${'${'}DAY}", "${'${'}MINUTE}", "${'${'}HOUR}", "${'${'}DAY}", "${'${'}MONTH}", "${'${'}YEAR}", "${'${'}coord:nominalTime()}", "${'${'}coord:formatTime(coord:nominalTime(), 'yyyyMMdd')}"]
 
   function zeroPadding(value) {
-    return (value < 10 ? '0':'') + value;
+    return (value < 10 ? '0' : '') + value;
   }
 
   function convertDatasetVariables(path, hasSameStart, customStart, hasSameFrequency, customFrequencyUnit, startInstance, instanceChoice, vm) {
@@ -508,6 +506,7 @@ from django.utils.translation import ugettext as _
       qty: 0,
       freq: "minutes"
     };
+
     if (startInstance != 0 && instanceChoice == "single") {
       _startDiffObj.qty = startInstance;
       if (hasSameFrequency) {
@@ -602,6 +601,5 @@ from django.utils.translation import ugettext as _
       });
     }
   }
-
   </script>
 </%def>

+ 14 - 14
apps/oozie/src/oozie/views/editor2.py

@@ -411,9 +411,6 @@ def _submit_workflow_helper(request, workflow, submit_action):
       if '/submit_single_action/' in submit_action:
         mapping['submit_single_action'] = True
 
-      if 'altus' in cluster.get('type', ''):
-        mapping['cluster'] = cluster.get('id')
-
       try:
         job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)
       except Exception as e:
@@ -432,17 +429,20 @@ def _submit_workflow_helper(request, workflow, submit_action):
     initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
     params_form = ParametersFormSet(initial=initial_params)
 
-
-    return render('/scheduler/submit_job_popup.mako', request, {
-                     'params_form': params_form,
-                     'name': workflow.name,
-                     'action': submit_action,
-                     'show_dryrun': True,
-                     'email_id': request.user.email,
-                     'is_oozie_mail_enabled': _is_oozie_mail_enabled(request.user),
-                     'return_json': request.GET.get('format') == 'json',
-                     'cluster_json': cluster_json
-                   }, force_template=True)
+    return render(
+      '/scheduler/submit_job_popup.mako',
+      request, {
+        'params_form': params_form,
+        'name': workflow.name,
+        'action': submit_action,
+        'show_dryrun': True,
+        'email_id': request.user.email,
+        'is_oozie_mail_enabled': _is_oozie_mail_enabled(request.user),
+        'return_json': request.GET.get('format') == 'json',
+        'cluster_json': cluster_json
+      },
+      force_template=True
+    )
 
 
 def _is_oozie_mail_enabled(user):

+ 4 - 3
desktop/core/src/desktop/lib/scheduler/api.py

@@ -24,6 +24,7 @@ from desktop.auth.backend import is_admin
 from desktop.conf import TASK_SERVER
 from desktop.models import Document2
 from desktop.lib.django_util import JsonResponse, render
+from desktop.lib.i18n import force_unicode
 from desktop.lib.scheduler.lib.api import get_api
 
 LOG = logging.getLogger(__name__)
@@ -46,7 +47,7 @@ def get_schedule(request):
 # To move to lib in case oozie is blacklisted
 #@check_document_access_permission()
 def submit_schedule(request, doc_id):
-  interface = request.GET.get('interface', request.POST.get('interface', 'oozie'))
+  interface = request.GET.get('interface', request.POST.get('interface', 'beat'))
   if doc_id.isdigit():
     coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
   else:
@@ -83,8 +84,8 @@ def submit_schedule(request, doc_id):
       request, {
           'params_form': params_form,
           'name': coordinator.name,
-          'action': reverse('oozie:editor_submit_coordinator',  kwargs={'doc_id': coordinator.id}),
-          'show_dryrun': True,
+          'action': '/scheduler/api/schedule/submit/%s' % coordinator.id,
+          'show_dryrun': False,
           'return_json': request.GET.get('format') == 'json',
           'interface': interface
       },

+ 5 - 6
desktop/core/src/desktop/lib/scheduler/lib/api.py

@@ -20,10 +20,10 @@ from builtins import object
 def get_api(request, interface):
   if interface == 'beat':
     from desktop.lib.scheduler.lib.beat import CeleryBeatApi
-    return CeleryBeatApi(user=request.user)
+    return CeleryBeatApi(interface=interface, user=request.user)
   elif interface == 'oozie':
     from desktop.lib.scheduler.lib.oozie import OozieApi
-    return OozieApi(user=request.user)
+    return OozieApi(interface=interface, user=request.user)
   else:
     raise PopupException(_('Scheduler connector interface not recognized: %s') % interface)
 
@@ -34,9 +34,8 @@ class Api(object):
     self.interface = interface
     self.user = user
 
-  def get_schedule():
-    return JsonResponse({
-    })
+  def get_schedule(self):
+    return JsonResponse({})
 
-  def submit_schedule(request, coordinator, mapping):
+  def submit_schedule(self, request, coordinator, mapping):
     pass

+ 29 - 1
desktop/core/src/desktop/lib/scheduler/lib/beat.py

@@ -1,7 +1,35 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
+from django_celery_beat.models import PeriodicTask, IntervalSchedule
 
 from desktop.lib.scheduler.lib.api import Api
 
 
 class CeleryBeatApi(Api):
-  pass
+
+  def submit_schedule(self, request, coordinator, mapping):
+    schedule, created = IntervalSchedule.objects.get_or_create(
+      every=10,
+      period=IntervalSchedule.SECONDS,
+    )
+
+    task, created = PeriodicTask.objects.get_or_create(
+      interval=schedule,
+      name='Scheduled query',
+      task='notebook.tasks.run_sync_query',
+    )

+ 17 - 1
desktop/core/src/desktop/lib/scheduler/lib/oozie.py

@@ -1,3 +1,19 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 from __future__ import absolute_import
 
 from desktop.lib.scheduler.lib.api import Api
@@ -7,5 +23,5 @@ from oozie.views.editor2 import _submit_coordinator
 
 class OozieApi(Api):
 
-  def submit_schedule(request, coordinator, mapping):
+  def submit_schedule(self, request, coordinator, mapping):
     return _submit_coordinator(request, coordinator, mapping)

+ 1 - 1
desktop/core/src/desktop/settings.py

@@ -687,7 +687,7 @@ if DEBUG and desktop.conf.ENABLE_DJANGO_DEBUG_TOOL.get():
 # Celery settings
 ################################################################
 
-if desktop.conf.TASK_SERVER.ENABLED.get():
+if desktop.conf.TASK_SERVER.ENABLED.get() or desktop.conf.TASK_SERVER.BEAT_ENABLED.get():
   CELERY_BROKER_URL = desktop.conf.TASK_SERVER.BROKER_URL.get()
 
   CELERY_ACCEPT_CONTENT = ['json']

+ 61 - 60
desktop/core/src/desktop/templates/scheduler/submit_job_popup.mako

@@ -34,72 +34,73 @@
   </div>
 
   <div class="modal-body">
-      <div id="param-container">
-
-       ${ params_form.management_form | n,unicode }
-
-       % for form in params_form.forms:
-          % for hidden in form.hidden_fields():
-            ${ hidden | n,unicode }
-          % endfor
-
-          <div class="control-group"
-            % if form['name'].form.initial.get('name').startswith('oozie.'):
-                style="display: none"
-            % endif
-          >
-            <label class="control-label">${ form['name'].form.initial.get('name') }</label>
-            <div class="controls">
-              ${ render_field(form['value'], show_label=False, extra_attrs={'class': 'filechooser-input input-xlarge'}) }
-              <div class="btn-group">
-                <a class="btn btn-default dropdown-toggle" data-toggle="dropdown">
-                  <i class="fa fa-calendar"></i>
-                  <span class="caret"></span>
-                </a>
-                <ul class="dropdown-menu pull-right" role="menu">
-                  <li>
-                    <a class="pointer now-link">
-                      ${ _('Now') }
-                    </a>
-                    <a class="pointer calendar-link">
-                      ${ _('Calendar') }
-                    </a>
-                  </li>
-                </ul>
-              </div>
+    <div id="param-container">
+
+      ${ params_form.management_form | n,unicode }
+
+      % for form in params_form.forms:
+        % for hidden in form.hidden_fields():
+          ${ hidden | n,unicode }
+        % endfor
+
+        <div class="control-group"
+          % if form['name'].form.initial.get('name').startswith('oozie.'):
+            style="display: none"
+          % endif
+        >
+          <label class="control-label">${ form['name'].form.initial.get('name') }</label>
+          <div class="controls">
+            ${ render_field(form['value'], show_label=False, extra_attrs={'class': 'filechooser-input input-xlarge'}) }
+            <div class="btn-group">
+              <a class="btn btn-default dropdown-toggle" data-toggle="dropdown">
+                <i class="fa fa-calendar"></i>
+                <span class="caret"></span>
+              </a>
+              <ul class="dropdown-menu pull-right" role="menu">
+                <li>
+                  <a class="pointer now-link">
+                    ${ _('Now') }
+                  </a>
+                  <a class="pointer calendar-link">
+                    ${ _('Calendar') }
+                  </a>
+                </li>
+              </ul>
             </div>
           </div>
+        </div>
+        % endfor
+    </div>
 
-         % endfor
-      </div>
-      <div class="clearfix"></div>
+    <div class="clearfix"></div>
 
-      % if show_dryrun:
-         <label class="checkbox" style="display: inline-block; margin-top: 5px">
-           <input type="checkbox" name="dryrun_checkbox" /> ${ _('Do a dryrun before submitting the job') }
-         </label>
-      % endif
-      % if is_oozie_mail_enabled:
-        <br/>
-        <label class="checkbox" style="display: inline-block; margin-top: 5px">
-          <input type="checkbox" name="email_checkbox"
-          % if not email_id:
-            disabled
-          % endif
-          />
-        % if email_id:
-          ${_('Send completion email to ')}<a href="/useradmin/users/edit/${user.username}#step2"> ${email_id} </a>
-        % else:
-          ${_('Email not set in ')}<a href="/useradmin/users/edit/${user.username}#step2"> ${_('profile.')} </a>
-        % endif
-        % if cluster_json:
-          <input type="hidden" name="cluster" value="${ cluster_json }"></input>
+    % if show_dryrun:
+      <label class="checkbox" style="display: inline-block; margin-top: 5px">
+        <input type="checkbox" name="dryrun_checkbox" /> ${ _('Do a dryrun before submitting the job') }
+      </label>
+    % endif
+
+    % if is_oozie_mail_enabled:
+      <br/>
+      <label class="checkbox" style="display: inline-block; margin-top: 5px">
+        <input type="checkbox" name="email_checkbox"
+        % if not email_id:
+          disabled
         % endif
-        </label>
-        %endif
-      % if return_json:
-        <input type="hidden" name="format" value="json">
+        />
+      % if email_id:
+        ${_('Send completion email to ')}<a href="/useradmin/users/edit/${user.username}#step2">${ email_id }</a>
+      % else:
+        ${_('Email not set in ')}<a href="/useradmin/users/edit/${user.username}#step2">${ _('profile.') }</a>
+      % endif
+      % if cluster_json:
+        <input type="hidden" name="cluster" value="${ cluster_json }"></input>
       % endif
+      </label>
+      %endif
+    % if return_json:
+      <input type="hidden" name="format" value="json">
+    % endif
   </div>
   <div class="modal-footer">
     <a href="#" class="btn" data-dismiss="modal">${ _('Cancel') }</a>