فهرست منبع

HUE-1658 [oozie] Adding backward compatibility flag

Set this to false to use the old frequency representation instead of the crontab.

[oozie]
  # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
  enable_cron_scheduling=false
Romain Rigaux 11 سال پیش
والد
کامیت
8c4a8ca

+ 19 - 6
apps/oozie/src/oozie/conf.py

@@ -19,7 +19,7 @@ import os.path
 
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
-from desktop.lib.conf import Config
+from desktop.lib.conf import Config, coerce_bool
 from desktop.lib import paths
 from liboozie.conf import get_oozie_status
 
@@ -30,30 +30,43 @@ DEFINITION_XSLT_DIR = Config(
   key="definition_xslt_dir",
   default=os.path.join(os.path.dirname(__file__), "importlib", "xslt"),
   help=_t("Location on local FS where the xslt files are stored for workflow import."),
-  private=True)
+  private=True
+)
 
 LOCAL_SAMPLE_DIR = Config(
   key="local_data_dir",
   default=os.path.join(os.path.dirname(__file__), "..", "..", "examples"),
   help=_t("Location on local filesystem where the examples are stored."),
-  private=True)
+  private=True
+)
 
 LOCAL_SAMPLE_DATA_DIR = Config(
   key="sample_data_dir",
   default=paths.get_thirdparty_root("sample_data"),
   help=_t("Location on local filesystem where the data for the examples is stored."),
-  private=True)
+  private=True
+)
 
 REMOTE_SAMPLE_DIR = Config(
   key="remote_data_dir",
   default="/user/hue/oozie/workspaces",
-  help=_t("Location on HDFS where the Oozie workflows are stored."))
+  help=_t("Location on HDFS where the Oozie workflows are stored.")
+)
 
 OOZIE_JOBS_COUNT = Config(
   key='oozie_jobs_count',
   default=100,
   type=int,
-  help=_t('Maximum number of Oozie workflows or coodinators or bundles to retrieve in one API call.'))
+  help=_t('Maximum number of Oozie workflows or coodinators or bundles to retrieve in one API call.')
+)
+
+ENABLE_CRON_SCHEDULING = Config( # Until Hue 4
+  key='enable_cron_scheduling',
+  default=True,
+  type=coerce_bool,
+  help=_t('Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.')
+)
+
 
 def config_validator(user):
   res = []

+ 7 - 2
apps/oozie/src/oozie/forms.py

@@ -25,10 +25,13 @@ from django.utils.functional import curry
 from django.utils.translation import ugettext_lazy as _t
 
 from desktop.lib.django_forms import MultiForm, SplitDateTimeWidget
+from desktop.models import Document
+
+from oozie.conf import ENABLE_CRON_SCHEDULING
 from oozie.models import Workflow, Node, Java, Mapreduce, Streaming, Coordinator,\
   Dataset, DataInput, DataOutput, Pig, Link, Hive, Sqoop, Ssh, Shell, DistCp, Fs,\
   Email, SubWorkflow, Generic, Bundle, BundledCoordinator
-from desktop.models import Document
+
 
 
 LOG = logging.getLogger(__name__)
@@ -338,7 +341,9 @@ class CoordinatorForm(forms.ModelForm):
 
   class Meta:
     model = Coordinator
-    exclude = ('owner', 'deployment_dir', 'frequency_number', 'frequency_unit')
+    exclude = ('owner', 'deployment_dir')
+    if ENABLE_CRON_SCHEDULING.get():
+        exclude += ('frequency_number', 'frequency_unit')
     widgets = {
       'description': forms.TextInput(attrs={'class': 'span5'}),
       'parameters': forms.widgets.HiddenInput(),

+ 4 - 1
apps/oozie/src/oozie/templates/editor/coordinator_utils.mako

@@ -26,8 +26,11 @@
     <div class="controls">
       <div class="row-fluid">
         <div class="span9">
+          <a data-bind="visible: isAdvancedCron" href="http://quartz-scheduler.org/api/2.0.0/org/quartz/CronExpression.html" class="pull-right" target="_blank">&nbsp;<i class="fa fa-question-circle" title="${ _('Check syntax ?') }"></i></a>
           <input data-bind="visible: isAdvancedCron" id="coord-frequency" name="cron_frequency" class="pull-right"/>
-          <span data-bind="visible: isAdvancedCron" class="pull-right" style="padding-right:20px">${ _('Crontab') }</span>&nbsp;
+          <span data-bind="visible: isAdvancedCron" class="pull-right" style="padding-right:20px">
+            ${ _('Crontab') }
+          </span>&nbsp;
           <input type="checkbox" name="isAdvancedCron" data-bind="checked: isAdvancedCron" /> (${ _('advanced') })
         </div>
         <div class="span3">

+ 15 - 2
apps/oozie/src/oozie/templates/editor/create_coordinator.mako

@@ -86,7 +86,18 @@ ${ layout.menubar(section='coordinators') }
           <div id="step2" class="stepDetails hide">
             <div class="alert alert-info"><h3>${ _('Frequency') }</h3></div>
             <div class="fieldWrapper">
-              ${ coordinator_utils.frequency_fields() }
+              % if enable_cron_scheduling:
+                ${ coordinator_utils.frequency_fields() }
+              % else:
+                <div class="row-fluid">
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_number']) }
+                  </div>
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_unit']) }
+                  </div>
+                </div>
+              % endif
             </div>
             <div class="fieldWrapper">
               <div class="row-fluid">
@@ -124,7 +135,9 @@ ${ layout.menubar(section='coordinators') }
 <script type="text/javascript" charset="utf-8">
   $(document).ready(function () {
 
-    initCoordinator(${ coordinator_frequency | n,unicode });
+    % if enable_cron_scheduling:
+      initCoordinator(${ coordinator_frequency | n,unicode });
+    % endif
 
     var currentStep = "step1";
 

+ 15 - 2
apps/oozie/src/oozie/templates/editor/edit_coordinator.mako

@@ -127,7 +127,18 @@ ${ layout.menubar(section='coordinators') }
               </div>
             </div>
             <div class="fieldWrapper">
-              ${ coordinator_utils.frequency_fields() }
+              % if enable_cron_scheduling:
+                ${ coordinator_utils.frequency_fields() }
+              % else:
+                <div class="row-fluid">
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_number']) }
+                  </div>
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_unit']) }
+                  </div>
+                </div>
+              % endif
             </div>
             <div class="fieldWrapper">
               <div class="row-fluid">
@@ -611,7 +622,9 @@ ${ layout.menubar(section='coordinators') }
         self.sla = ko.mapping.fromJS(${ coordinator.sla_jsescaped | n,unicode });
       };
 
-    initCoordinator(${ coordinator_frequency | n,unicode });
+      % if enable_cron_scheduling:
+        initCoordinator(${ coordinator_frequency | n,unicode });
+      % endif
 
       window.slaModel = new slaModel();
       ko.applyBindings(window.slaModel, document.getElementById('slaEditord'));

+ 5 - 0
apps/oozie/src/oozie/templates/editor/gen/coordinator.xml.mako

@@ -17,6 +17,7 @@
 
 <%!
   from oozie.utils import smart_path
+  from oozie.conf import ENABLE_CRON_SCHEDULING
 %>
 
 <%namespace name="common" file="workflow-common.xml.mako" />
@@ -51,7 +52,11 @@
 
 
 <coordinator-app name="${ coord.name }"
+  % if ENABLE_CRON_SCHEDULING.get():
   frequency="${ coord.cron_frequency['frequency'] }"
+  % else:
+  frequency="${ coord.frequency }"
+  % endif
   start="${ coord.start_utc }" end="${ coord.end_utc }" timezone="${ coord.timezone }"
   xmlns="${ 'uri:oozie:coordinator:0.4' if coord.sla_enabled else coord.schema_version | n,unicode }"
   ${ 'xmlns:sla="uri:oozie:sla:0.2"' if coord.sla_enabled else '' | n,unicode }>

+ 43 - 1
apps/oozie/src/oozie/tests.py

@@ -44,6 +44,7 @@ from liboozie.oozie_api_test import OozieServerProvider
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
 
+from oozie.conf import ENABLE_CRON_SCHEDULING
 from oozie.models import Workflow, Node, Kill, Link, Job, Coordinator, History,\
   find_parameters, NODE_TYPES, Bundle
 from oozie.utils import workflow_to_dict, model_to_dict, smart_path
@@ -1470,6 +1471,45 @@ class TestEditor(OozieMockBase):
   def test_coordinator_gen_xml(self):
     coord = create_coordinator(self.wf, self.c, self.user)
 
+    finish = ENABLE_CRON_SCHEDULING.set_for_testing(False)
+
+    try:
+      assert_true(
+  """
+<coordinator-app name="MyCoord"
+  frequency="${coord:days(1)}"
+  start="2012-07-01T00:00Z" end="2012-07-04T00:00Z" timezone="America/Los_Angeles"
+  xmlns="uri:oozie:coordinator:0.2"
+  >
+  <controls>
+    <timeout>100</timeout>
+    <concurrency>3</concurrency>
+    <execution>FIFO</execution>
+    <throttle>10</throttle>
+  </controls>
+  <action>
+    <workflow>
+      <app-path>${wf_application_path}</app-path>
+      <configuration>
+        <property>
+          <name>username</name>
+          <value>${coord:user()}</value>
+        </property>
+        <property>
+          <name>SLEEP</name>
+          <value>1000</value>
+        </property>
+        <property>
+          <name>market</name>
+          <value>US</value>
+        </property>
+      </configuration>
+   </workflow>
+  </action>
+</coordinator-app>""" in coord.to_xml(), coord.to_xml())
+    finally:
+      finish()
+
     assert_true(
 """<coordinator-app name="MyCoord"
   frequency="0 0 * * *"
@@ -3447,7 +3487,9 @@ COORDINATOR_DICT = {
     u'concurrency': [u'3'],
     u'execution': [u'FIFO'],
     u'throttle': [u'10'],
-    u'schema_version': [u'uri:oozie:coordinator:0.2']
+    u'schema_version': [u'uri:oozie:coordinator:0.2'],
+    u'cron_frequency': [u'0 0 * * *'],
+    u'isAdvancedCron': [u'on'],
 }
 BUNDLE_DICT = {
     u'name': [u'MyBundle'], u'description': [u'Description of my bundle'],

+ 12 - 3
apps/oozie/src/oozie/views/editor.py

@@ -43,6 +43,7 @@ from liboozie.submittion import Submission
 from filebrowser.lib.archives import archive_factory
 from oozie.decorators import check_job_access_permission, check_job_edition_permission,\
                              check_dataset_access_permission, check_dataset_edition_permission
+from oozie.conf import ENABLE_CRON_SCHEDULING
 from oozie.importlib.workflows import import_workflow as _import_workflow
 from oozie.importlib.coordinators import import_coordinator as _import_coordinator
 from oozie.management.commands import oozie_setup
@@ -383,12 +384,15 @@ def create_coordinator(request, workflow=None):
   else:
     coordinator = Coordinator(owner=request.user, schema_version="uri:oozie:coordinator:0.2")
 
+  enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
+
   if request.method == 'POST':
     coordinator_form = CoordinatorForm(request.POST, instance=coordinator, user=request.user)
 
     if coordinator_form.is_valid():
       coordinator = coordinator_form.save()
-      coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
+      if enable_cron_scheduling:
+        coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
       coordinator.save()
       Document.objects.link(coordinator, owner=coordinator.owner, name=coordinator.name, description=coordinator.description)
       return redirect(reverse('oozie:edit_coordinator', kwargs={'coordinator': coordinator.id}) + "#step3")
@@ -401,6 +405,7 @@ def create_coordinator(request, workflow=None):
     'coordinator': coordinator,
     'coordinator_form': coordinator_form,
     'coordinator_frequency': json.dumps(coordinator.cron_frequency),
+    'enable_cron_scheduling': enable_cron_scheduling,
   })
 
 
@@ -462,6 +467,8 @@ def edit_coordinator(request, coordinator):
   NewDataOutputFormSet = inlineformset_factory(Coordinator, DataOutput, form=DataOutputForm, extra=0, can_order=False, can_delete=False)
   NewDataOutputFormSet.form = staticmethod(curry(DataOutputForm, coordinator=coordinator))
 
+  enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
+
   if request.method == 'POST':
     coordinator_form = CoordinatorForm(request.POST, instance=coordinator, user=request.user)
     dataset_formset = DatasetFormSet(request.POST, request.FILES, instance=coordinator)
@@ -479,7 +486,8 @@ def edit_coordinator(request, coordinator):
       new_data_input_formset.save()
       new_data_output_formset.save()
       coordinator.sla = json.loads(request.POST.get('sla'))
-      coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
+      if enable_cron_scheduling:
+        coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
       coordinator.save()
 
       request.info(_('Coordinator saved.'))
@@ -503,7 +511,8 @@ def edit_coordinator(request, coordinator):
     'new_data_input_formset': new_data_input_formset,
     'new_data_output_formset': new_data_output_formset,
     'history': history,
-    'coordinator_frequency': json.dumps(coordinator.cron_frequency)
+    'coordinator_frequency': json.dumps(coordinator.cron_frequency),
+    'enable_cron_scheduling': enable_cron_scheduling,
   })
 
 

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -501,6 +501,9 @@
   # Maximum of Oozie workflows or coodinators to retrieve in one API call.
   ## oozie_jobs_count=100
 
+  # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
+  ## enable_cron_scheduling=true
+
 
 ###########################################################################
 # Settings to configure Beeswax with Hive

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -542,6 +542,9 @@
   # Maximum of Oozie workflows or coodinators to retrieve in one API call.
   ## oozie_jobs_count=100
 
+  # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
+  ## enable_cron_scheduling=true
+
 
 ###########################################################################
 # Settings to configure Beeswax with Hive