Browse Source

HUE-1658 [oozie] Adding backward compatibility flag

Set this to false to use the old frequency representation instead of the crontab.

[oozie]
  # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
  enable_cron_scheduling=false
Romain Rigaux 11 years ago
parent
commit
8c4a8ca

+ 19 - 6
apps/oozie/src/oozie/conf.py

@@ -19,7 +19,7 @@ import os.path
 
 
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
 
-from desktop.lib.conf import Config
+from desktop.lib.conf import Config, coerce_bool
 from desktop.lib import paths
 from desktop.lib import paths
 from liboozie.conf import get_oozie_status
 from liboozie.conf import get_oozie_status
 
 
@@ -30,30 +30,43 @@ DEFINITION_XSLT_DIR = Config(
   key="definition_xslt_dir",
   key="definition_xslt_dir",
   default=os.path.join(os.path.dirname(__file__), "importlib", "xslt"),
   default=os.path.join(os.path.dirname(__file__), "importlib", "xslt"),
   help=_t("Location on local FS where the xslt files are stored for workflow import."),
   help=_t("Location on local FS where the xslt files are stored for workflow import."),
-  private=True)
+  private=True
+)
 
 
 LOCAL_SAMPLE_DIR = Config(
 LOCAL_SAMPLE_DIR = Config(
   key="local_data_dir",
   key="local_data_dir",
   default=os.path.join(os.path.dirname(__file__), "..", "..", "examples"),
   default=os.path.join(os.path.dirname(__file__), "..", "..", "examples"),
   help=_t("Location on local filesystem where the examples are stored."),
   help=_t("Location on local filesystem where the examples are stored."),
-  private=True)
+  private=True
+)
 
 
 LOCAL_SAMPLE_DATA_DIR = Config(
 LOCAL_SAMPLE_DATA_DIR = Config(
   key="sample_data_dir",
   key="sample_data_dir",
   default=paths.get_thirdparty_root("sample_data"),
   default=paths.get_thirdparty_root("sample_data"),
   help=_t("Location on local filesystem where the data for the examples is stored."),
   help=_t("Location on local filesystem where the data for the examples is stored."),
-  private=True)
+  private=True
+)
 
 
 REMOTE_SAMPLE_DIR = Config(
 REMOTE_SAMPLE_DIR = Config(
   key="remote_data_dir",
   key="remote_data_dir",
   default="/user/hue/oozie/workspaces",
   default="/user/hue/oozie/workspaces",
-  help=_t("Location on HDFS where the Oozie workflows are stored."))
+  help=_t("Location on HDFS where the Oozie workflows are stored.")
+)
 
 
 OOZIE_JOBS_COUNT = Config(
 OOZIE_JOBS_COUNT = Config(
   key='oozie_jobs_count',
   key='oozie_jobs_count',
   default=100,
   default=100,
   type=int,
   type=int,
-  help=_t('Maximum number of Oozie workflows or coodinators or bundles to retrieve in one API call.'))
+  help=_t('Maximum number of Oozie workflows or coodinators or bundles to retrieve in one API call.')
+)
+
+ENABLE_CRON_SCHEDULING = Config( # Until Hue 4
+  key='enable_cron_scheduling',
+  default=True,
+  type=coerce_bool,
+  help=_t('Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.')
+)
+
 
 
 def config_validator(user):
 def config_validator(user):
   res = []
   res = []

+ 7 - 2
apps/oozie/src/oozie/forms.py

@@ -25,10 +25,13 @@ from django.utils.functional import curry
 from django.utils.translation import ugettext_lazy as _t
 from django.utils.translation import ugettext_lazy as _t
 
 
 from desktop.lib.django_forms import MultiForm, SplitDateTimeWidget
 from desktop.lib.django_forms import MultiForm, SplitDateTimeWidget
+from desktop.models import Document
+
+from oozie.conf import ENABLE_CRON_SCHEDULING
 from oozie.models import Workflow, Node, Java, Mapreduce, Streaming, Coordinator,\
 from oozie.models import Workflow, Node, Java, Mapreduce, Streaming, Coordinator,\
   Dataset, DataInput, DataOutput, Pig, Link, Hive, Sqoop, Ssh, Shell, DistCp, Fs,\
   Dataset, DataInput, DataOutput, Pig, Link, Hive, Sqoop, Ssh, Shell, DistCp, Fs,\
   Email, SubWorkflow, Generic, Bundle, BundledCoordinator
   Email, SubWorkflow, Generic, Bundle, BundledCoordinator
-from desktop.models import Document
+
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -338,7 +341,9 @@ class CoordinatorForm(forms.ModelForm):
 
 
   class Meta:
   class Meta:
     model = Coordinator
     model = Coordinator
-    exclude = ('owner', 'deployment_dir', 'frequency_number', 'frequency_unit')
+    exclude = ('owner', 'deployment_dir')
+    if ENABLE_CRON_SCHEDULING.get():
+        exclude += ('frequency_number', 'frequency_unit')
     widgets = {
     widgets = {
       'description': forms.TextInput(attrs={'class': 'span5'}),
       'description': forms.TextInput(attrs={'class': 'span5'}),
       'parameters': forms.widgets.HiddenInput(),
       'parameters': forms.widgets.HiddenInput(),

+ 4 - 1
apps/oozie/src/oozie/templates/editor/coordinator_utils.mako

@@ -26,8 +26,11 @@
     <div class="controls">
     <div class="controls">
       <div class="row-fluid">
       <div class="row-fluid">
         <div class="span9">
         <div class="span9">
+          <a data-bind="visible: isAdvancedCron" href="http://quartz-scheduler.org/api/2.0.0/org/quartz/CronExpression.html" class="pull-right" target="_blank">&nbsp;<i class="fa fa-question-circle" title="${ _('Check syntax ?') }"></i></a>
           <input data-bind="visible: isAdvancedCron" id="coord-frequency" name="cron_frequency" class="pull-right"/>
           <input data-bind="visible: isAdvancedCron" id="coord-frequency" name="cron_frequency" class="pull-right"/>
-          <span data-bind="visible: isAdvancedCron" class="pull-right" style="padding-right:20px">${ _('Crontab') }</span>&nbsp;
+          <span data-bind="visible: isAdvancedCron" class="pull-right" style="padding-right:20px">
+            ${ _('Crontab') }
+          </span>&nbsp;
           <input type="checkbox" name="isAdvancedCron" data-bind="checked: isAdvancedCron" /> (${ _('advanced') })
           <input type="checkbox" name="isAdvancedCron" data-bind="checked: isAdvancedCron" /> (${ _('advanced') })
         </div>
         </div>
         <div class="span3">
         <div class="span3">

+ 15 - 2
apps/oozie/src/oozie/templates/editor/create_coordinator.mako

@@ -86,7 +86,18 @@ ${ layout.menubar(section='coordinators') }
           <div id="step2" class="stepDetails hide">
           <div id="step2" class="stepDetails hide">
             <div class="alert alert-info"><h3>${ _('Frequency') }</h3></div>
             <div class="alert alert-info"><h3>${ _('Frequency') }</h3></div>
             <div class="fieldWrapper">
             <div class="fieldWrapper">
-              ${ coordinator_utils.frequency_fields() }
+              % if enable_cron_scheduling:
+                ${ coordinator_utils.frequency_fields() }
+              % else:
+                <div class="row-fluid">
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_number']) }
+                  </div>
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_unit']) }
+                  </div>
+                </div>
+              % endif
             </div>
             </div>
             <div class="fieldWrapper">
             <div class="fieldWrapper">
               <div class="row-fluid">
               <div class="row-fluid">
@@ -124,7 +135,9 @@ ${ layout.menubar(section='coordinators') }
 <script type="text/javascript" charset="utf-8">
 <script type="text/javascript" charset="utf-8">
   $(document).ready(function () {
   $(document).ready(function () {
 
 
-    initCoordinator(${ coordinator_frequency | n,unicode });
+    % if enable_cron_scheduling:
+      initCoordinator(${ coordinator_frequency | n,unicode });
+    % endif
 
 
     var currentStep = "step1";
     var currentStep = "step1";
 
 

+ 15 - 2
apps/oozie/src/oozie/templates/editor/edit_coordinator.mako

@@ -127,7 +127,18 @@ ${ layout.menubar(section='coordinators') }
               </div>
               </div>
             </div>
             </div>
             <div class="fieldWrapper">
             <div class="fieldWrapper">
-              ${ coordinator_utils.frequency_fields() }
+              % if enable_cron_scheduling:
+                ${ coordinator_utils.frequency_fields() }
+              % else:
+                <div class="row-fluid">
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_number']) }
+                  </div>
+                  <div class="span6">
+                    ${ utils.render_field_no_popover(coordinator_form['frequency_unit']) }
+                  </div>
+                </div>
+              % endif
             </div>
             </div>
             <div class="fieldWrapper">
             <div class="fieldWrapper">
               <div class="row-fluid">
               <div class="row-fluid">
@@ -611,7 +622,9 @@ ${ layout.menubar(section='coordinators') }
         self.sla = ko.mapping.fromJS(${ coordinator.sla_jsescaped | n,unicode });
         self.sla = ko.mapping.fromJS(${ coordinator.sla_jsescaped | n,unicode });
       };
       };
 
 
-    initCoordinator(${ coordinator_frequency | n,unicode });
+      % if enable_cron_scheduling:
+        initCoordinator(${ coordinator_frequency | n,unicode });
+      % endif
 
 
       window.slaModel = new slaModel();
       window.slaModel = new slaModel();
       ko.applyBindings(window.slaModel, document.getElementById('slaEditord'));
       ko.applyBindings(window.slaModel, document.getElementById('slaEditord'));

+ 5 - 0
apps/oozie/src/oozie/templates/editor/gen/coordinator.xml.mako

@@ -17,6 +17,7 @@
 
 
 <%!
 <%!
   from oozie.utils import smart_path
   from oozie.utils import smart_path
+  from oozie.conf import ENABLE_CRON_SCHEDULING
 %>
 %>
 
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 <%namespace name="common" file="workflow-common.xml.mako" />
@@ -51,7 +52,11 @@
 
 
 
 
 <coordinator-app name="${ coord.name }"
 <coordinator-app name="${ coord.name }"
+  % if ENABLE_CRON_SCHEDULING.get():
   frequency="${ coord.cron_frequency['frequency'] }"
   frequency="${ coord.cron_frequency['frequency'] }"
+  % else:
+  frequency="${ coord.frequency }"
+  % endif
   start="${ coord.start_utc }" end="${ coord.end_utc }" timezone="${ coord.timezone }"
   start="${ coord.start_utc }" end="${ coord.end_utc }" timezone="${ coord.timezone }"
   xmlns="${ 'uri:oozie:coordinator:0.4' if coord.sla_enabled else coord.schema_version | n,unicode }"
   xmlns="${ 'uri:oozie:coordinator:0.4' if coord.sla_enabled else coord.schema_version | n,unicode }"
   ${ 'xmlns:sla="uri:oozie:sla:0.2"' if coord.sla_enabled else '' | n,unicode }>
   ${ 'xmlns:sla="uri:oozie:sla:0.2"' if coord.sla_enabled else '' | n,unicode }>

+ 43 - 1
apps/oozie/src/oozie/tests.py

@@ -44,6 +44,7 @@ from liboozie.oozie_api_test import OozieServerProvider
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
 
 
+from oozie.conf import ENABLE_CRON_SCHEDULING
 from oozie.models import Workflow, Node, Kill, Link, Job, Coordinator, History,\
 from oozie.models import Workflow, Node, Kill, Link, Job, Coordinator, History,\
   find_parameters, NODE_TYPES, Bundle
   find_parameters, NODE_TYPES, Bundle
 from oozie.utils import workflow_to_dict, model_to_dict, smart_path
 from oozie.utils import workflow_to_dict, model_to_dict, smart_path
@@ -1470,6 +1471,45 @@ class TestEditor(OozieMockBase):
   def test_coordinator_gen_xml(self):
   def test_coordinator_gen_xml(self):
     coord = create_coordinator(self.wf, self.c, self.user)
     coord = create_coordinator(self.wf, self.c, self.user)
 
 
+    finish = ENABLE_CRON_SCHEDULING.set_for_testing(False)
+
+    try:
+      assert_true(
+  """
+<coordinator-app name="MyCoord"
+  frequency="${coord:days(1)}"
+  start="2012-07-01T00:00Z" end="2012-07-04T00:00Z" timezone="America/Los_Angeles"
+  xmlns="uri:oozie:coordinator:0.2"
+  >
+  <controls>
+    <timeout>100</timeout>
+    <concurrency>3</concurrency>
+    <execution>FIFO</execution>
+    <throttle>10</throttle>
+  </controls>
+  <action>
+    <workflow>
+      <app-path>${wf_application_path}</app-path>
+      <configuration>
+        <property>
+          <name>username</name>
+          <value>${coord:user()}</value>
+        </property>
+        <property>
+          <name>SLEEP</name>
+          <value>1000</value>
+        </property>
+        <property>
+          <name>market</name>
+          <value>US</value>
+        </property>
+      </configuration>
+   </workflow>
+  </action>
+</coordinator-app>""" in coord.to_xml(), coord.to_xml())
+    finally:
+      finish()
+
     assert_true(
     assert_true(
 """<coordinator-app name="MyCoord"
 """<coordinator-app name="MyCoord"
   frequency="0 0 * * *"
   frequency="0 0 * * *"
@@ -3447,7 +3487,9 @@ COORDINATOR_DICT = {
     u'concurrency': [u'3'],
     u'concurrency': [u'3'],
     u'execution': [u'FIFO'],
     u'execution': [u'FIFO'],
     u'throttle': [u'10'],
     u'throttle': [u'10'],
-    u'schema_version': [u'uri:oozie:coordinator:0.2']
+    u'schema_version': [u'uri:oozie:coordinator:0.2'],
+    u'cron_frequency': [u'0 0 * * *'],
+    u'isAdvancedCron': [u'on'],
 }
 }
 BUNDLE_DICT = {
 BUNDLE_DICT = {
     u'name': [u'MyBundle'], u'description': [u'Description of my bundle'],
     u'name': [u'MyBundle'], u'description': [u'Description of my bundle'],

+ 12 - 3
apps/oozie/src/oozie/views/editor.py

@@ -43,6 +43,7 @@ from liboozie.submittion import Submission
 from filebrowser.lib.archives import archive_factory
 from filebrowser.lib.archives import archive_factory
 from oozie.decorators import check_job_access_permission, check_job_edition_permission,\
 from oozie.decorators import check_job_access_permission, check_job_edition_permission,\
                              check_dataset_access_permission, check_dataset_edition_permission
                              check_dataset_access_permission, check_dataset_edition_permission
+from oozie.conf import ENABLE_CRON_SCHEDULING
 from oozie.importlib.workflows import import_workflow as _import_workflow
 from oozie.importlib.workflows import import_workflow as _import_workflow
 from oozie.importlib.coordinators import import_coordinator as _import_coordinator
 from oozie.importlib.coordinators import import_coordinator as _import_coordinator
 from oozie.management.commands import oozie_setup
 from oozie.management.commands import oozie_setup
@@ -383,12 +384,15 @@ def create_coordinator(request, workflow=None):
   else:
   else:
     coordinator = Coordinator(owner=request.user, schema_version="uri:oozie:coordinator:0.2")
     coordinator = Coordinator(owner=request.user, schema_version="uri:oozie:coordinator:0.2")
 
 
+  enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
+
   if request.method == 'POST':
   if request.method == 'POST':
     coordinator_form = CoordinatorForm(request.POST, instance=coordinator, user=request.user)
     coordinator_form = CoordinatorForm(request.POST, instance=coordinator, user=request.user)
 
 
     if coordinator_form.is_valid():
     if coordinator_form.is_valid():
       coordinator = coordinator_form.save()
       coordinator = coordinator_form.save()
-      coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
+      if enable_cron_scheduling:
+        coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
       coordinator.save()
       coordinator.save()
       Document.objects.link(coordinator, owner=coordinator.owner, name=coordinator.name, description=coordinator.description)
       Document.objects.link(coordinator, owner=coordinator.owner, name=coordinator.name, description=coordinator.description)
       return redirect(reverse('oozie:edit_coordinator', kwargs={'coordinator': coordinator.id}) + "#step3")
       return redirect(reverse('oozie:edit_coordinator', kwargs={'coordinator': coordinator.id}) + "#step3")
@@ -401,6 +405,7 @@ def create_coordinator(request, workflow=None):
     'coordinator': coordinator,
     'coordinator': coordinator,
     'coordinator_form': coordinator_form,
     'coordinator_form': coordinator_form,
     'coordinator_frequency': json.dumps(coordinator.cron_frequency),
     'coordinator_frequency': json.dumps(coordinator.cron_frequency),
+    'enable_cron_scheduling': enable_cron_scheduling,
   })
   })
 
 
 
 
@@ -462,6 +467,8 @@ def edit_coordinator(request, coordinator):
   NewDataOutputFormSet = inlineformset_factory(Coordinator, DataOutput, form=DataOutputForm, extra=0, can_order=False, can_delete=False)
   NewDataOutputFormSet = inlineformset_factory(Coordinator, DataOutput, form=DataOutputForm, extra=0, can_order=False, can_delete=False)
   NewDataOutputFormSet.form = staticmethod(curry(DataOutputForm, coordinator=coordinator))
   NewDataOutputFormSet.form = staticmethod(curry(DataOutputForm, coordinator=coordinator))
 
 
+  enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
+
   if request.method == 'POST':
   if request.method == 'POST':
     coordinator_form = CoordinatorForm(request.POST, instance=coordinator, user=request.user)
     coordinator_form = CoordinatorForm(request.POST, instance=coordinator, user=request.user)
     dataset_formset = DatasetFormSet(request.POST, request.FILES, instance=coordinator)
     dataset_formset = DatasetFormSet(request.POST, request.FILES, instance=coordinator)
@@ -479,7 +486,8 @@ def edit_coordinator(request, coordinator):
       new_data_input_formset.save()
       new_data_input_formset.save()
       new_data_output_formset.save()
       new_data_output_formset.save()
       coordinator.sla = json.loads(request.POST.get('sla'))
       coordinator.sla = json.loads(request.POST.get('sla'))
-      coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
+      if enable_cron_scheduling:
+        coordinator.cron_frequency = {'frequency': request.POST.get('cron_frequency'), 'isAdvancedCron': request.POST.get('isAdvancedCron') == 'on'}
       coordinator.save()
       coordinator.save()
 
 
       request.info(_('Coordinator saved.'))
       request.info(_('Coordinator saved.'))
@@ -503,7 +511,8 @@ def edit_coordinator(request, coordinator):
     'new_data_input_formset': new_data_input_formset,
     'new_data_input_formset': new_data_input_formset,
     'new_data_output_formset': new_data_output_formset,
     'new_data_output_formset': new_data_output_formset,
     'history': history,
     'history': history,
-    'coordinator_frequency': json.dumps(coordinator.cron_frequency)
+    'coordinator_frequency': json.dumps(coordinator.cron_frequency),
+    'enable_cron_scheduling': enable_cron_scheduling,
   })
   })
 
 
 
 

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -501,6 +501,9 @@
   # Maximum of Oozie workflows or coodinators to retrieve in one API call.
   # Maximum of Oozie workflows or coodinators to retrieve in one API call.
   ## oozie_jobs_count=100
   ## oozie_jobs_count=100
 
 
+  # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
+  ## enable_cron_scheduling=true
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure Beeswax with Hive
 # Settings to configure Beeswax with Hive

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -542,6 +542,9 @@
   # Maximum of Oozie workflows or coodinators to retrieve in one API call.
   # Maximum of Oozie workflows or coodinators to retrieve in one API call.
   ## oozie_jobs_count=100
   ## oozie_jobs_count=100
 
 
+  # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
+  ## enable_cron_scheduling=true
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure Beeswax with Hive
 # Settings to configure Beeswax with Hive