Browse Source

[oozie] Adding control fields to Coordinator

Adding timeout, throttle, execution and concurrency fields
Adding 'advanced' toggle in coordinator page
Adding tests
Romain Rigaux 13 năm trước cách đây
mục cha
commit
2b7936b65f

+ 30 - 1
apps/oozie/src/oozie/models.py

@@ -973,7 +973,28 @@ class Coordinator(Job):
                              help_text=_t('When we need to start the last workflow.'))
   workflow = models.ForeignKey(Workflow, null=True,
                                help_text=_t('The corresponding workflow we want to schedule repeatedly.'))
-
+  timeout_number = models.SmallIntegerField(default=1, choices=FREQUENCY_NUMBERS,
+                                            help_text=_t('Timeout for its coordinator actions, this is, how long the coordinator action will be in '
+                                                         'WAITING or READY status before giving up on its execution.'))
+  timeout_unit = models.CharField(max_length=20, choices=FREQUENCY_UNITS, default='days',
+                                    help_text=_t('It represents the unit of time of the timeous.'))
+
+
+  concurrency = models.PositiveSmallIntegerField(null=True, blank=True, choices=FREQUENCY_NUMBERS,
+                                 help_text=_t('Concurrency for its coordinator actions, this is, how many coordinator actions are '
+                                              'allowed to run concurrently ( RUNNING status) before the coordinator engine '
+                                              'starts throttling them.'))
+  execution = models.CharField(max_length=10, null=True, blank=True,
+                               choices=(('FIFO', 'FIFO (oldest first) default'),
+                                        ('LIFO', 'LIFO (newest first)'),
+                                        ('LAST_ONLY', 'LAST_ONLY (discards all older materializations)')),
+                                 help_text=_t('Execution strategy of its coordinator actions when there is backlog of coordinator '
+                                              'actions in the coordinator engine. The different execution strategies are \'oldest first\', '
+                                              '\'newest first\' and \'last one only\'. A backlog normally happens because of delayed '
+                                              'input data, concurrency control or because manual re-runs of coordinator jobs.'))
+  throttle = models.PositiveSmallIntegerField(null=True, blank=True, choices=FREQUENCY_NUMBERS,
+                                 help_text=_t('The materialization or creation throttle value for its coordinator actions, this is, '
+                                              'how many maximum coordinator actions are allowed to be in WAITING state concurrently.'))
   HUE_ID = 'hue-id-w'
 
   def get_type(self):
@@ -1050,6 +1071,14 @@ class Coordinator(Job):
   def text_frequency(self):
     return '%(number)d %(unit)s' % {'unit': self.frequency_unit, 'number': self.frequency_number}
 
+  @property
+  def timeout(self):
+    return '${coord:%(unit)s(%(number)d)}' % {'unit': self.timeout_unit, 'number': self.timeout_number}
+
+  @property
+  def text_timeout(self):
+    return '%(number)d %(unit)s' % {'unit': self.timeout_unit, 'number': self.timeout_number}
+
   def find_parameters(self):
     params = set()
 

+ 22 - 24
apps/oozie/src/oozie/templates/editor/create_workflow.mako

@@ -36,33 +36,31 @@ ${ layout.menubar(section='workflows') }
     <div style="min-height:300px">
       <form class="form-horizontal" id="workflowForm" action="${ url('oozie:create_workflow') }" method="POST">
 
-       <div class="row-fluid">
-          <div class="span2">
-          </div>
-          <div class="span8">
-              <h2>${ _('Properties') }</h2>
-              <br/>
-                <fieldset>
-                   ${ utils.render_field(workflow_form['name']) }
-                   ${ utils.render_field(workflow_form['description']) }
+      <div class="row-fluid">
+        <div class="span2">
+        </div>
+        <div class="span8">
+          <h2>${ _('Properties') }</h2>
+          <br/>
+          <fieldset>
+          ${ utils.render_field(workflow_form['name']) }
+          ${ utils.render_field(workflow_form['description']) }
 
-              <div class="control-group ">
-                <label class="control-label">
-                  <a href="#" id="advanced-btn" onclick="$('#advanced-container').toggle('hide')">
-                    <i class="icon-share-alt"></i> ${ _('advanced') }</a>
-                </label>
-                <div class="controls">
-                </div>
-              </div>
+          <div class="control-group ">
+            <label class="control-label">
+              <a href="#" id="advanced-btn" onclick="$('#advanced-container').toggle('hide')">
+                <i class="icon-share-alt"></i> ${ _('advanced') }
+              </a>
+            </label>
+            <div class="controls"></div>
+          </div>
 
-                   <div id="advanced-container" class="hide">
-                     ${ utils.render_field(workflow_form['deployment_dir']) }
-                   </a>
-               </fieldset>
+            <div id="advanced-container" class="hide">
+              ${ utils.render_field(workflow_form['deployment_dir']) }
            </div>
+         </fieldset>
 
-          <div class="span2">
-          </div>
+        <div class="span2"></div>
         </div>
       </div>
 
@@ -84,4 +82,4 @@ ${ layout.menubar(section='workflows') }
 
 ${ utils.path_chooser_libs(True) }
 
-${commonfooter(messages)}
+${ commonfooter(messages) }

+ 23 - 1
apps/oozie/src/oozie/templates/editor/edit_coordinator.mako

@@ -60,7 +60,29 @@ ${ layout.menubar(section='coordinators') }
                ${ utils.render_field(coordinator_form['description']) }
                ${ utils.render_field(coordinator_form['workflow']) }
                ${ utils.render_field(coordinator_form['is_shared']) }
-               ${ properties.print_key_value(_('Parameters'), 'parameters', coordinator_form, parameters) }
+
+               <div class="control-group ">
+                 <label class="control-label">
+                   <a href="#" id="advanced-btn" onclick="$('#advanced-container').toggle('hide')">
+                     <i class="icon-share-alt"></i> ${ _('advanced') }
+                   </a>
+                 </label>
+                 <div class="controls"></div>
+               </div>
+
+               <div id="advanced-container" class="hide">
+                 ${ properties.print_key_value(_('Parameters'), 'parameters', coordinator_form, parameters) }
+                 <div class="row-fluid">
+                   <div class="span6">
+                   ${ utils.render_field(coordinator_form['timeout_number']) }
+                 </div>
+                 <div class="span6">
+                   ${ utils.render_field(coordinator_form['timeout_unit']) }
+                 </div>
+                 ${ utils.render_field(coordinator_form['concurrency']) }
+                 ${ utils.render_field(coordinator_form['execution']) }
+                 ${ utils.render_field(coordinator_form['throttle']) }
+              </div>
              </div>
 
             <hr/>

+ 14 - 5
apps/oozie/src/oozie/templates/editor/gen/coordinator.xml.mako

@@ -19,13 +19,22 @@
   frequency="${ coord.frequency }"
   start="${ coord.start_utc }" end="${ coord.end_utc }" timezone="${ coord.timezone }"
   xmlns="uri:oozie:coordinator:0.1">
-  <!--
+  % if (coord.timeout_number and coord.timeout_unit) or coord.concurrency or coord.execution or coord.throttle:
   <controls>
-    <timeout>[TIME_PERIOD]</timeout>
-    <concurrency>[CONCURRENCY]</concurrency>
-    <execution>[EXECUTION_STRATEGY]</execution>
+    % if coord.timeout_number and coord.timeout_unit:
+    <timeout>${ coord.timeout }</timeout>
+    % endif
+    % if coord.concurrency:
+    <concurrency>${ coord.concurrency }</concurrency>
+    % endif
+    % if coord.execution:
+    <execution>${ coord.execution }</execution>
+    % endif
+    % if coord.throttle:
+    <throttle>${ coord.throttle }</throttle>
+    % endif
   </controls>
-  -->
+  % endif
 
   % if coord.dataset_set.exists():
   <datasets>

+ 10 - 6
apps/oozie/src/oozie/tests.py

@@ -791,13 +791,12 @@ class TestEditor:
         '  frequency="${coord:days(1)}"\n'
         '  start="2012-07-01T00:00Z" end="2012-07-04T00:00Z" timezone="America/Los_Angeles"\n'
         '  xmlns="uri:oozie:coordinator:0.1">\n'
-        '  <!--\n'
         '  <controls>\n'
-        '    <timeout>[TIME_PERIOD]</timeout>\n'
-        '    <concurrency>[CONCURRENCY]</concurrency>\n'
-        '    <execution>[EXECUTION_STRATEGY]</execution>\n'
+        '    <timeout>${coord:hours(2)}</timeout>\n'
+        '    <concurrency>3</concurrency>\n'
+        '    <execution>FIFO</execution>\n'
+        '    <throttle>10</throttle>\n'
         '  </controls>\n'
-        '  -->\n'
         '  <action>\n'
         '    <workflow>\n'
         '      <app-path>${wf_application_path}</app-path>\n'
@@ -875,7 +874,12 @@ def create_coordinator(workflow):
                         u'start_0': [u'07/01/2012'], u'start_1': [u'12:00 AM'],
                         u'end_0': [u'07/04/2012'], u'end_1': [u'12:00 AM'],
                         u'timezone': [u'America/Los_Angeles'],
-                        u'parameters': [u'[{"name":"market","value":"US,France"}]']})
+                        u'parameters': [u'[{"name":"market","value":"US,France"}]'],
+                        u'timeout_number': [u'2'], u'timeout_unit': [u'hours'],
+                        u'concurrency': [u'3'],
+                        u'execution': [u'FIFO'],
+                        u'throttle': [u'10']
+  })
   assert_equal(coord_count + 1, Coordinator.objects.count(), response)
 
   return Coordinator.objects.get()