瀏覽代碼

[oozie] Adding control fields to Coordinator

Adding timeout, throttle, execution and concurrency fields
Adding 'advanced' toggle in coordinator page
Adding tests
Romain Rigaux 13 年之前
父節點
當前提交
2b7936b

+ 30 - 1
apps/oozie/src/oozie/models.py

@@ -973,7 +973,28 @@ class Coordinator(Job):
                              help_text=_t('When we need to start the last workflow.'))
   workflow = models.ForeignKey(Workflow, null=True,
                                help_text=_t('The corresponding workflow we want to schedule repeatedly.'))
-
+  timeout_number = models.SmallIntegerField(default=1, choices=FREQUENCY_NUMBERS,
+                                            help_text=_t('Timeout for its coordinator actions, this is, how long the coordinator action will be in '
+                                                         'WAITING or READY status before giving up on its execution.'))
+  timeout_unit = models.CharField(max_length=20, choices=FREQUENCY_UNITS, default='days',
+                                    help_text=_t('It represents the unit of time of the timeous.'))
+
+
+  concurrency = models.PositiveSmallIntegerField(null=True, blank=True, choices=FREQUENCY_NUMBERS,
+                                 help_text=_t('Concurrency for its coordinator actions, this is, how many coordinator actions are '
+                                              'allowed to run concurrently ( RUNNING status) before the coordinator engine '
+                                              'starts throttling them.'))
+  execution = models.CharField(max_length=10, null=True, blank=True,
+                               choices=(('FIFO', 'FIFO (oldest first) default'),
+                                        ('LIFO', 'LIFO (newest first)'),
+                                        ('LAST_ONLY', 'LAST_ONLY (discards all older materializations)')),
+                                 help_text=_t('Execution strategy of its coordinator actions when there is backlog of coordinator '
+                                              'actions in the coordinator engine. The different execution strategies are \'oldest first\', '
+                                              '\'newest first\' and \'last one only\'. A backlog normally happens because of delayed '
+                                              'input data, concurrency control or because manual re-runs of coordinator jobs.'))
+  throttle = models.PositiveSmallIntegerField(null=True, blank=True, choices=FREQUENCY_NUMBERS,
+                                 help_text=_t('The materialization or creation throttle value for its coordinator actions, this is, '
+                                              'how many maximum coordinator actions are allowed to be in WAITING state concurrently.'))
   HUE_ID = 'hue-id-w'
 
   def get_type(self):
@@ -1050,6 +1071,14 @@ class Coordinator(Job):
   def text_frequency(self):
     return '%(number)d %(unit)s' % {'unit': self.frequency_unit, 'number': self.frequency_number}
 
+  @property
+  def timeout(self):
+    return '${coord:%(unit)s(%(number)d)}' % {'unit': self.timeout_unit, 'number': self.timeout_number}
+
+  @property
+  def text_timeout(self):
+    return '%(number)d %(unit)s' % {'unit': self.timeout_unit, 'number': self.timeout_number}
+
   def find_parameters(self):
     params = set()
 

+ 22 - 24
apps/oozie/src/oozie/templates/editor/create_workflow.mako

@@ -36,33 +36,31 @@ ${ layout.menubar(section='workflows') }
     <div style="min-height:300px">
       <form class="form-horizontal" id="workflowForm" action="${ url('oozie:create_workflow') }" method="POST">
 
-       <div class="row-fluid">
-          <div class="span2">
-          </div>
-          <div class="span8">
-              <h2>${ _('Properties') }</h2>
-              <br/>
-                <fieldset>
-                   ${ utils.render_field(workflow_form['name']) }
-                   ${ utils.render_field(workflow_form['description']) }
+      <div class="row-fluid">
+        <div class="span2">
+        </div>
+        <div class="span8">
+          <h2>${ _('Properties') }</h2>
+          <br/>
+          <fieldset>
+          ${ utils.render_field(workflow_form['name']) }
+          ${ utils.render_field(workflow_form['description']) }
 
-              <div class="control-group ">
-                <label class="control-label">
-                  <a href="#" id="advanced-btn" onclick="$('#advanced-container').toggle('hide')">
-                    <i class="icon-share-alt"></i> ${ _('advanced') }</a>
-                </label>
-                <div class="controls">
-                </div>
-              </div>
+          <div class="control-group ">
+            <label class="control-label">
+              <a href="#" id="advanced-btn" onclick="$('#advanced-container').toggle('hide')">
+                <i class="icon-share-alt"></i> ${ _('advanced') }
+              </a>
+            </label>
+            <div class="controls"></div>
+          </div>
 
-                   <div id="advanced-container" class="hide">
-                     ${ utils.render_field(workflow_form['deployment_dir']) }
-                   </a>
-               </fieldset>
+            <div id="advanced-container" class="hide">
+              ${ utils.render_field(workflow_form['deployment_dir']) }
            </div>
+         </fieldset>
 
-          <div class="span2">
-          </div>
+        <div class="span2"></div>
         </div>
       </div>
 
@@ -84,4 +82,4 @@ ${ layout.menubar(section='workflows') }
 
 ${ utils.path_chooser_libs(True) }
 
-${commonfooter(messages)}
+${ commonfooter(messages) }

+ 23 - 1
apps/oozie/src/oozie/templates/editor/edit_coordinator.mako

@@ -60,7 +60,29 @@ ${ layout.menubar(section='coordinators') }
                ${ utils.render_field(coordinator_form['description']) }
                ${ utils.render_field(coordinator_form['workflow']) }
                ${ utils.render_field(coordinator_form['is_shared']) }
-               ${ properties.print_key_value(_('Parameters'), 'parameters', coordinator_form, parameters) }
+
+               <div class="control-group ">
+                 <label class="control-label">
+                   <a href="#" id="advanced-btn" onclick="$('#advanced-container').toggle('hide')">
+                     <i class="icon-share-alt"></i> ${ _('advanced') }
+                   </a>
+                 </label>
+                 <div class="controls"></div>
+               </div>
+
+               <div id="advanced-container" class="hide">
+                 ${ properties.print_key_value(_('Parameters'), 'parameters', coordinator_form, parameters) }
+                 <div class="row-fluid">
+                   <div class="span6">
+                   ${ utils.render_field(coordinator_form['timeout_number']) }
+                 </div>
+                 <div class="span6">
+                   ${ utils.render_field(coordinator_form['timeout_unit']) }
+                 </div>
+                 ${ utils.render_field(coordinator_form['concurrency']) }
+                 ${ utils.render_field(coordinator_form['execution']) }
+                 ${ utils.render_field(coordinator_form['throttle']) }
+              </div>
              </div>
 
             <hr/>

+ 14 - 5
apps/oozie/src/oozie/templates/editor/gen/coordinator.xml.mako

@@ -19,13 +19,22 @@
   frequency="${ coord.frequency }"
   start="${ coord.start_utc }" end="${ coord.end_utc }" timezone="${ coord.timezone }"
   xmlns="uri:oozie:coordinator:0.1">
-  <!--
+  % if (coord.timeout_number and coord.timeout_unit) or coord.concurrency or coord.execution or coord.throttle:
   <controls>
-    <timeout>[TIME_PERIOD]</timeout>
-    <concurrency>[CONCURRENCY]</concurrency>
-    <execution>[EXECUTION_STRATEGY]</execution>
+    % if coord.timeout_number and coord.timeout_unit:
+    <timeout>${ coord.timeout }</timeout>
+    % endif
+    % if coord.concurrency:
+    <concurrency>${ coord.concurrency }</concurrency>
+    % endif
+    % if coord.execution:
+    <execution>${ coord.execution }</execution>
+    % endif
+    % if coord.throttle:
+    <throttle>${ coord.throttle }</throttle>
+    % endif
   </controls>
-  -->
+  % endif
 
   % if coord.dataset_set.exists():
   <datasets>

+ 10 - 6
apps/oozie/src/oozie/tests.py

@@ -791,13 +791,12 @@ class TestEditor:
         '  frequency="${coord:days(1)}"\n'
         '  start="2012-07-01T00:00Z" end="2012-07-04T00:00Z" timezone="America/Los_Angeles"\n'
         '  xmlns="uri:oozie:coordinator:0.1">\n'
-        '  <!--\n'
         '  <controls>\n'
-        '    <timeout>[TIME_PERIOD]</timeout>\n'
-        '    <concurrency>[CONCURRENCY]</concurrency>\n'
-        '    <execution>[EXECUTION_STRATEGY]</execution>\n'
+        '    <timeout>${coord:hours(2)}</timeout>\n'
+        '    <concurrency>3</concurrency>\n'
+        '    <execution>FIFO</execution>\n'
+        '    <throttle>10</throttle>\n'
         '  </controls>\n'
-        '  -->\n'
         '  <action>\n'
         '    <workflow>\n'
         '      <app-path>${wf_application_path}</app-path>\n'
@@ -875,7 +874,12 @@ def create_coordinator(workflow):
                         u'start_0': [u'07/01/2012'], u'start_1': [u'12:00 AM'],
                         u'end_0': [u'07/04/2012'], u'end_1': [u'12:00 AM'],
                         u'timezone': [u'America/Los_Angeles'],
-                        u'parameters': [u'[{"name":"market","value":"US,France"}]']})
+                        u'parameters': [u'[{"name":"market","value":"US,France"}]'],
+                        u'timeout_number': [u'2'], u'timeout_unit': [u'hours'],
+                        u'concurrency': [u'3'],
+                        u'execution': [u'FIFO'],
+                        u'throttle': [u'10']
+  })
   assert_equal(coord_count + 1, Coordinator.objects.count(), response)
 
   return Coordinator.objects.get()