Răsfoiți Sursa

HUE-1546 [oozie] Support same coordinators into a bundle

Romain Rigaux 12 ani în urmă
părinte
comite
a5694b6bbc

+ 16 - 7
apps/oozie/src/oozie/models.py

@@ -157,7 +157,7 @@ class Job(models.Model):
 
   def __str__(self):
     res = '%s - %s' % (force_unicode(self.name), self.owner)
-    return res.encode('utf-8', 'xmlcharrefreplace')
+    return force_unicode(res)
 
   def get_full_node(self):
     try:
@@ -489,7 +489,8 @@ class Workflow(Job):
     if mapping is None:
       mapping = {}
     tmpl = 'editor/gen/workflow.xml.mako'
-    return re.sub(re.compile('\s*\n+', re.MULTILINE), '\n', django_mako.render_to_string(tmpl, {'workflow': self, 'mapping': mapping})).encode('utf-8', 'xmlcharrefreplace')
+    xml = re.sub(re.compile('\s*\n+', re.MULTILINE), '\n', django_mako.render_to_string(tmpl, {'workflow': self, 'mapping': mapping}))
+    return force_unicode(xml)
 
 
 class Link(models.Model):
@@ -1367,6 +1368,9 @@ class Coordinator(Job):
   def find_parameters(self):
     params = self.workflow.find_parameters()
 
+    for param in find_parameters(self, ['job_properties']):
+      params[param] = ''
+
     for dataset in self.dataset_set.all():
       for param in find_parameters(dataset, ['uri']):
         if param not in set(DATASET_FREQUENCY):
@@ -1516,7 +1520,7 @@ class BundledCoordinator(models.Model):
   coordinator = models.ForeignKey(Coordinator, verbose_name=_t('Coordinator'),
                                   help_text=_t('The coordinator to batch with other coordinators.'))
 
-  parameters = models.TextField(default='[{"name":"oozie.use.system.libpath","value":"true"}]', verbose_name=_t('Oozie parameters'),
+  parameters = models.TextField(default='[{"name":"oozie.use.system.libpath","value":"true"}]', verbose_name=_t('Parameters'),
                                 help_text=_t('Constants used at the submission time (e.g. market=US, oozie.use.system.libpath=true).'))
 
   def get_parameters(self):
@@ -1542,7 +1546,12 @@ class Bundle(Job):
     if mapping is None:
       mapping = {}
     tmpl = "editor/gen/bundle.xml.mako"
-    return re.sub(re.compile('\s*\n+', re.MULTILINE), '\n', django_mako.render_to_string(tmpl, {'bundle': self, 'mapping': mapping})).encode('utf-8', 'xmlcharrefreplace')
+
+    return force_unicode(
+              re.sub(re.compile('\s*\n+', re.MULTILINE), '\n', django_mako.render_to_string(tmpl, {
+                'bundle': self,
+                'mapping': mapping
+           })))
 
   def clone(self, new_owner=None):
     bundleds = BundledCoordinator.objects.filter(bundle=self)
@@ -1578,12 +1587,12 @@ class Bundle(Job):
   def find_parameters(self):
     params = {}
 
-    for bundled in self.coordinators.all():
+    for bundled in BundledCoordinator.objects.filter(bundle=self):
       for param in bundled.coordinator.find_parameters():
         params[param] = ''
 
-      for param in find_parameters(bundled, ['parameters']):
-        params[param] = ''
+      for param in bundled.get_parameters():
+        params.pop(param['name'], None)
 
     return params
 

+ 11 - 8
apps/oozie/src/oozie/templates/dashboard/list_oozie_bundle.mako

@@ -62,9 +62,11 @@ ${ layout.menubar(section='dashboard') }
 
         % if bundle:
             <li class="nav-header">${ _('Coordinators') }</li>
-          % for bundled in bundle.coordinators.all():
+          % for bundled in bundle.coordinators.distinct():
             <li rel="tooltip" title="${ bundled.coordinator.name }" class="white">
-              <i class="icon-eye-open"></i> <span class="dataset">${ bundled.coordinator.name }</span>
+              <a href="${ bundled.coordinator.get_absolute_url() }">
+                <i class="icon-eye-open"></i> <span class="dataset">${ bundled.coordinator.name }</span>
+              </a>
             </li>
           % endfor
         % endif
@@ -430,13 +432,14 @@ ${ layout.menubar(section='dashboard') }
 
         $("#status span").attr("class", "label").addClass(getStatusClass(data.status)).text(data.status);
 
-        if (data.id && data.status != "RUNNING" && data.status != "SUSPENDED" && data.status != "KILLED" && data.status != "FAILED"){
-          $("#kill-btn").hide();
-          $("#rerun-btn").show();
-        }
-
-        if (data.id && data.status == "KILLED") {
+        if (data.id && (data.status == "KILLED" || data.status == "SUCCEEDED" ||  data.status == "DONEWITHERROR" || data.status == "FAILED")) {
           $("#kill-btn").hide();
+          if (data.status != "KILLED" ) {
+            $("#rerun-btn").show();
+          }
+        } else {
+          $("#kill-btn").show();
+          $("#rerun-btn").hide();
         }
 
         if (data.id && (data.status == "RUNNING" || data.status == "RUNNINGWITHERROR")){

+ 21 - 19
apps/oozie/src/oozie/templates/editor/gen/bundle.xml.mako

@@ -15,6 +15,9 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 
+<%!
+  from oozie.models import BundledCoordinator
+%>
 
 <bundle-app name="${ bundle.name }"
   xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
@@ -34,23 +37,22 @@
      <kick-off-time>${ bundle.kick_off_time_utc }</kick-off-time>
   </controls>
 
-  % if bundle.coordinators:
-    % for bundled in bundle.coordinators.all():
-    <coordinator name='${ bundled.coordinator.name }' >
-       <app-path>${'${'}nameNode}${ mapping.pop('coord_%s_dir' % bundled.coordinator.id) }</app-path>
-         <configuration>
-           <property>
-              <name>wf_application_path</name>
-              <value>${ mapping.pop('wf_%s_dir' % bundled.coordinator.workflow.id) }</value>
-          </property>
-           % for param in bundled.get_parameters():
-           <property>
-              <name>${ param['name'] }</name>
-              <value>${ param['value'] }</value>
-          </property>
-          % endfor
-        </configuration>
-    </coordinator>
-    % endfor
-  % endif
+
+  % for bundled in BundledCoordinator.objects.filter(bundle=bundle):
+  <coordinator name='${ bundled.coordinator.name }-${ bundled.id }' >
+     <app-path>${'${'}nameNode}${ mapping['coord_%s_dir' % bundled.coordinator.id] }</app-path>
+       <configuration>
+         <property>
+            <name>wf_application_path</name>
+            <value>${ mapping['wf_%s_dir' % bundled.coordinator.workflow.id] }</value>
+        </property>
+         % for param in bundled.get_parameters():
+         <property>
+            <name>${ param['name'] }</name>
+            <value>${ param['value'] }</value>
+        </property>
+        % endfor
+      </configuration>
+  </coordinator>
+  % endfor
 </bundle-app>

+ 13 - 14
apps/oozie/src/oozie/tests.py

@@ -1499,7 +1499,6 @@ class TestEditorBundle(OozieMockBase):
         u'create-bundled-coordinator-parameters': [u'[{"name":"market","value":"US"}]'],
         u'schema_version': [u'uri:oozie:bundle:0.2', u'uri:oozie:bundle:0.2'], u'coordinators-MAX_NUM_FORMS': [u'0'],
         u'coordinators-INITIAL_FORMS': [u'0'],
-        #u'create-bundled-coordinator-coordinator': [u'?'],
         u'parameters': [u'[{"name":"oozie.use.system.libpath","value":"true"}]'], u'coordinators-TOTAL_FORMS': [u'0'],
         u'description': [u'ss']
     }
@@ -1532,19 +1531,19 @@ class TestEditorBundle(OozieMockBase):
   <controls>
      <kick-off-time>2012-07-01T00:00Z</kick-off-time>
   </controls>
-    <coordinator name='MyCoord' >
-       <app-path>${nameNode}/deployment_path_coord</app-path>
-         <configuration>
-           <property>
-              <name>wf_application_path</name>
-              <value>/deployment_path_wf</value>
-          </property>
-           <property>
-              <name>market</name>
-              <value>US</value>
-          </property>
-        </configuration>
-    </coordinator>
+  <coordinator name='MyCoord-1' >
+     <app-path>${nameNode}/deployment_path_coord</app-path>
+       <configuration>
+         <property>
+            <name>wf_application_path</name>
+            <value>/deployment_path_wf</value>
+        </property>
+         <property>
+            <name>market</name>
+            <value>US</value>
+        </property>
+      </configuration>
+  </coordinator>
 </bundle-app>""" in xml, xml)
 
 

+ 3 - 7
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -163,21 +163,17 @@ class Submission(object):
     return deployment_dir
 
   def _update_properties(self, jobtracker_addr, deployment_dir):
-    properties = {
+    self.properties.update({
       'jobTracker': jobtracker_addr,
       'nameNode': self.fs.fs_defaultfs,
-    }
+    })
 
     if self.job:
-      properties.update({
+      self.properties.update({
         self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
         self.job.HUE_ID: self.job.id
       })
 
-    properties.update(self.properties)
-
-    self.properties = properties
-
   def _create_deployment_dir(self):
     """
     Return the job deployment directory in HDFS, creating it if necessary.

+ 1 - 1
desktop/libs/liboozie/src/liboozie/types.py

@@ -270,7 +270,7 @@ class BundleAction(Action):
 
 
 class Job(object):
-  RUNNING_STATUSES = set(['PREP', 'RUNNING', 'SUSPENDED', 'PREP', # Workflow
+  RUNNING_STATUSES = set(['PREP', 'RUNNING', 'SUSPENDED', # Workflow
                           'RUNNING', 'PREPSUSPENDED', 'SUSPENDED', 'PREPPAUSED', 'PAUSED' # Coordinator
                           ])
   """