Browse Source

HUE-8927 [oozie] Fix Oozie bundle cannot be submitted

Ying Chen 6 years ago
parent
commit
bdd3d3d69d

+ 3 - 2
apps/oozie/src/oozie/models2.py

@@ -3631,7 +3631,7 @@ class Bundle(Job):
               'description': '',
               'description': '',
               'deployment_dir': '',
               'deployment_dir': '',
               'schema_version': 'uri:oozie:bundle:0.2',
               'schema_version': 'uri:oozie:bundle:0.2',
-              'kickoff': datetime.today(),
+              'kickoff': datetime.utcnow(),
               'parameters': [{'name': 'oozie.use.system.libpath', 'value': 'true'}]
               'parameters': [{'name': 'oozie.use.system.libpath', 'value': 'true'}]
           }
           }
       }
       }
@@ -3647,7 +3647,8 @@ class Bundle(Job):
   def get_data_for_json(self):
   def get_data_for_json(self):
     _data = self.data.copy()
     _data = self.data.copy()
 
 
-    _data['properties']['kickoff'] = _data['properties']['kickoff'].strftime('%Y-%m-%dT%H:%M:%S')
+    if _data['properties']['kickoff'] and type(_data['properties']['kickoff']) is datetime:
+      _data['properties']['kickoff'] = _data['properties']['kickoff'].strftime('%Y-%m-%dT%H:%M:%S')
 
 
     return _data
     return _data
 
 

+ 1 - 1
apps/oozie/src/oozie/templates/editor2/gen/bundle.xml.mako

@@ -35,7 +35,7 @@
   % endif
   % endif
 
 
   <controls>
   <controls>
-     <kick-off-time>${ bundle.kick_off_time_utc }</kick-off-time>
+     <kick-off-time>${ bundle.data['properties']['kickoff'] }</kick-off-time>
   </controls>
   </controls>
 
 
 
 

+ 27 - 5
apps/oozie/src/oozie/tests.py

@@ -51,13 +51,14 @@ from jobsub.models import OozieDesign, OozieMapreduceAction
 from liboozie import oozie_api
 from liboozie import oozie_api
 from liboozie.conf import OOZIE_URL
 from liboozie.conf import OOZIE_URL
 from liboozie.oozie_api_tests import OozieServerProvider
 from liboozie.oozie_api_tests import OozieServerProvider
+from liboozie.submission2 import Submission
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
 
 
 from oozie.conf import ENABLE_CRON_SCHEDULING, ENABLE_V2
 from oozie.conf import ENABLE_CRON_SCHEDULING, ENABLE_V2
 from oozie.models import Dataset, Workflow, Node, Kill, Link, Job, Coordinator, History,\
 from oozie.models import Dataset, Workflow, Node, Kill, Link, Job, Coordinator, History,\
   find_parameters, NODE_TYPES, Bundle
   find_parameters, NODE_TYPES, Bundle
-from oozie.models2 import _get_hiveserver2_url
+from oozie.models2 import _get_hiveserver2_url, Bundle as Bundle2
 from oozie.utils import workflow_to_dict, model_to_dict, smart_path, contains_symlink, convert_to_server_timezone
 from oozie.utils import workflow_to_dict, model_to_dict, smart_path, contains_symlink, convert_to_server_timezone
 from oozie.importlib.workflows import import_workflow
 from oozie.importlib.workflows import import_workflow
 from oozie.importlib.jobdesigner import convert_jobsub_design
 from oozie.importlib.jobdesigner import convert_jobsub_design
@@ -240,7 +241,8 @@ class MockOozieApi(object):
            hive2=org.apache.oozie.action.hadoop.Hive2Credentials
            hive2=org.apache.oozie.action.hadoop.Hive2Credentials
     """
     """
     return {
     return {
-        'oozie.credentials.credentialclasses': oozie_credentialclasses
+        'oozie.credentials.credentialclasses': oozie_credentialclasses,
+        'oozie.processing.timezone': 'GMT-0800'
     }
     }
 
 
   def get_job_status(self, job_id):
   def get_job_status(self, job_id):
@@ -2115,7 +2117,6 @@ class TestEditorBundle(OozieMockBase):
 
 
 
 
   def test_bundle_gen_xml(self):
   def test_bundle_gen_xml(self):
-    raise SkipTest()
     bundle = create_bundle(self.c, self.user)
     bundle = create_bundle(self.c, self.user)
 
 
     assert_true(
     assert_true(
@@ -2129,10 +2130,31 @@ class TestEditorBundle(OozieMockBase):
     </property>
     </property>
   </parameters>
   </parameters>
   <controls>
   <controls>
-     <kick-off-time>2012-07-01T00:00Z</kick-off-time>
+     <kick-off-time>%s</kick-off-time>
+  </controls>
+</bundle-app>
+""" % bundle.kick_off_time_utc in bundle.to_xml(), bundle.to_xml())
+
+  def test_model2_bundle_gen_xml(self):
+    bundle = Bundle2()
+    converted_kickoff_time = convert_to_server_timezone(bundle.kick_off_time_utc)
+    Submission(self.user, bundle)
+
+    assert_true(
+"""<bundle-app name="My Bundle"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns="uri:oozie:bundle:0.2">
+  <parameters>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+  </parameters>
+  <controls>
+     <kick-off-time>%s</kick-off-time>
   </controls>
   </controls>
 </bundle-app>
 </bundle-app>
-""" in bundle.to_xml(), bundle.to_xml())
+""" % converted_kickoff_time in bundle.to_xml(), bundle.to_xml())
 
 
 
 
   def test_create_bundled_coordinator(self):
   def test_create_bundled_coordinator(self):

+ 3 - 1
apps/oozie/src/oozie/utils.py

@@ -137,7 +137,9 @@ def contains_symlink(path, mapping):
   return any([var in mapping and '#' in mapping[var] for var in vars]) or '#' in path
   return any([var in mapping and '#' in mapping[var] for var in vars]) or '#' in path
 
 
 def utc_datetime_format(utc_time):
 def utc_datetime_format(utc_time):
-  return utc_time.strftime(UTC_TIME_FORMAT)
+  if utc_time and type(utc_time) is datetime:
+    return utc_time.strftime(UTC_TIME_FORMAT)
+  return utc_time
 
 
 
 
 def oozie_to_django_datetime(dt_string):
 def oozie_to_django_datetime(dt_string):

+ 2 - 0
apps/oozie/src/oozie/views/editor2.py

@@ -19,6 +19,7 @@ from builtins import str
 import json
 import json
 import logging
 import logging
 
 
+from datetime import datetime
 from django.urls import reverse
 from django.urls import reverse
 from django.forms.formsets import formset_factory
 from django.forms.formsets import formset_factory
 from django.shortcuts import redirect
 from django.shortcuts import redirect
@@ -916,6 +917,7 @@ def copy_bundle(request):
 @check_document_access_permission()
 @check_document_access_permission()
 def submit_bundle(request, doc_id):
 def submit_bundle(request, doc_id):
   bundle = Bundle(document=Document2.objects.get(id=doc_id))
   bundle = Bundle(document=Document2.objects.get(id=doc_id))
+  bundle._data['properties']['kickoff'] = datetime.utcnow()
   ParametersFormSet = formset_factory(ParameterForm, extra=0)
   ParametersFormSet = formset_factory(ParameterForm, extra=0)
 
 
   if request.method == 'POST':
   if request.method == 'POST':

+ 3 - 1
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -89,12 +89,14 @@ class Submission(object):
       local_tz = self.job.data.get('properties')['timezone']
       local_tz = self.job.data.get('properties')['timezone']
 
 
     # Modify start_date & end_date only when it's a coordinator
     # Modify start_date & end_date only when it's a coordinator
-    from oozie.models2 import Coordinator
+    from oozie.models2 import Coordinator, Bundle
     if type(self.job) is Coordinator:
     if type(self.job) is Coordinator:
       if 'start_date' in self.properties:
       if 'start_date' in self.properties:
         properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz)
         properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz)
       if 'end_date' in self.properties:
       if 'end_date' in self.properties:
         properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz)
         properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz)
+    elif type(self.job) is Bundle:
+      self.job.data['properties']['kickoff'] = convert_to_server_timezone(self.job.kick_off_time_utc, local_tz)
 
 
     if 'nominal_time' in self.properties:
     if 'nominal_time' in self.properties:
       properties['nominal_time'] = convert_to_server_timezone(self.properties['nominal_time'], local_tz)
       properties['nominal_time'] = convert_to_server_timezone(self.properties['nominal_time'], local_tz)