Przeglądaj źródła

HUE-1910 [oozie] Support oozie.processing.timezone

krish 10 lat temu
rodzic
commit
db3f15d

+ 12 - 14
apps/oozie/src/oozie/models2.py

@@ -29,6 +29,7 @@ from django.core.urlresolvers import reverse
 from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 
+from desktop.conf import TIME_ZONE
 from desktop.lib import django_mako
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
@@ -40,7 +41,7 @@ from liboozie.submission2 import Submission
 from liboozie.submission2 import create_directories
 
 from oozie.conf import REMOTE_SAMPLE_DIR
-from oozie.utils import utc_datetime_format, UTC_TIME_FORMAT
+from oozie.utils import utc_datetime_format, UTC_TIME_FORMAT, convert_to_server_timezone
 from hadoop.fs.exceptions import WebHdfsException
 
 
@@ -1820,7 +1821,7 @@ class Coordinator(Job):
               'frequency_unit': 'days',
               'cron_frequency': '0 0 * * *',
               'cron_advanced': False,
-              'timezone': 'America/Los_Angeles',
+              'timezone': TIME_ZONE.get(),
               'start': '${start_date}',
               'end': '${end_date}',
               'workflow': None,
@@ -1832,8 +1833,8 @@ class Coordinator(Job):
               'credentials': [],
               'parameters': [
                   {'name': 'oozie.use.system.libpath', 'value': True},
-                  {'name': 'start_date', 'value':  datetime.today().strftime('%Y-%m-%dT%H:%MZ')},
-                  {'name': 'end_date', 'value': (datetime.today() + timedelta(days=7)).strftime('%Y-%m-%dT%H:%MZ')}
+                  {'name': 'start_date', 'value':  datetime.today().strftime('%Y-%m-%dT%H:%M')},
+                  {'name': 'end_date', 'value': (datetime.today() + timedelta(days=7)).strftime('%Y-%m-%dT%H:%M')}
               ],
               'sla': Workflow.SLA_DEFAULT
           }
@@ -1852,11 +1853,11 @@ class Coordinator(Job):
 
     start_date = filter(lambda a: a['name'] == 'start_date', self._data['properties']['parameters'])
     if start_date and type(start_date[0]['value']) == datetime:
-      start_date[0]['value'] = start_date[0]['value'].strftime('%Y-%m-%dT%H:%M:%SZ')
+      start_date[0]['value'] = start_date[0]['value'].strftime('%Y-%m-%dT%H:%M:%S')
 
     end_date = filter(lambda a: a['name'] == 'end_date', self._data['properties']['parameters'])
     if end_date and type(end_date[0]['value']) == datetime:
-      end_date[0]['value'] = end_date[0]['value'].strftime('%Y-%m-%dT%H:%M:%SZ')
+      end_date[0]['value'] = end_date[0]['value'].strftime('%Y-%m-%dT%H:%M:%S')
 
     return _data
 
@@ -1942,11 +1943,11 @@ class Coordinator(Job):
     return [Dataset(dataset, self) for dataset in self.data['variables'] if dataset['dataset_type'] == 'output_path']
 
   @property
-  def start_utc(self):
+  def start_server_tz(self):
     return self.data['properties']['start']
 
   @property
-  def end_utc(self):
+  def end_server_tz(self):
     return self.data['properties']['end']
 
   @property
@@ -2011,9 +2012,6 @@ class Dataset():
 
   @property
   def data(self):
-    if type(self._data['start']) == unicode:
-      self._data['start'] = parse(self._data['start'])
-
     self._data['name'] = self._data['workflow_variable']
 
     return self._data
@@ -2040,11 +2038,11 @@ class Dataset():
     return '${coord:%(unit)s(%(number)s)}' % {'unit': frequency_unit, 'number': frequency_number}
 
   @property
-  def start_utc(self):
+  def start_server_tz(self):
     if self.data['same_start']:
-      return self.coordinator.start_utc
+      return self.coordinator.start_server_tz
     else:
-      return utc_datetime_format(self.data['start'])
+      return convert_to_server_timezone(self.data['start'], self.data['timezone'])
 
   @property
   def timezone(self):

+ 2 - 2
apps/oozie/src/oozie/static/oozie/js/coordinator-editor.ko.js

@@ -114,7 +114,7 @@ var Coordinator = function (vm, coordinator) {
       'show_advanced': false,
       'use_done_flag': false,
       'done_flag': '_SUCCESS',
-      'timezone': 'America/Los_Angeles',
+      'timezone': coordinator.properties.timezone,
       'same_timezone': true,
       'instance_choice': 'default',
       'is_advanced_start_instance': false,
@@ -126,7 +126,7 @@ var Coordinator = function (vm, coordinator) {
       'same_frequency': true,
       'frequency_number': 1,
       'frequency_unit': 'days',
-      'start': new Date(),
+      'start': moment().format("YYYY-MM-DD[T]HH:mm"),
       'same_start': true,
 
       'shared_dataset_uuid': '' // If reusing a shared dataset

+ 1 - 1
apps/oozie/src/oozie/templates/dashboard/list_oozie_coordinator.mako

@@ -808,7 +808,7 @@ ${ layout.menubar(section='coordinators', dashboard=True) }
     $(".confirmationModal").bind('confirmation', function () {
       var _this = this;
       var IN_DATETIME_FORMAT = "MM/DD/YYYY hh:mm A";
-      var OUT_DATETIME_FORMAT = "YYYY-MM-DD[T]HH:mm[Z]";
+      var OUT_DATETIME_FORMAT = "YYYY-MM-DD[T]HH:mm";
 
       var params = { 'notification': $(_this).attr("data-message") };
       if ($(this).attr("id") == "edit-coord-btn") {

+ 8 - 10
apps/oozie/src/oozie/templates/editor2/coordinator_editor.mako

@@ -121,9 +121,6 @@ ${ layout.menubar(section='coordinators', is_editor=True, pullright=buttons) }
       </div>
 
       <div class="card card-home" data-bind="visible: coordinator.properties.workflow" style="margin-top: 20px">
-        <div class="alert alert-warning pull-right" style="border: none">
-          ${ _('UTC time only. (e.g. if you want 10pm PST (UTC+8) set it 8 hours later to 6am the next day.') }
-        </div>
         <h1 class="card-heading simple">${ _('How often?') }
         </h1>
 
@@ -172,6 +169,14 @@ ${ layout.menubar(section='coordinators', is_editor=True, pullright=buttons) }
                 </div>
               </div>
 
+              <div class="control-group" style="margin-bottom: 20">
+                <label class="control-label">${ _('Timezone') }</label>
+                <div class="controls">
+                  <select data-bind="options: $root.availableTimezones, select2: { placeholder: '${ _ko("Select a Timezone") }', update: coordinator.properties.timezone}" style="width: 180px"></select>
+                  <span class="help-inline"></span>
+                </div>
+              </div>
+
               <div class="control-group">
                 <label class="control-label">${ _('From') }</label>
                 <div class="controls">
@@ -208,13 +213,6 @@ ${ layout.menubar(section='coordinators', is_editor=True, pullright=buttons) }
                   <span class="help-inline"></span>
                 </div>
               </div>
-              <div class="control-group" style="margin-bottom: 0">
-                <label class="control-label">${ _('Timezone') }</label>
-                <div class="controls">
-                  <select data-bind="options: $root.availableTimezones, select2: { placeholder: '${ _ko("Select a Timezone") }', update: coordinator.properties.timezone}" style="width: 180px"></select>
-                  <span class="help-inline"></span>
-                </div>
-              </div>
 
             </form>
 

+ 9 - 4
apps/oozie/src/oozie/templates/editor2/gen/coordinator.xml.mako

@@ -57,7 +57,7 @@
   % else:
   frequency="${ coord.frequency }"
   % endif
-  start="${ coord.start_utc }" end="${ coord.end_utc }" timezone="${ coord.data['properties']['timezone'] }"
+  start="${ coord.start_server_tz }" end="${ coord.end_server_tz }" timezone="${ coord.data['properties']['timezone'] }"
   xmlns="${ 'uri:oozie:coordinator:0.4' if coord.sla_enabled else coord.data['properties']['schema_version'] | n,unicode }"
   ${ 'xmlns:sla="uri:oozie:sla:0.2"' if coord.sla_enabled else '' | n,unicode }>
   % if coord.data['properties']['timeout'] or coord.data['properties'].get('concurrency') or coord.data['properties']['execution'] or coord.data['properties'].get('throttle'):
@@ -81,7 +81,7 @@
   <datasets>
     % for dataset in coord.datasets:
     <dataset name="${ dataset.data['name'] }" frequency="${ dataset.frequency }"
-             initial-instance="${ dataset.start_utc }" timezone="${ dataset.timezone }">
+             initial-instance="${ dataset.start_server_tz }" timezone="${ dataset.timezone }">
       <uri-template>${ smart_path(dataset.data['dataset_variable'], mapping, is_coordinator=True) }</uri-template>
       % if dataset.data['use_done_flag']:
       <done-flag>${ dataset.data['done_flag'] }</done-flag>
@@ -132,8 +132,13 @@
         % endfor
         % for property in coord.properties:
         <property>
-          <name>${ property['name'] }</name>
-          <value>${ property['value'] }</value>
+          % if property['name'] in ['start_date', 'end_date']:
+            <name>${ property['name'] }</name>
+            <value>${'${' + property['name']}}</value>
+          % else:
+            <name>${ property['name'] }</name>
+            <value>${ property['value'] }</value>
+          % endif
         </property>
         % endfor
       </configuration>

+ 1 - 1
apps/oozie/src/oozie/templates/editor2/submit_job_popup.mako

@@ -129,7 +129,7 @@
   });
 
   $(".now-link").on("click", function(){
-    $(this).parents(".controls").find("input[type='text']").val(moment().format("YYYY-MM-DD[T]HH:mm") + 'Z');
+    $(this).parents(".controls").find("input[type='text']").val(moment().format("YYYY-MM-DD[T]HH:mm"));
   });
 
   $(".calendar-link").on("click", function(){

+ 14 - 1
apps/oozie/src/oozie/tests.py

@@ -50,7 +50,7 @@ from oozie.conf import ENABLE_CRON_SCHEDULING, ENABLE_V2
 from oozie.models import Dataset, Workflow, Node, Kill, Link, Job, Coordinator, History,\
   find_parameters, NODE_TYPES, Bundle
 from oozie.models2 import _get_hiveserver2_url
-from oozie.utils import workflow_to_dict, model_to_dict, smart_path, contains_symlink
+from oozie.utils import workflow_to_dict, model_to_dict, smart_path, contains_symlink, convert_to_server_timezone
 from oozie.importlib.workflows import import_workflow
 from oozie.importlib.jobdesigner import convert_jobsub_design
 
@@ -3778,6 +3778,19 @@ class TestUtils(OozieMockBase):
     assert_false(contains_symlink('${output}', {'output': '${path}'}))
     assert_true(contains_symlink('${output_dir}', {'output': '/path/out', 'output_dir': 'hdfs://nn/path/out#out'}))
 
+  def test_convert_to_server_timezone(self):
+    # To UTC
+    assert_equal(convert_to_server_timezone('2015-07-01T10:10', local_tz='America/Los_Angeles', server_tz='UTC', user='test'), u'2015-07-01T17:10Z')
+    assert_equal(convert_to_server_timezone('2015-07-01T10:10', local_tz='Europe/Paris', server_tz='UTC', user='test'), u'2015-07-01T08:10Z')
+    # To GMT(+/-)####
+    assert_equal(convert_to_server_timezone('2015-07-01T10:10', local_tz='Asia/Jayapura', server_tz='GMT+0800', user='test'), u'2015-07-01T09:10+0800')
+    assert_equal(convert_to_server_timezone('2015-07-01T10:10', local_tz='Australia/LHI', server_tz='GMT-0530', user='test'), u'2015-06-30T18:10+0530')
+    # Previously created coordinators might have 'Z' appended, we consider them as UTC local time
+    assert_equal(convert_to_server_timezone('2015-07-01T10:10Z', local_tz='America/Los_Angeles', server_tz='UTC', user='test'), u'2015-07-01T10:10Z')
+    assert_equal(convert_to_server_timezone('2015-07-01T10:10Z', local_tz='Asia/Jayapura', server_tz='GMT+0800', user='test'), u'2015-07-01T18:10+0800')
+    assert_equal(convert_to_server_timezone('2015-07-01T10:10Z', local_tz='Australia/LHI', server_tz='GMT-0530', user='test'), u'2015-07-01T04:40+0530')
+
+
 # Utils
 WORKFLOW_DICT = {
     u'deployment_dir': [u''], u'name': [u'wf-name-1'], u'description': [u''],

+ 32 - 0
apps/oozie/src/oozie/utils.py

@@ -20,10 +20,13 @@ import logging
 import re
 import urlparse
 from datetime import datetime
+from dateutil import tz
+from dateutil import parser
 
 from django.utils.formats import localize_input
 from django.utils.translation import ugettext as _
 from desktop.lib.parameterization import find_variables
+from liboozie.oozie_api import get_oozie, DEFAULT_USER
 
 
 LOG = logging.getLogger(__name__)
@@ -163,3 +166,32 @@ def oozie_to_hue_frequency(frequency_string):
     return matches.group('frequency_unit'), matches.group('frequency_number')
   else:
     raise InvalidFrequency(_('invalid frequency: %s') % frequency_string)
+
+def convert_to_server_timezone(date, local_tz='UTC', server_tz='UTC', user=DEFAULT_USER):
+  api = get_oozie(user)
+  oozie_conf = api.get_configuration()
+  if server_tz is None:
+    server_tz = oozie_conf['oozie.processing.timezone']
+
+  if date and date.startswith('$'):
+    return date
+
+  # To support previously created jobs
+  if date.endswith('Z'):
+    date = date[:-1]
+    local_tz = 'UTC'
+
+  try:
+    date_local_tz = parser.parse(date)
+    date_local_tz = date_local_tz.replace(tzinfo=tz.gettz(local_tz))
+    date_server_tz = date_local_tz.astimezone(tz.gettz(server_tz))
+
+    date_server_tz = date_server_tz.strftime('%Y-%m-%dT%H:%M')
+    # Oozie timezone is either UTC or GMT(+/-)####
+    if 'UTC' == server_tz:
+      return date_server_tz + u'Z'
+    else:
+      return date_server_tz + u'+' + re.split('[+-]', server_tz)[1]
+  except TypeError, ValueError:
+    LOG.error("Failed to convert Oozie timestamp: %s" % date)
+  return None

+ 8 - 2
apps/oozie/src/oozie/views/dashboard.py

@@ -30,6 +30,7 @@ from django.utils.translation import ugettext as _
 from django.core.urlresolvers import reverse
 from django.shortcuts import redirect
 
+from desktop.conf import TIME_ZONE
 from desktop.lib.django_util import JsonResponse, render
 from desktop.lib.json_utils import JSONEncoderForHTML
 from desktop.lib.exceptions_renderable import PopupException
@@ -43,13 +44,13 @@ from hadoop.fs.hadoopfs import Hdfs
 from liboozie.oozie_api import get_oozie
 from liboozie.credentials import Credentials
 from liboozie.submission2 import Submission
-from liboozie.types import Workflow as OozieWorkflow, Coordinator as CoordinatorWorkflow, Bundle as BundleWorkflow
 
 from oozie.conf import OOZIE_JOBS_COUNT, ENABLE_CRON_SCHEDULING, ENABLE_V2
 from oozie.forms import RerunForm, ParameterForm, RerunCoordForm, RerunBundleForm, UpdateCoordinatorForm
 from oozie.models import Workflow as OldWorkflow, Job, utc_datetime_format, Bundle, Coordinator, get_link, History as OldHistory
 from oozie.models2 import History, Workflow, WORKFLOW_NODE_PROPERTIES
 from oozie.settings import DJANGO_APPS
+from oozie.utils import convert_to_server_timezone
 
 
 def get_history():
@@ -108,7 +109,12 @@ def manage_oozie_jobs(request, job_id, action):
       if request.POST.get('clear_pause_time') == 'true':
         pause_time_val = ''
 
-      params = {'value': 'endtime=%s' % (request.POST.get('end_time')) + ';'
+      end_time_val = request.POST.get('end_time')
+      if end_time_val:
+        end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get())
+      if pause_time_val:
+        pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get())
+      params = {'value': 'endtime=%s' % (end_time_val) + ';'
                             'pausetime=%s' % (pause_time_val) + ';'
                             'concurrency=%s' % (request.POST.get('concurrency'))}
     elif action == 'ignore':

+ 1 - 1
apps/oozie/src/oozie/views/editor2.py

@@ -568,7 +568,7 @@ def submit_coordinator(request, doc_id):
 def _submit_coordinator(request, coordinator, mapping):
   try:
     wf_doc = Document2.objects.get(uuid=coordinator.data['properties']['workflow'])
-    wf_dir = Submission(request.user, Workflow(document=wf_doc), request.fs, request.jt, mapping).deploy()
+    wf_dir = Submission(request.user, Workflow(document=wf_doc), request.fs, request.jt, mapping, local_tz=coordinator.data['properties']['timezone']).deploy()
 
     properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
     properties.update(mapping)

+ 0 - 6
desktop/core/src/desktop/static/desktop/js/ko.hue-bindings.js

@@ -1594,12 +1594,6 @@ ko.bindingHandlers.timepicker = {
       showMeridian: false,
       defaultTime: false
     });
-    _el.on("change", function () {
-      if (_el.val().substr(-1) != "Z") {
-        _el.val(_el.val() + "Z");
-        _el.trigger("change");
-      }
-    });
   }
 }
 

+ 9 - 1
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -29,6 +29,7 @@ from desktop.lib.parameterization import find_variables
 from desktop.models import Document2
 from hadoop import cluster
 from hadoop.fs.hadoopfs import Hdfs
+from oozie.utils import convert_to_server_timezone
 
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
@@ -63,7 +64,7 @@ class Submission(object):
   - submit
   - rerun
   """
-  def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None):
+  def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None):
     self.job = job
     self.user = user
     self.fs = fs
@@ -76,6 +77,13 @@ class Submission(object):
     else:
       self.properties = {}
 
+    if local_tz and isinstance(self.job.data, dict):
+      local_tz = self.job.data.get('properties')['timezone']
+    if 'start_date' in self.properties:
+      properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz)
+    if 'end_date' in self.properties:
+      properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz)
+
     self.properties['security_enabled'] = self.api.security_enabled
 
   def __str__(self):