Explorar el Código

HUE-1501 [oozie] Support coordinator datasets from non HDFS

Romain Rigaux hace 12 años
padre
commit
d554856
Se han modificado 2 ficheros con 6 adiciones y 5 borrados
  1. 2 2
      apps/oozie/src/oozie/tests.py
  2. 4 3
      apps/oozie/src/oozie/utils.py

+ 2 - 2
apps/oozie/src/oozie/tests.py

@@ -1259,7 +1259,7 @@ class TestEditor(OozieMockBase):
 
     self.c.post(reverse('oozie:create_coordinator_dataset', args=[coord.id]), {
                           u'create-name': [u'MyDataset2'], u'create-frequency_number': [u'1'], u'create-frequency_unit': [u'days'],
-                          u'create-uri': [u'/data/out/${YEAR}${MONTH}${DAY}'],
+                          u'create-uri': [u's3n://a-server/data/out/${YEAR}${MONTH}${DAY}'],
                           u'create-instance_choice': [u'single'],
                           u'instance_start': [u'-1'],
                           u'create-advanced_start_instance': [u'0'],
@@ -1290,7 +1290,7 @@ class TestEditor(OozieMockBase):
     </dataset>
     <dataset name="MyDataset2" frequency="${coord:days(1)}"
              initial-instance="2012-07-01T00:00Z" timezone="America/Los_Angeles">
-      <uri-template>${nameNode}/data/out/${YEAR}${MONTH}${DAY}</uri-template>
+      <uri-template>s3n://a-server/data/out/${YEAR}${MONTH}${DAY}</uri-template>
       <done-flag></done-flag>
     </dataset>
   </datasets>

+ 4 - 3
apps/oozie/src/oozie/utils.py

@@ -18,6 +18,7 @@
 import json
 import logging
 import re
+import urlparse
 from datetime import datetime
 
 from django.utils.formats import localize_input
@@ -97,7 +98,7 @@ def smart_path(path, mapping):
   # This dynamic checking enable the use of <prepares> statements in a workflow scheduled manually of by a coordinator.
   # The logic is a bit complicated but Oozie is not consistent with data paths, prepare, coordinator paths and Fs action.
 
-  if not path.startswith('$') and not path.startswith('/') and not path.startswith('hdfs://'):
+  if not path.startswith('$') and not path.startswith('/') and not urlparse.urlsplit(path).scheme:
     path = '/user/%(username)s/%(path)s' % {'username': '${wf:user()}', 'path': path}
 
   if path.startswith('$'):
@@ -106,10 +107,10 @@ def smart_path(path, mapping):
       prefix = '${%s}' % var
       if path.startswith(prefix):
         if var in mapping:
-          if not mapping[var].startswith('hdfs://') and not mapping[var].startswith('$'):
+          if not urlparse.urlsplit(mapping[var]).scheme and not mapping[var].startswith('$'):
             path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}
   else:
-    if not path.startswith('hdfs://'):
+    if not urlparse.urlsplit(path).scheme:
       path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}
 
   return path