Jelajahi Sumber

HUE-1922 [oozie] Smarter file symlinking in workflow action

krish 10 tahun lalu
induk
melakukan
2ead0dd5fc

+ 5 - 1
apps/oozie/src/oozie/templates/editor2/gen/workflow-common.xml.mako

@@ -17,7 +17,7 @@
 
 <%!
   import posixpath
-  from oozie.utils import smart_path
+  from oozie.utils import smart_path, contains_symlink
 %>
 
 
@@ -59,7 +59,11 @@
 <%def name="distributed_cache(files, archives)">
     % for f in files:
         % if f:
+          % if contains_symlink(f['value'], mapping):
+            <file>${ f['value'] }</file>
+          % else:
             <file>${ filelink(f['value']) }</file>
+          % endif
         % endif
     % endfor
     % for a in archives:

+ 11 - 1
apps/oozie/src/oozie/tests.py

@@ -48,7 +48,7 @@ from oozie.conf import ENABLE_CRON_SCHEDULING, ENABLE_V2
 from oozie.models import Workflow, Node, Kill, Link, Job, Coordinator, History,\
   find_parameters, NODE_TYPES, Bundle
 from oozie.models2 import _get_hiveserver2_url
-from oozie.utils import workflow_to_dict, model_to_dict, smart_path
+from oozie.utils import workflow_to_dict, model_to_dict, smart_path, contains_symlink
 from oozie.importlib.workflows import import_workflow
 from oozie.importlib.jobdesigner import convert_jobsub_design
 
@@ -3622,6 +3622,16 @@ class TestUtils(OozieMockBase):
     assert_equal('${output}', smart_path('${output}', {'output': '${path}'}))
     assert_equal('${output_dir}', smart_path('${output_dir}', {'output': '/path/out', 'output_dir': 'hdfs://nn/path/out'}))
 
+  def test_contains_symlink(self):
+    assert_false(contains_symlink('out', {'output': '/path/out'}))
+    assert_true(contains_symlink('out#out', {'output': '/path/out'}))
+    assert_false(contains_symlink('${output}', {'output': '/path/out'}))
+    assert_true(contains_symlink('hdfs://nn${output}', {'output': '/path/out#out'}))
+    assert_false(contains_symlink('hdfs://nn${output}', {'output': '/path/out'}))
+    assert_true(contains_symlink('hdfs://nn#${output}', {'output': 'output'}))
+    assert_false(contains_symlink('${output}', {}))
+    assert_false(contains_symlink('${output}', {'output': '${path}'}))
+    assert_true(contains_symlink('${output_dir}', {'output': '/path/out', 'output_dir': 'hdfs://nn/path/out#out'}))
 
 # Utils
 WORKFLOW_DICT = {

+ 3 - 0
apps/oozie/src/oozie/utils.py

@@ -118,6 +118,9 @@ def smart_path(path, mapping, is_coordinator=False):
 
   return path
 
+def contains_symlink(path, mapping):
+  vars = find_variables(path)
+  return any([var in mapping and '#' in mapping[var] for var in vars]) or '#' in path
 
 def utc_datetime_format(utc_time):
   return utc_time.strftime(UTC_TIME_FORMAT)