Преглед изворни кода

HUE-2514 [oozie] Add more reducers when running terasort on larger cluster

This fixes test_submit_java_action when run on larger
clusters by fixing the number of reducers used when
running the terasort java action.

The following argument is added to the TeraSort action in
workflow.zip/workflow.xml:

     <action name="TeraSort">
         <java>
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>
             <main-class>org.apache.hadoop.examples.terasort.TeraSort</main-class>
+            <arg>-Dmapred.reduce.tasks=${terasort_reducers}</arg>
             <arg>${output_dir}/teragen</arg>
             <arg>${output_dir}/terasort</arg>
         </java>
         <ok to="end"/>
         <error to="kill"/>
     </action>
Peter Slawski пре 11 година
родитељ
комит
c86f67c

BIN
apps/oozie/examples/managed/terasort/workflow.zip


+ 2 - 1
apps/oozie/src/oozie/test_data/workflows/0.4/test-java-different-error-links.xml

@@ -17,6 +17,7 @@
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>
             <main-class>org.apache.hadoop.examples.terasort.TeraSort</main-class>
+            <arg>-Dmapred.reduce.tasks=${terasort_reducers}</arg>
             <arg>${output_dir}/teragen</arg>
             <arg>${output_dir}/terasort</arg>
         </java>
@@ -24,4 +25,4 @@
         <error to="nonsense"/>
     </action>
     <end name="end"/>
-</workflow-app>
+</workflow-app>

+ 2 - 1
apps/oozie/src/oozie/test_data/workflows/0.4/test-java-multiple-kill.xml

@@ -17,6 +17,7 @@
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>
             <main-class>org.apache.hadoop.examples.terasort.TeraSort</main-class>
+            <arg>-Dmapred.reduce.tasks=${terasort_reducers}</arg>
             <arg>${output_dir}/teragen</arg>
             <arg>${output_dir}/terasort</arg>
         </java>
@@ -30,4 +31,4 @@
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
     <end name="end"/>
-</workflow-app>
+</workflow-app>

+ 2 - 1
apps/oozie/src/oozie/test_data/workflows/0.4/test-java.xml

@@ -17,6 +17,7 @@
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>
             <main-class>org.apache.hadoop.examples.terasort.TeraSort</main-class>
+            <arg>-Dmapred.reduce.tasks=${terasort_reducers}</arg>
             <arg>${output_dir}/teragen</arg>
             <arg>${output_dir}/terasort</arg>
         </java>
@@ -27,4 +28,4 @@
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
     <end name="end"/>
-</workflow-app>
+</workflow-app>

+ 5 - 4
apps/oozie/src/oozie/tests.py

@@ -2277,7 +2277,7 @@ class TestImportWorkflow04(OozieMockBase):
     assert_equal('org.apache.hadoop.examples.terasort.TeraGen', nodes[0].main_class)
     assert_equal('${records} ${output_dir}/teragen', nodes[0].args)
     assert_equal('org.apache.hadoop.examples.terasort.TeraSort', nodes[1].main_class)
-    assert_equal('${output_dir}/teragen ${output_dir}/terasort', nodes[1].args)
+    assert_equal('-Dmapred.reduce.tasks=${terasort_reducers} ${output_dir}/teragen ${output_dir}/terasort', nodes[1].args)
     assert_true(nodes[0].capture_output)
     assert_false(nodes[1].capture_output)
     workflow.delete(skip_trash=True)
@@ -2382,7 +2382,7 @@ class TestImportWorkflow04(OozieMockBase):
     assert_equal('org.apache.hadoop.examples.terasort.TeraGen', nodes[0].main_class)
     assert_equal('${records} ${output_dir}/teragen', nodes[0].args)
     assert_equal('org.apache.hadoop.examples.terasort.TeraSort', nodes[1].main_class)
-    assert_equal('${output_dir}/teragen ${output_dir}/terasort', nodes[1].args)
+    assert_equal('-Dmapred.reduce.tasks=${terasort_reducers} ${output_dir}/teragen ${output_dir}/terasort', nodes[1].args)
     assert_true(nodes[0].capture_output)
     assert_false(nodes[1].capture_output)
     workflow.delete(skip_trash=True)
@@ -2406,7 +2406,7 @@ class TestImportWorkflow04(OozieMockBase):
     assert_equal('org.apache.hadoop.examples.terasort.TeraGen', nodes[0].main_class)
     assert_equal('${records} ${output_dir}/teragen', nodes[0].args)
     assert_equal('org.apache.hadoop.examples.terasort.TeraSort', nodes[1].main_class)
-    assert_equal('${output_dir}/teragen ${output_dir}/terasort', nodes[1].args)
+    assert_equal('-Dmapred.reduce.tasks=${terasort_reducers} ${output_dir}/teragen ${output_dir}/terasort', nodes[1].args)
     assert_true(nodes[0].capture_output)
     assert_false(nodes[1].capture_output)
     assert_equal(1, len(Link.objects.filter(parent__workflow=workflow).filter(parent__name='TeraGenWorkflow').filter(name='error').filter(child__node_type='java')))
@@ -3028,7 +3028,8 @@ class TestOozieSubmissions(OozieBase):
                            data={u'form-MAX_NUM_FORMS': [u''],
                                 u'form-0-name': [u'records'], u'form-0-value': [u'10'],
                                 u'form-1-name': [u' output_dir '], u'form-1-value': [u'${nameNode}/user/test/out/terasort'],
-                                u'form-INITIAL_FORMS': [u'2'], u'form-TOTAL_FORMS': [u'2']},
+                                u'form-2-name': [u'terasort_reducers'], u'form-2-value': [u'3'],
+                                u'form-INITIAL_FORMS': [u'3'], u'form-TOTAL_FORMS': [u'3']},
                            follow=True)
     job = OozieServerProvider.wait_until_completion(response.context['oozie_workflow'].id)
     assert_equal('SUCCEEDED', job.status)