瀏覽代碼

HUE-1784 [oozie] False positive variable when submit from HDFS

Simple variable names like $var were only valid in Hue 2. It was a Hue
thing, Oozie only supports ${var} parameters.
To be more consistent only the curly bracket form is valid, e.g.: ${var}
Renamed test files to *_tests.py
Romain Rigaux 12 年之前
父節點
當前提交
fadfcd7c52

+ 2 - 5
apps/hbase/src/hbase/tests.py

@@ -14,12 +14,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from liboozie.oozie_api_test import OozieServerProvider
 
-try:
-  import json
-except ImportError:
-  import simplejson as json
+
+import json
 import time
 
 from django.contrib.auth.models import User

+ 1 - 1
apps/jobbrowser/src/jobbrowser/tests.py

@@ -32,7 +32,7 @@ from desktop.models import Document
 from hadoop import cluster
 from hadoop.conf import YARN_CLUSTERS
 from hadoop.yarn import resource_manager_api, mapreduce_api, history_server_api
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from oozie.models import Workflow
 
 from jobbrowser import models, views

+ 2 - 2
apps/jobsub/src/jobsub/parameterization.py

@@ -62,7 +62,7 @@ def recursive_walk(function, data):
 
   return function(data)
 
-def find_variables(data):
+def find_variables(data, include_named=True):
   """
   Finds all substitutable variables.
 
@@ -74,7 +74,7 @@ def find_variables(data):
     if not isinstance(val, basestring):
       return
     for match in Template.pattern.finditer(val):
-      name = match.group('named') or match.group('braced')
+      name = (include_named and match.group('named')) or match.group('braced')
       if name is not None:
         found.add(name)
 

+ 1 - 1
apps/jobsub/src/jobsub/tests.py

@@ -25,7 +25,7 @@ from django.core.urlresolvers import reverse
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.models import Document
 
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from oozie.models import Workflow, Node, Action, Start, Kill, End, Link
 
 

+ 1 - 1
apps/oozie/src/oozie/tests.py

@@ -40,7 +40,7 @@ from desktop.models import Document
 from jobsub.models import OozieDesign, OozieMapreduceAction
 from liboozie import oozie_api
 from liboozie.conf import OOZIE_URL
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
 

+ 3 - 5
apps/pig/src/pig/tests.py

@@ -15,10 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-  import json
-except ImportError:
-  import simplejson as json
+
+import json
 import time
 
 from django.contrib.auth.models import User
@@ -29,7 +27,7 @@ from nose.tools import assert_true, assert_equal
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
 from hadoop import pseudo_hdfs4
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from oozie.tests import OozieBase
 
 from pig.models import create_or_update_script, PigScript

+ 0 - 0
desktop/libs/liboozie/src/liboozie/credentials_test.py → desktop/libs/liboozie/src/liboozie/credentials_tests.py


+ 0 - 0
desktop/libs/liboozie/src/liboozie/oozie_api_test.py → desktop/libs/liboozie/src/liboozie/oozie_api_tests.py


+ 1 - 1
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -170,7 +170,7 @@ class Submission(object):
 
   def _get_external_parameters(self, xml, properties=None):
     from oozie.models import DATASET_FREQUENCY
-    parameters = dict([(var, '') for var in find_variables(xml) if not self._is_coordinator() or var not in DATASET_FREQUENCY])
+    parameters = dict([(var, '') for var in find_variables(xml, include_named=False) if not self._is_coordinator() or var not in DATASET_FREQUENCY])
 
     if properties:
       parameters.update(dict([line.strip().split('=')

+ 6 - 0
desktop/libs/liboozie/src/liboozie/submittions_test.py → desktop/libs/liboozie/src/liboozie/submittion_tests.py

@@ -136,6 +136,12 @@ class TestSubmission(OozieMockBase):
               <argument>INPUT=${input}</argument>
               <argument>-param</argument>
               <argument>OUTPUT=${output}</argument>
+            <configuration>
+              <property>
+                <name>mapred.input.format.class</name>
+                <value>org.apache.hadoop.examples.SleepJob$SleepInputFormat</value>
+              </property>
+            </configuration>
         </pig>
         <ok to="end"/>
         <error to="kill"/>