Browse Source

HUE-1784 [oozie] False positive variable when submit from HDFS

Simple variable names like $var were only valid in Hue 2. It was a Hue
thing, Oozie only supports ${var} parameters.
To be more consistent only the curly bracket form is valid, e.g.: ${var}
Renamed test files to *_tests.py
Romain Rigaux 12 years ago
parent
commit
fadfcd7c52

+ 2 - 5
apps/hbase/src/hbase/tests.py

@@ -14,12 +14,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
-from liboozie.oozie_api_test import OozieServerProvider
 
 
-try:
-  import json
-except ImportError:
-  import simplejson as json
+
+import json
 import time
 import time
 
 
 from django.contrib.auth.models import User
 from django.contrib.auth.models import User

+ 1 - 1
apps/jobbrowser/src/jobbrowser/tests.py

@@ -32,7 +32,7 @@ from desktop.models import Document
 from hadoop import cluster
 from hadoop import cluster
 from hadoop.conf import YARN_CLUSTERS
 from hadoop.conf import YARN_CLUSTERS
 from hadoop.yarn import resource_manager_api, mapreduce_api, history_server_api
 from hadoop.yarn import resource_manager_api, mapreduce_api, history_server_api
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from oozie.models import Workflow
 from oozie.models import Workflow
 
 
 from jobbrowser import models, views
 from jobbrowser import models, views

+ 2 - 2
apps/jobsub/src/jobsub/parameterization.py

@@ -62,7 +62,7 @@ def recursive_walk(function, data):
 
 
   return function(data)
   return function(data)
 
 
-def find_variables(data):
+def find_variables(data, include_named=True):
   """
   """
   Finds all substitutable variables.
   Finds all substitutable variables.
 
 
@@ -74,7 +74,7 @@ def find_variables(data):
     if not isinstance(val, basestring):
     if not isinstance(val, basestring):
       return
       return
     for match in Template.pattern.finditer(val):
     for match in Template.pattern.finditer(val):
-      name = match.group('named') or match.group('braced')
+      name = (include_named and match.group('named')) or match.group('braced')
       if name is not None:
       if name is not None:
         found.add(name)
         found.add(name)
 
 

+ 1 - 1
apps/jobsub/src/jobsub/tests.py

@@ -25,7 +25,7 @@ from django.core.urlresolvers import reverse
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.models import Document
 from desktop.models import Document
 
 
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from oozie.models import Workflow, Node, Action, Start, Kill, End, Link
 from oozie.models import Workflow, Node, Action, Start, Kill, End, Link
 
 
 
 

+ 1 - 1
apps/oozie/src/oozie/tests.py

@@ -40,7 +40,7 @@ from desktop.models import Document
 from jobsub.models import OozieDesign, OozieMapreduceAction
 from jobsub.models import OozieDesign, OozieMapreduceAction
 from liboozie import oozie_api
 from liboozie import oozie_api
 from liboozie.conf import OOZIE_URL
 from liboozie.conf import OOZIE_URL
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
 from liboozie.types import WorkflowList, Workflow as OozieWorkflow, Coordinator as OozieCoordinator,\
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
   Bundle as OozieBundle, CoordinatorList, WorkflowAction, BundleList
 
 

+ 3 - 5
apps/pig/src/pig/tests.py

@@ -15,10 +15,8 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-try:
-  import json
-except ImportError:
-  import simplejson as json
+
+import json
 import time
 import time
 
 
 from django.contrib.auth.models import User
 from django.contrib.auth.models import User
@@ -29,7 +27,7 @@ from nose.tools import assert_true, assert_equal
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
 from desktop.lib.test_utils import grant_access
 from hadoop import pseudo_hdfs4
 from hadoop import pseudo_hdfs4
-from liboozie.oozie_api_test import OozieServerProvider
+from liboozie.oozie_api_tests import OozieServerProvider
 from oozie.tests import OozieBase
 from oozie.tests import OozieBase
 
 
 from pig.models import create_or_update_script, PigScript
 from pig.models import create_or_update_script, PigScript

+ 0 - 0
desktop/libs/liboozie/src/liboozie/credentials_test.py → desktop/libs/liboozie/src/liboozie/credentials_tests.py


+ 0 - 0
desktop/libs/liboozie/src/liboozie/oozie_api_test.py → desktop/libs/liboozie/src/liboozie/oozie_api_tests.py


+ 1 - 1
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -170,7 +170,7 @@ class Submission(object):
 
 
   def _get_external_parameters(self, xml, properties=None):
   def _get_external_parameters(self, xml, properties=None):
     from oozie.models import DATASET_FREQUENCY
     from oozie.models import DATASET_FREQUENCY
-    parameters = dict([(var, '') for var in find_variables(xml) if not self._is_coordinator() or var not in DATASET_FREQUENCY])
+    parameters = dict([(var, '') for var in find_variables(xml, include_named=False) if not self._is_coordinator() or var not in DATASET_FREQUENCY])
 
 
     if properties:
     if properties:
       parameters.update(dict([line.strip().split('=')
       parameters.update(dict([line.strip().split('=')

+ 6 - 0
desktop/libs/liboozie/src/liboozie/submittions_test.py → desktop/libs/liboozie/src/liboozie/submittion_tests.py

@@ -136,6 +136,12 @@ class TestSubmission(OozieMockBase):
               <argument>INPUT=${input}</argument>
               <argument>INPUT=${input}</argument>
               <argument>-param</argument>
               <argument>-param</argument>
               <argument>OUTPUT=${output}</argument>
               <argument>OUTPUT=${output}</argument>
+            <configuration>
+              <property>
+                <name>mapred.input.format.class</name>
+                <value>org.apache.hadoop.examples.SleepJob$SleepInputFormat</value>
+              </property>
+            </configuration>
         </pig>
         </pig>
         <ok to="end"/>
         <ok to="end"/>
         <error to="kill"/>
         <error to="kill"/>