Эх сурвалжийг харах

[core] Move parameterization lib to desktop from jobsub

That way blacklisting the jobsub app does not brake oozie, hive, impala.

Conflicts:

	apps/beeswax/src/beeswax/api.py
Romain Rigaux 10 жил өмнө
parent
commit
82775b0

+ 1 - 1
apps/beeswax/src/beeswax/api.py

@@ -28,7 +28,7 @@ from desktop.context_processors import get_app_name
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
-from jobsub.parameterization import substitute_variables
+from desktop.lib.parameterization import substitute_variables
 from metastore import parser
 
 import beeswax.models

+ 1 - 1
apps/beeswax/src/beeswax/server/dbms.py

@@ -25,7 +25,7 @@ from django.utils.translation import ugettext as _
 
 from desktop.lib.django_util import format_preserving_redirect
 from desktop.lib.i18n import smart_str
-from jobsub.parameterization import substitute_variables
+from desktop.lib.parameterization import substitute_variables
 from filebrowser.views import location_to_url
 
 from beeswax import hive_site

+ 1 - 1
apps/beeswax/src/beeswax/tests.py

@@ -49,7 +49,7 @@ from desktop.lib.test_utils import grant_access, add_to_group
 from desktop.lib.security_util import get_localhost_name
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.pseudo_hdfs4 import is_live_cluster
-from jobsub.parameterization import substitute_variables
+from desktop.lib.parameterization import substitute_variables
 
 import desktop.conf as desktop_conf
 

+ 1 - 2
apps/beeswax/src/beeswax/views.py

@@ -41,8 +41,7 @@ from desktop.lib.django_util import login_notrequired, get_desktop_uri_prefix
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_unicode
 from desktop.models import Document
-
-from jobsub.parameterization import find_variables
+from desktop.lib.parameterization import find_variables
 
 import beeswax.forms
 import beeswax.design

+ 2 - 1
apps/jobsub/src/jobsub/models.py

@@ -20,10 +20,11 @@ import logging
 from django.db import models
 from django.core import urlresolvers
 from django.contrib.auth.models import User
-from jobsub.parameterization import find_parameters, bind_parameters
+from desktop.lib.parameterization import find_parameters, bind_parameters
 
 from django.utils.translation import ugettext_lazy as _
 
+
 LOG = logging.getLogger(__name__)
 
 

+ 2 - 1
apps/oozie/src/oozie/utils.py

@@ -23,11 +23,12 @@ from datetime import datetime
 
 from django.utils.formats import localize_input
 from django.utils.translation import ugettext as _
-from jobsub.parameterization import find_variables
+from desktop.lib.parameterization import find_variables
 
 
 LOG = logging.getLogger(__name__)
 
+
 JSON_FIELDS = ('parameters', 'job_properties', 'files', 'archives', 'prepares', 'params',
                'deletes', 'mkdirs', 'moves', 'chmods', 'touchzs')
 BOOLEAN_FIELDS = ('propagate_configuration','capture_output')

+ 3 - 21
apps/jobsub/src/jobsub/parameterization.py → desktop/core/src/desktop/lib/parameterization.py

@@ -14,33 +14,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-Support for parameterizing job designs.
-
-The current incarnation supports $input and $output only.
-
-One possible design for this is rather complicated.
-Pretty much every job (including streaming) can be specified
-as "hadoop $opts", and then "streaming" could be a template
-where $opts = "streaming-jar -map $map -reduce $reduce" and so on.
-This requires a template language that can auto-generate
-forms (and hence, needs types for each variable).  It might
-also need validation rules.  This punts on that right now
-and explicitly only supports two variables.
-
-TODO(philip): This also needs methods for simply
-indicating which variables need substitution, to
-prompt the user only for those.
-
-DEPRECATED!!!
-Jobsub uses oozie models now.
-"""
 
 import logging
+
 from string import Template
 
+
 LOG = logging.getLogger(__name__)
 
+
 def recursive_walk(function, data):
   """
   Recursively processes dicts, lists and tuples, returning a copy

+ 2 - 5
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -25,19 +25,16 @@ from django.utils.translation import ugettext as _
 
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
+from desktop.lib.parameterization import find_variables
+from desktop.models import Document2
 from hadoop import cluster
 from hadoop.fs.hadoopfs import Hdfs
 
-from desktop.models import Document2
-
-from jobsub.parameterization import find_variables
-
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
 from liboozie.credentials import Credentials
 
 
-
 LOG = logging.getLogger(__name__)
 
 def submit_dryrun(run_func):

+ 1 - 1
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -26,12 +26,12 @@ from django.utils.translation import ugettext as _
 
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
+from desktop.lib.parameterization import find_variables
 from hadoop import cluster
 from hadoop.fs.hadoopfs import Hdfs
 
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
-from jobsub.parameterization import find_variables
 from liboozie.credentials import Credentials
 
 LOG = logging.getLogger(__name__)