Ver código fonte

[oozie] Set the correct example tag to the oozie workflow examples

Romain Rigaux 12 anos atrás
pai
commit
0ef94e4

+ 1 - 1
apps/oozie/src/oozie/importlib/bundles.py

@@ -114,7 +114,7 @@ def import_bundle_root(bundle, bundle_definition_root, metadata=None):
     bundle.save()
   except:
     # There was an error importing the bundle so delete every thing associated with it.
-    bundle.delete()
+    bundle.delete(skip_trash=True)
     raise
 
 

+ 1 - 1
apps/oozie/src/oozie/importlib/coordinators.py

@@ -188,7 +188,7 @@ def import_coordinator_root(coordinator, coordinator_definition_root, metadata=N
     coordinator.save()
   except:
     # There was an error importing the coordinator so delete every thing associated with it.
-    coordinator.delete()
+    coordinator.delete(skip_trash=True)
     raise
 
 

+ 6 - 4
apps/oozie/src/oozie/importlib/jobdesigner.py

@@ -15,14 +15,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import re
 
-from jobsub.models import OozieDesign, OozieMapreduceAction, OozieStreamingAction,\
-                          OozieJavaAction
+from jobsub.models import OozieMapreduceAction, OozieStreamingAction, OozieJavaAction
 
 from oozie.models import Mapreduce, Java, Streaming
 
 
+LOG = logging.getLogger(__name__)
+
+
 def get_root_action(design):
   root = design.root_action
   if root is None:
@@ -34,8 +37,7 @@ def get_root_action(design):
   elif root.action_type == OozieJavaAction.ACTION_TYPE:
     return root.ooziejavaaction
 
-  LOG.error("Oozie action type '%s' is not valid (jobsub_oozieaction.id %s)"
-            % (root.action_type, root.id))
+  LOG.error("Oozie action type '%s' is not valid (jobsub_oozieaction.id %s)" % (root.action_type, root.id))
   return None
 
 

+ 1 - 1
apps/oozie/src/oozie/importlib/workflows.py

@@ -616,7 +616,7 @@ def import_workflow_root(workflow, workflow_definition_root, metadata=None, fs=N
     workflow.name = workflow_definition_root.get('name')
     workflow.save()
   except:
-    workflow.delete()
+    workflow.delete(skip_trash=True)
     raise
 
 

+ 7 - 26
apps/oozie/src/oozie/management/commands/oozie_setup.py

@@ -37,17 +37,10 @@ from useradmin.models import install_sample_user
 
 LOG = logging.getLogger(__name__)
 
+
 class Command(NoArgsCommand):
-  def _handle_docs(self, objects, extra=None):
-    for obj in objects:
-      if obj.doc.exists():
-        doc = obj.doc.get()
-        if extra:
-          doc.extra = extra
-        doc.save()
 
   def _import_workflows(self, directory, managed=True):
-    imported = []
     for example_directory_name in os.listdir(directory):
       if os.path.isdir(os.path.join(directory, example_directory_name)):
         with open(os.path.join(directory, example_directory_name, 'workflow.zip')) as fp:
@@ -65,11 +58,9 @@ class Command(NoArgsCommand):
           workflow.save()
           Workflow.objects.initialize(workflow)
           import_workflow_root(workflow=workflow, workflow_definition_root=workflow_root, metadata=metadata, fs=self.fs)
-          imported.append(workflow)
-    return imported
+          workflow.doc.all().delete() # Delete doc as it messes up the example sharing
 
   def _import_coordinators(self, directory):
-    imported = []
     for example_directory_name in os.listdir(directory):
       if os.path.isdir(os.path.join(directory, example_directory_name)):
         with open(os.path.join(directory, example_directory_name, 'coordinator.zip')) as fp:
@@ -84,11 +75,8 @@ class Command(NoArgsCommand):
           coordinator.name = coordinator_root.get('name')
           coordinator.save()
           import_coordinator_root(coordinator=coordinator, coordinator_definition_root=coordinator_root, metadata=metadata)
-          imported.append(coordinator)
-    return imported
 
   def _import_bundles(self, directory):
-    imported = []
     for example_directory_name in os.listdir(directory):
       if os.path.isdir(os.path.join(directory, example_directory_name)):
         with open(os.path.join(directory, example_directory_name, 'bundle.zip')) as fp:
@@ -103,29 +91,21 @@ class Command(NoArgsCommand):
           bundle.name = bundle_root.get('name')
           bundle.save()
           import_bundle_root(bundle=bundle, bundle_definition_root=bundle_root, metadata=metadata)
-          imported.append(bundle)
-    return imported
 
   def install_examples(self):
     data_dir = LOCAL_SAMPLE_DIR.get()
 
     managed_dir = os.path.join(data_dir, 'managed')
-    managed_workflows = self._import_workflows(managed_dir, managed=True)
-    self._handle_docs(managed_workflows)
+    self._import_workflows(managed_dir, managed=True)
 
     unmanaged_dir = os.path.join(data_dir, 'unmanaged')
-    unmanaged_workflows = self._import_workflows(unmanaged_dir, managed=False)
-    self._handle_docs(unmanaged_workflows, "jobsub")
+    self._import_workflows(unmanaged_dir, managed=False)
 
     coordinators_dir = os.path.join(data_dir, 'coordinators')
-    coordinators = self._import_coordinators(coordinators_dir)
-    self._handle_docs(coordinators)
+    self._import_coordinators(coordinators_dir)
 
     bundles_dir = os.path.join(data_dir, 'bundles')
-    bundles = self._import_bundles(bundles_dir)
-    self._handle_docs(bundles)
-
-    Document.objects.sync()
+    self._import_bundles(bundles_dir)
 
   def handle_noargs(self, **options):
     self.user = install_sample_user()
@@ -153,4 +133,5 @@ class Command(NoArgsCommand):
     # Load jobs
     LOG.info(_("Installing examples..."))
     self.install_examples()
+
     Document.objects.sync()

+ 2 - 2
apps/useradmin/src/useradmin/models.py

@@ -58,6 +58,7 @@ from django.utils.translation import ugettext_lazy as _t
 
 from desktop import appmanager
 from desktop.lib.exceptions_renderable import PopupException
+from desktop.models import SAMPLE_USERNAME
 from hadoop import cluster
 
 import useradmin.conf
@@ -277,8 +278,6 @@ models.signals.post_syncdb.connect(update_app_permissions)
 models.signals.post_syncdb.connect(get_default_user_group)
 
 
-SAMPLE_USERNAME = 'sample'
-
 def install_sample_user():
   """
   Setup the de-activated sample user with a certain id. Do not create a user profile.
@@ -293,6 +292,7 @@ def install_sample_user():
     except Exception, e:
       LOG.info('Sample user race condition: %s' % e)
       user = auth_models.User.objects.get(username=SAMPLE_USERNAME)
+      LOG.info('Sample user race condition, got: %s' % user)
 
   fs = cluster.get_hdfs()
   fs.do_as_user(SAMPLE_USERNAME, fs.create_home_dir)

+ 8 - 6
desktop/core/src/desktop/models.py

@@ -31,10 +31,12 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop import appmanager
 
 
-
 LOG = logging.getLogger(__name__)
 
 
+SAMPLE_USERNAME = 'sample'
+
+
 class UserPreferences(models.Model):
   """Holds arbitrary key/value strings."""
   user = models.ForeignKey(auth_models.User)
@@ -61,8 +63,8 @@ class DocumentTagManager(models.Manager):
     tags = self
 
     try:
-      sample_user = auth_models.User.objects.get(username='sample')
-      tags = tags.filter(Q(owner=user) | Q(owner=sample_user, tags__tag='example'))
+      sample_user = auth_models.User.objects.get(username=SAMPLE_USERNAME)
+      tags = tags.filter(Q(owner=user) | Q(owner=sample_user, tag=DocumentTag.EXAMPLE))
     except:
       tags = tags.filter(owner=user)
 
@@ -263,7 +265,7 @@ class DocumentManager(models.Manager):
             if not job.managed:
               doc.extra = 'jobsub'
               doc.save()
-        if job.owner.username == 'sample':
+        if job.owner.username == SAMPLE_USERNAME:
           job.doc.get().share_to_default()
     except Exception, e:
       LOG.warn(force_unicode(e))
@@ -282,7 +284,7 @@ class DocumentManager(models.Manager):
           doc.tags.add(tag)
           if job.is_trashed:
             doc.send_to_trash()
-        if job.owner.username == 'sample':
+        if job.owner.username == SAMPLE_USERNAME:
           job.doc.get().share_to_default()
     except Exception, e:
       LOG.warn(force_unicode(e))
@@ -299,7 +301,7 @@ class DocumentManager(models.Manager):
           doc = Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
           tag = DocumentTag.objects.get_example_tag(user=job.owner)
           doc.tags.add(tag)
-        if job.owner.username == 'sample':
+        if job.owner.username == SAMPLE_USERNAME:
           job.doc.get().share_to_default()
     except Exception, e:
       LOG.warn(force_unicode(e))