Răsfoiți Sursa

HUE-6267 [editor] Add MapReduce Job Editor example for Hue 4

Jenny Kim 8 ani în urmă
părinte
comite
4968807e4e

+ 9 - 1
apps/about/src/about/templates/admin_wizard.mako

@@ -19,6 +19,7 @@ from django.core.urlresolvers import reverse
 from django.utils.encoding import smart_unicode
 from django.utils.translation import ugettext as _
 
+from desktop.conf import IS_HUE_4
 from desktop.views import commonheader, commonfooter
 from metadata.conf import OPTIMIZER, has_optimizer
 %>
@@ -138,7 +139,14 @@ ${ layout.menubar(section='quick_start') }
                     </a>
                   </li>
               % endif
-              % if 'jobsub' in app_names:
+              % if IS_HUE_4.get():
+                  <li>
+                    <a href="javascript:void(0)" class="installBtn" data-loading-text="${ _('Installing...') }"
+                       data-sample-url="${ url('oozie:install_examples') }">
+                      <i class="fa fa-download"></i> ${ _('Job Editor') }
+                    </a>
+                  </li>
+              % elif 'jobsub' in app_names:
                   <li>
                     <a href="javascript:void(0)" class="installBtn" data-loading-text="${ _('Installing...') }"
                        data-sample-url="${ url('oozie:install_examples') }">

+ 82 - 8
apps/oozie/src/oozie/management/commands/oozie_setup.py

@@ -22,19 +22,22 @@ from lxml import etree
 
 from django.core import management
 from django.core.management.base import NoArgsCommand
+from django.db import transaction
 from django.utils.translation import ugettext as _
 
-from hadoop import cluster
-
-from desktop.conf import USE_NEW_EDITOR
+from desktop.conf import USE_NEW_EDITOR, IS_HUE_4
 from desktop.models import Directory, Document, Document2, Document2Permission
+from hadoop import cluster
 from liboozie.submittion import create_directories
+from notebook.models import make_notebook
+
+from useradmin.models import get_default_user_group, install_sample_user
+
 from oozie.conf import LOCAL_SAMPLE_DATA_DIR, LOCAL_SAMPLE_DIR, REMOTE_SAMPLE_DIR, ENABLE_V2
 from oozie.models import Workflow, Coordinator, Bundle
 from oozie.importlib.workflows import import_workflow_root
 from oozie.importlib.coordinators import import_coordinator_root
 from oozie.importlib.bundles import import_bundle_root
-from useradmin.models import get_default_user_group, install_sample_user
 
 
 LOG = logging.getLogger(__name__)
@@ -104,6 +107,61 @@ class Command(NoArgsCommand):
           bundle.save()
           import_bundle_root(bundle=bundle, bundle_definition_root=bundle_root, metadata=metadata)
 
+  def _install_mapreduce_example(self):
+    doc2 = None
+    name = 'MapReduce Sleep Job (example)'
+
+    if Document2.objects.filter(owner=self.user, name=name, type='query-mapreduce').exists():
+      LOG.info("Sample mapreduce editor job already installed.")
+      doc2 = Document2.objects.get(owner=self.user, name=name, type='query-mapreduce')
+    else:
+      snippet_properties = {
+        'app_jar': '/user/hue/oozie/workspaces/lib/hadoop-examples.jar',
+        'hadoopProperties': ['mapred.mapper.class=org.apache.hadoop.examples.SleepJob',
+          'mapred.reducer.class=org.apache.hadoop.examples.SleepJob',
+          'mapred.mapoutput.key.class=org.apache.hadoop.io.IntWritable',
+          'mapred.mapoutput.value.class=org.apache.hadoop.io.NullWritable',
+          'mapred.output.format.class=org.apache.hadoop.mapred.lib.NullOutputFormat',
+          'mapred.input.format.class=org.apache.hadoop.examples.SleepJob$SleepInputFormat',
+          'mapred.partitioner.class=org.apache.hadoop.examples.SleepJob',
+          'sleep.job.map.sleep.time=5', 'sleep.job.reduce.sleep.time=10'],
+        'archives': [],
+        'jars': []
+      }
+
+      notebook = make_notebook(
+        name=name,
+        description='Sleep: Example MapReduce job',
+        editor_type='mapreduce',
+        statement='',
+        status='ready',
+        snippet_properties=snippet_properties,
+        is_saved=True
+      )
+
+      # Remove files, functions, settings from snippet properties
+      data = notebook.get_data()
+      data['snippets'][0]['properties'].pop('functions')
+      data['snippets'][0]['properties'].pop('settings')
+
+      try:
+        with transaction.atomic():
+          doc2 = Document2.objects.create(
+            owner=self.user,
+            name=data['name'],
+            type='query-mapreduce',
+            description=data['description'],
+            data=json.dumps(data)
+          )
+      except Exception, e:
+        LOG.exception("Failed to create sample mapreduce job document: %s" % e)
+        # Just to be sure we delete Doc2 object incase of exception.
+        # Possible when there are mixed InnoDB and MyISAM tables
+        if doc2 and Document2.objects.filter(id=doc2.id).exists():
+          doc2.delete()
+
+    return doc2
+
   def install_examples(self):
     data_dir = LOCAL_SAMPLE_DIR.get()
 
@@ -147,7 +205,23 @@ class Command(NoArgsCommand):
       name=Document2.EXAMPLES_DIR
     )
 
-    if USE_NEW_EDITOR.get():
+    if IS_HUE_4.get():
+      # Install editor oozie examples without doc1 link
+      LOG.info("Using Hue 4, will install oozie editor samples.")
+
+      example_jobs = []
+      mr_job = self._install_mapreduce_example()
+      if mr_job:
+        example_jobs.append(mr_job)
+
+      # If documents exist but have been trashed, recover from Trash
+      for doc in example_jobs:
+        if doc.parent_directory != examples_dir:
+          doc.parent_directory = examples_dir
+          doc.save()
+
+    elif USE_NEW_EDITOR.get():
+      # Install as link-workflow doc2 to old Job Designs
       docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
       for doc in docs:
         if doc.content_object:
@@ -176,6 +250,6 @@ class Command(NoArgsCommand):
     oozie_examples.update(parent_directory=examples_dir)
     examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
 
-    self.install_examples()
-
-    Document.objects.sync()
+    if not IS_HUE_4.get():
+      self.install_examples()
+      Document.objects.sync()

+ 56 - 47
apps/pig/src/pig/management/commands/pig_setup.py

@@ -41,6 +41,61 @@ LOG = logging.getLogger(__name__)
 
 class Command(NoArgsCommand):
 
+  def install_pig_script(self, sample_user):
+    doc2 = None
+    name = 'UpperText (example)'
+
+    if Document2.objects.filter(owner=sample_user, name=name, type='query-pig').exists():
+      LOG.info("Sample pig editor script already installed.")
+      doc2 = Document2.objects.get(owner=sample_user, name=name, type='query-pig')
+    else:
+      statement = """data = LOAD '/user/hue/pig/examples/data/midsummer.txt' as (text:CHARARRAY);
+
+upper_case = FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);
+
+STORE upper_case INTO '${output}';
+"""
+      snippet_properties = {
+        'hadoopProperties': [],
+        'parameters': [],
+        'resources': []
+      }
+
+      notebook = make_notebook(
+        name=name,
+        description='UpperText: Example Pig script',
+        editor_type='pig',
+        statement=statement,
+        status='ready',
+        snippet_properties=snippet_properties,
+        is_saved=True
+      )
+
+      # Remove files, functions, settings from snippet properties
+      data = notebook.get_data()
+      data['snippets'][0]['properties'].pop('files')
+      data['snippets'][0]['properties'].pop('functions')
+      data['snippets'][0]['properties'].pop('settings')
+
+      try:
+        with transaction.atomic():
+          doc2 = Document2.objects.create(
+            owner=sample_user,
+            name=data['name'],
+            type='query-pig',
+            description=data['description'],
+            data=json.dumps(data)
+          )
+      except Exception, e:
+        LOG.exception("Failed to create sample pig script document: %s" % e)
+        # Just to be sure we delete Doc2 object incase of exception.
+        # Possible when there are mixed InnoDB and MyISAM tables
+        if doc2 and Document2.objects.filter(id=doc2.id).exists():
+          doc2.delete()
+
+    return doc2
+
+
   def handle_noargs(self, **options):
     fs = cluster.get_hdfs()
     create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
@@ -70,53 +125,7 @@ class Command(NoArgsCommand):
     if IS_HUE_4.get():
       # Install editor pig script without doc1 link
       LOG.info("Using Hue 4, will install pig editor sample.")
-      name = 'UpperText (example)'
-
-      if Document2.objects.filter(owner=sample_user, name=name, type='query-pig').exists():
-        LOG.info("Sample pig editor script already installed.")
-        doc2 = Document2.objects.get(owner=sample_user, name=name, type='query-pig')
-      else:
-        statement = """data = LOAD '/user/hue/pig/examples/data/midsummer.txt' as (text:CHARARRAY);
-
-upper_case = FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);
-
-STORE upper_case INTO '${output}';
-"""
-        snippet_properties = {
-          'hadoopProperties': [],
-          'parameters': [],
-          'resources': []
-        }
-
-        notebook = make_notebook(
-          name=name,
-          editor_type='pig',
-          statement=statement,
-          status='ready',
-          snippet_properties=snippet_properties,
-          is_saved=True
-        )
-
-        # Remove files, functions, settings from snippet properties
-        data = notebook.get_data()
-        data['snippets'][0]['properties'].pop('files')
-        data['snippets'][0]['properties'].pop('functions')
-        data['snippets'][0]['properties'].pop('settings')
-
-        try:
-          with transaction.atomic():
-            doc2 = Document2.objects.create(
-              owner=sample_user,
-              name=data['name'],
-              type='query-pig',
-              description=data['description'],
-              data=json.dumps(data)
-          )
-        except Exception, e:
-          # Just to be sure we delete Doc2 object incase of exception.
-          # Possible when there are mixed InnoDB and MyISAM tables
-          if doc2 and Document2.objects.filter(id=doc2.id).exists():
-            doc2.delete()
+      doc2 = self.install_pig_script(sample_user)
     else:
       # Install old pig script fixture
       LOG.info("Using Hue 3, will install pig script fixture.")