ソースを参照

HUE-6094 [editor] Convert jobsub Java document types to new editor

Jenny Kim 8 年 前
コミット
50be558

+ 64 - 0
desktop/core/src/desktop/converter_tests.py

@@ -418,6 +418,70 @@ class TestDocumentConverter(object):
       wf.delete()
 
 
+  def test_convert_java(self):
+    wf = Workflow.objects.new_workflow(self.user)
+    wf.save()
+    Workflow.objects.initialize(wf)
+    Link.objects.filter(parent__workflow=wf).delete()
+    action = add_node(wf, 'action-name-1', 'java', [wf.start], {
+      'name': 'MyTeragen',
+      "description": "Generate N number of records",
+      "main_class": "org.apache.hadoop.examples.terasort.TeraGen",
+      "args": "1000 ${output_dir}/teragen",
+      "files": '["my_file","my_file2"]',
+      "job_xml": "",
+      "java_opts": "-Dexample-property=natty",
+      "jar_path": "/user/hue/oozie/workspaces/lib/hadoop-examples.jar",
+      'job_properties': '[{"name": "mapred.job.queue.name", "value": "test"}]',
+      "prepares": '[{"value":"/test","type":"mkdir"}]',
+      "archives": '[{"dummy":"","name":"my_archive"},{"dummy":"","name":"my_archive2"}]',
+      "capture_output": "on",
+    })
+    Link(parent=action, child=wf.end, name="ok").save()
+
+    # Setting doc.last_modified to older date
+    doc = Document.objects.get(id=wf.doc.get().id)
+    Document.objects.filter(id=doc.id).update(
+      last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
+    doc = Document.objects.get(id=doc.id)
+
+    try:
+      if IS_HUE_4.get():
+        # Test that corresponding doc2 is created after convert
+        assert_false(Document2.objects.filter(owner=self.user, type='query-java').exists())
+
+        converter = DocumentConverter(self.user)
+        converter.convert()
+
+        doc2 = Document2.objects.get(owner=self.user, type='query-java')
+
+        # Verify snippet values
+        assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
+        assert_equal('/user/hue/oozie/workspaces/lib/hadoop-examples.jar', doc2.data_dict['snippets'][0]['properties']['app_jar'])
+        assert_equal('org.apache.hadoop.examples.terasort.TeraGen', doc2.data_dict['snippets'][0]['properties']['class'])
+        assert_equal('1000 ${output_dir}/teragen', doc2.data_dict['snippets'][0]['properties']['args'])
+        assert_equal('-Dexample-property=natty', doc2.data_dict['snippets'][0]['properties']['java_opts'])
+        assert_equal(['mapred.job.queue.name=test'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
+        assert_equal(['my_archive', 'my_archive2'], doc2.data_dict['snippets'][0]['properties']['archives'])
+        assert_equal([{'type': 'file', 'path': 'my_file'}, {'type': 'file', 'path': 'my_file2'}], doc2.data_dict['snippets'][0]['properties']['files'])
+        assert_equal(True, doc2.data_dict['snippets'][0]['properties']['capture_output'])
+      else:
+        # Test that corresponding doc2 is created after convert
+        assert_false(Document2.objects.filter(owner=self.user, type='link-workflow').exists())
+
+        converter = DocumentConverter(self.user)
+        converter.convert()
+
+        doc2 = Document2.objects.get(owner=self.user, type='link-workflow')
+
+        # Verify absolute_url
+        response = self.client.get(doc2.get_absolute_url())
+        assert_equal(200, response.status_code)
+        assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S'))
+    finally:
+      wf.delete()
+
+
   def test_convert_pig_script(self):
     attrs = {
       'user': self.user,

+ 4 - 2
desktop/core/src/desktop/converters.py

@@ -26,8 +26,8 @@ from desktop.conf import IS_HUE_4
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission
 from notebook.api import _historify
-from notebook.models import import_saved_beeswax_query, import_saved_pig_script, import_saved_mapreduce_job, \
-  import_saved_shell_job
+from notebook.models import import_saved_beeswax_query, import_saved_java_job, import_saved_mapreduce_job, \
+  import_saved_pig_script, import_saved_shell_job
 
 
 LOG = logging.getLogger(__name__)
@@ -160,6 +160,8 @@ class DocumentConverter(object):
                 notebook = import_saved_mapreduce_job(doc.content_object)
               elif node.node_type == 'shell':
                 notebook = import_saved_shell_job(doc.content_object)
+              elif node.node_type == 'java':
+                notebook = import_saved_java_job(doc.content_object)
 
             if notebook:
               data = notebook.get_data()

+ 55 - 0
desktop/libs/notebook/src/notebook/models.py

@@ -358,6 +358,61 @@ def import_saved_shell_job(wf):
     return notebook
 
 
+def import_saved_java_job(wf):
+    snippet_properties = {}
+    node = wf.start.get_child('to')
+
+    snippet_properties['app_jar'] = node.jar_path
+    snippet_properties['class'] = node.main_class
+    snippet_properties['args'] = node.args if node.args else ''
+    snippet_properties['java_opts'] = node.java_opts if node.java_opts else ''
+
+    snippet_properties['hadoopProperties'] = []
+    try:
+      properties = json.loads(node.job_properties)
+      if properties:
+        for prop in properties:
+          snippet_properties['hadoopProperties'].append("%s=%s" % (prop.get('name'), prop.get('value')))
+    except ValueError, e:
+      LOG.warn('Failed to parse job properties for Java job design "%s".' % wf.name)
+
+    snippet_properties['files'] = []
+    try:
+      files = json.loads(node.files)
+      for filepath in files:
+        snippet_properties['files'].append({'type': 'file', 'path': filepath})
+    except ValueError, e:
+      LOG.warn('Failed to parse files for Java job design "%s".' % wf.name)
+
+    snippet_properties['archives'] = []
+    try:
+      archives = json.loads(node.archives)
+      for archive in archives:
+        snippet_properties['archives'].append(archive['name'])
+    except ValueError, e:
+      LOG.warn('Failed to parse archives for Java job design "%s".' % wf.name)
+
+    snippet_properties['capture_output'] = node.capture_output
+
+    notebook = make_notebook(
+        name=wf.name,
+        description=wf.description,
+        editor_type='java',
+        statement='',
+        status='ready',
+        snippet_properties=snippet_properties,
+        is_saved=True
+    )
+
+    # Remove functions, settings from snippet properties
+    data = notebook.get_data()
+    data['snippets'][0]['properties'].pop('functions')
+    data['snippets'][0]['properties'].pop('settings')
+
+    notebook.data = json.dumps(data)
+    return notebook
+
+
 def _convert_type(btype, bdata):
   from beeswax.models import HQL, IMPALA, RDBMS, SPARK