Эх сурвалжийг харах

HUE-6095 [editor] Import old Pig script to the new Editor

Jenny Kim 8 жил өмнө
parent
commit
32c6912a54

+ 9 - 0
apps/pig/src/pig/templates/app.mako

@@ -43,6 +43,15 @@ ${ commonheader(None, "pig", user, request) | n,unicode }
       </div>
   </div>
 
+  <div class="container-fluid">
+    % if is_hue_4:
+      <div class="alert">
+        ${ _('This is the old Pig Editor, it is recommended to instead use the new ') }
+        <a href="${ url('notebook:editor') }?type=pig" target="_blank">${_('Editor')}</a>
+      </div>
+    % endif
+  </div>
+
   <div class="container-fluid">
     <div id="scripts" class="row-fluid mainSection hide">
       <div class="card card-small">

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -763,6 +763,10 @@
       ## Name of the collection handler
       # options='{"collection": "default"}'
 
+    [[[pig]]]
+      name=Pig
+      interface=oozie
+
     [[[java]]]
       name=Java
       interface=oozie

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -765,6 +765,10 @@
       ## Name of the collection handler
       # options='{"collection": "default"}'
 
+    [[[pig]]]
+      name=Pig
+      interface=oozie
+
     [[[java]]]
       name=Java
       interface=oozie

+ 43 - 15
desktop/core/src/desktop/converter_tests.py

@@ -22,6 +22,7 @@ import json
 from nose.tools import assert_equal, assert_false, assert_true
 from django.contrib.auth.models import User
 
+from desktop.conf import IS_HUE_4
 from desktop.converters import DocumentConverter
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
@@ -313,11 +314,21 @@ class TestDocumentConverter(object):
       'id': 1000,
       'name': 'Test',
       'script': 'A = LOAD "$data"; STORE A INTO "$output";',
-      'parameters': [],
-      'resources': [],
-      'hadoopProperties': []
+      'hadoopProperties': [
+        {u'name': u'mapred.job.queue.name', u'value': u'pig'},
+        {u'name': u'mapreduce.task.profile', u'value': u'true'}
+      ],
+      'parameters': [
+        {u'name': u'input', u'value': u'/user/test/data'},
+        {u'name': u'verbose', u'value': u'true'}
+      ],
+      'resources': [
+        {u'type': u'file', u'value': u'/user/test/test.txt'},
+        {u'type': u'archive', u'value': u'/user/test/test.jar'}
+      ],
     }
     pig_script = create_or_update_script(**attrs)
+    pig_script.save()
 
     # Setting doc.last_modified to older date
     doc = Document.objects.get(id=pig_script.doc.get().id)
@@ -325,18 +336,35 @@ class TestDocumentConverter(object):
     doc = Document.objects.get(id=doc.id)
 
     try:
-      # Test that corresponding doc2 is created after convert
-      assert_false(Document2.objects.filter(owner=self.user, type='link-pigscript').exists())
-
-      converter = DocumentConverter(self.user)
-      converter.convert()
-
-      doc2 = Document2.objects.get(owner=self.user, type='link-pigscript')
-
-      # Verify absolute_url
-      response = self.client.get(doc2.get_absolute_url())
-      assert_equal(200, response.status_code)
-      assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S'))
+      if IS_HUE_4.get():
+        # Test that corresponding doc2 is created after convert
+        assert_false(Document2.objects.filter(owner=self.user, type='query-pig').exists())
+
+        converter = DocumentConverter(self.user)
+        converter.convert()
+
+        doc2 = Document2.objects.get(owner=self.user, type='query-pig')
+
+        # Verify snippet values
+        assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
+        assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement'], doc2.data_dict)
+        assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement_raw'])
+        assert_equal(['mapred.job.queue.name=pig', 'mapreduce.task.profile=true'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
+        assert_equal(['input=/user/test/data', 'verbose=true'], doc2.data_dict['snippets'][0]['properties']['parameters'])
+        assert_equal(['/user/test/test.txt', '/user/test/test.jar'], doc2.data_dict['snippets'][0]['properties']['resources'])
+      else:
+        # Test that corresponding doc2 is created after convert
+        assert_false(Document2.objects.filter(owner=self.user, type='link-pigscript').exists())
+
+        converter = DocumentConverter(self.user)
+        converter.convert()
+
+        doc2 = Document2.objects.get(owner=self.user, type='link-pigscript')
+
+        # Verify absolute_url
+        response = self.client.get(doc2.get_absolute_url())
+        assert_equal(200, response.status_code)
+        assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S'))
     finally:
       pig_script.delete()
 

+ 20 - 6
desktop/core/src/desktop/converters.py

@@ -22,10 +22,11 @@ import time
 from django.db import transaction
 from django.utils.translation import ugettext as _
 
+from desktop.conf import IS_HUE_4
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission
 from notebook.api import _historify
-from notebook.models import import_saved_beeswax_query
+from notebook.models import import_saved_beeswax_query, import_saved_pig_script
 
 
 LOG = logging.getLogger(__name__)
@@ -134,19 +135,32 @@ class DocumentConverter(object):
     try:
       from pig.models import PigScript
 
-      # TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
       docs = self._get_unconverted_docs(PigScript)
+
       for doc in docs:
         try:
           if doc.content_object:
-            data = doc.content_object.dict
-            data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
-            doc2 = self._create_doc2(
+            if IS_HUE_4.get():
+              notebook = import_saved_pig_script(doc.content_object)
+              data = notebook.get_data()
+
+              doc2 = self._create_doc2(
+                document=doc,
+                doctype=data['type'],
+                name=data['name'],
+                description=data['description'],
+                data=notebook.get_json()
+              )
+            else:
+              data = doc.content_object.dict
+              data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
+              doc2 = self._create_doc2(
                 document=doc,
                 doctype='link-pigscript',
                 description=doc.description,
                 data=json.dumps(data)
-            )
+              )
+
             self.imported_doc_count += 1
         except Exception, e:
           self.failed_doc_ids.append(doc.id)

+ 37 - 0
desktop/libs/notebook/src/notebook/models.py

@@ -203,6 +203,43 @@ def import_saved_beeswax_query(bquery):
   )
 
 
+def import_saved_pig_script(pig_script):
+  snippet_properties = {}
+
+  if pig_script.dict.get('hadoopProperties'):
+    snippet_properties['hadoopProperties'] = []
+    for prop in pig_script.dict.get('hadoopProperties'):
+      snippet_properties['hadoopProperties'].append("%s=%s" % (prop.get('name'), prop.get('value')))
+
+  if pig_script.dict.get('parameters'):
+    snippet_properties['parameters'] = []
+    for param in pig_script.dict.get('parameters'):
+      snippet_properties['parameters'].append("%s=%s" % (param.get('name'), param.get('value')))
+
+  if pig_script.dict.get('resources'):
+    snippet_properties['resources'] = []
+    for resource in pig_script.dict.get('resources'):
+      snippet_properties['resources'].append(resource.get('value'))
+
+  notebook = make_notebook(
+    name=pig_script.dict.get('name'),
+    editor_type='pig',
+    statement=pig_script.dict.get('script'),
+    status='ready',
+    snippet_properties=snippet_properties,
+    is_saved=True
+  )
+
+  # Remove files, functions, settings from snippet properties
+  data = notebook.get_data()
+  data['snippets'][0]['properties'].pop('files')
+  data['snippets'][0]['properties'].pop('functions')
+  data['snippets'][0]['properties'].pop('settings')
+
+  notebook.data = json.dumps(data)
+  return notebook
+
+
 def _convert_type(btype, bdata):
   from beeswax.models import HQL, IMPALA, RDBMS, SPARK