Ver Fonte

[oozie] Workflow layout converter from Hue 3.7 format

Romain Rigaux há 11 anos atrás
pai
commit
14fa64ee36

+ 153 - 45
apps/oozie/src/oozie/models2.py

@@ -19,6 +19,7 @@ import json
 import logging
 import re
 import time
+import uuid
 
 from string import Template
 
@@ -33,6 +34,7 @@ from liboozie.submission2 import Submission
 from liboozie.submission2 import create_directories
 
 from oozie.conf import REMOTE_SAMPLE_DIR
+from oozie.models import Workflow as OldWorflows
 
 
 LOG = logging.getLogger(__name__)
@@ -64,23 +66,26 @@ class Workflow():
       self.data = data
     else:
       self.data = json.dumps({
-          'layout': [
-              {"size":12, "rows":[
-                    {"widgets":[{"size":12, "name":"Start", "id":"3f107997-04cc-8733-60a9-a4bb62cebffc", "widgetType":"start-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]},
-                    {"widgets":[{"size":12, "name":"End", "id":"33430f0f-ebfa-c3ec-f237-3e77efa03d0a", "widgetType":"end-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]}], 
-                 "drops":[ "temp"],
-                 "klass":"card card-home card-column span12"}              
-          ],
+          'layout': [{
+              "size":12, "rows":[
+                  {"widgets":[{"size":12, "name":"Start", "id":"3f107997-04cc-8733-60a9-a4bb62cebffc", "widgetType":"start-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]},
+                  {"widgets":[{"size":12, "name":"End", "id":"33430f0f-ebfa-c3ec-f237-3e77efa03d0a", "widgetType":"end-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]}
+              ], 
+              "drops":[ "temp"],
+              "klass":"card card-home card-column span12"
+          }],
           'workflow': workflow if workflow is not None else {
-              "id": None,"uuid":"549e2697-97cf-f931-2ce4-83dfdd03b7e7","name":"My Workflow",
+              "id": None, 
+              "uuid": None,
+              "name": "My Workflow",
               "properties": {
-                    "job_xml": "",
-                    "sla_enabled": False,
-                    "schema_version": "uri:oozie:workflow:0.4",
-                    "sla_workflow_enabled": False,
-                    "credentials": [],
-                    "properties": [],
-                    "sla": Workflow.SLA_DEFAULT
+                  "job_xml": "",
+                  "sla_enabled": False,
+                  "schema_version": "uri:oozie:workflow:0.4",
+                  "sla_workflow_enabled": False,
+                  "credentials": [],
+                  "properties": [],
+                  "sla": Workflow.SLA_DEFAULT
               },
               "nodes":[
                   {"id":"3f107997-04cc-8733-60a9-a4bb62cebffc","name":"Start","type":"start-widget","properties":{},"children":[{'to': '33430f0f-ebfa-c3ec-f237-3e77efa03d0a'}]},            
@@ -263,8 +268,14 @@ class Node():
     return 'editor/gen2/workflow-%s.xml.mako' % self.data['type']    
 
 
+class Action(object):
+  
+  @classmethod
+  def get_fields(cls):
+    return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()] + [('sla', Workflow.SLA_DEFAULT), ('credentials', [])]
 
-class PigAction():
+
+class PigAction(Action):
   TYPE = 'pig'
   FIELDS = {
      'script_path': { 
@@ -320,16 +331,12 @@ class PigAction():
      }
   }
 
-  @classmethod
-  def get_fields(cls):
-    return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()]
-  
   @classmethod
   def get_mandatory_fields(cls):
     return [cls.FIELDS['script_path']]
 
 
-class JavaAction():
+class JavaAction(Action):
   TYPE = 'java'
   FIELDS = {
      'jar_path': { 
@@ -403,16 +410,12 @@ class JavaAction():
      }
   }
 
-  @classmethod
-  def get_fields(cls):
-    return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()]
-  
   @classmethod
   def get_mandatory_fields(cls):
     return [cls.FIELDS['jar_path'], cls.FIELDS['main_class']]
   
   
-class HiveAction():
+class HiveAction(Action):
   TYPE = 'hive'
   FIELDS = {
      'script_path': { 
@@ -462,16 +465,12 @@ class HiveAction():
      }
   }
 
-  @classmethod
-  def get_fields(cls):
-    return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()]
-  
   @classmethod
   def get_mandatory_fields(cls):
     return [cls.FIELDS['script_path']]
 
 
-class SubWorkflowAction():
+class SubWorkflowAction(Action):
   TYPE = 'subworkflow'
   FIELDS = {
      'workflow': { 
@@ -494,16 +493,12 @@ class SubWorkflowAction():
      }
   }
 
-  @classmethod
-  def get_fields(cls):
-    return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()]
-  
   @classmethod
   def get_mandatory_fields(cls):
     return []
 
 
-class KillAction():
+class KillAction(Action):
   TYPE = 'kill'
   FIELDS = {
      'message': { 
@@ -514,22 +509,14 @@ class KillAction():
      }
   }
 
-  @classmethod
-  def get_fields(cls):
-    return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()]
-  
   @classmethod
   def get_mandatory_fields(cls):
     return [cls.FIELDS['message']]
 
 
-class JoinAction():
+class JoinAction(Action):
   TYPE = 'join'
   FIELDS = {}
-
-  @classmethod
-  def get_fields(cls):
-    return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()]
   
   @classmethod
   def get_mandatory_fields(cls):
@@ -587,3 +574,124 @@ def find_json_parameters(fields):
 
   return params
 
+
+
+def import_workflows_from_hue_3_7():
+#[<Start: start>, <Pig: Pig>, [<Kill: kill>], [<End: end>]]
+#[<Start: start>, <Mapreduce: Sleep>, [<Kill: kill>], [<End: end>]]
+#[<Start: start>, <Java: TeraGenWorkflow>, <Java: TeraSort>, [<Kill: kill>], [<End: end>]]
+#[<Start: start>, <Hive: Hive>, [<Kill: kill>], [<End: end>]]
+#[<Start: start>, [<Fork: fork-34>, [[<Mapreduce: Sleep-1>, <Mapreduce: Sleep-10>], [<Mapreduce: Sleep-5>, [<Fork: fork-38>, [[<Mapreduce: Sleep-3>], [<Mapreduce: Sleep-4>]], <Join: join-39>]]], <Join: join-35>], [<Kill: kill>], [<End: end>]]
+#[<Start: start>, <Pig: Pig>, [<Kill: kill>], [<End: end>]]
+#[<Start: start>, <Mapreduce: Sleep>, [<Kill: kill>], [<End: end>]]
+  
+  uuids = {}
+  
+  old_wf = OldWorflows.objects.filter(managed=True).filter(is_trashed=False)[10].get_full_node() 
+  old_nodes = old_wf.get_hierarchy()
+  
+  wf = Workflow()
+  wf_rows = []
+  wf_nodes = []
+  
+  data = wf.get_data()
+  
+  # UUIDs node mapping
+  for node in old_wf.node_list:    
+    if node.name == 'kill':
+      node_uuid = '17c9c895-5a16-7443-bb81-f34b30b21548'
+    elif node.name == 'start':
+      node_uuid = '3f107997-04cc-8733-60a9-a4bb62cebffc'
+    elif node.name == 'end':
+      node_uuid = '33430f0f-ebfa-c3ec-f237-3e77efa03d0a'
+    else:
+      node_uuid = str(uuid.uuid4())
+
+    uuids[node.id] = node_uuid
+    
+  # Workflow
+  data['workflow']['uuid'] = str(uuid.uuid4())
+  data['workflow']['name'] = old_wf.name
+  data['workflow']['properties']['properties'] = json.loads(old_wf.job_properties)
+  data['workflow']['properties']['job_xml'] = old_wf.job_xml
+  data['workflow']['properties']['description'] = old_wf.description
+  data['workflow']['properties']['schema_version'] = old_wf.schema_version
+  data['workflow']['properties']['deployment_dir'] = old_wf.deployment_dir
+  data['workflow']['properties']['parameters'] = json.loads(old_wf.parameters)
+    
+  # Layout
+  rows = data['layout'][0]['rows']
+  
+  def _create_layout(nodes, size=12):
+    wf_rows = []
+    
+    for node in nodes:      
+      if type(node) == list and len(node) == 1:
+        node = node[0]
+      if type(node) != list:
+        if node.node_type != 'kill': # No kill widget displayed yet
+          wf_rows.append({"widgets":[{"size":size, "name": node.name.title(), "id":  uuids[node.id], "widgetType": "%s-widget" % node.node_type, "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size}]})
+      else:
+        if node[0].node_type == 'fork':
+          wf_rows.append({"widgets":[{"size":size, "name": node[0].name.title(), "id":  uuids[node[0].id], "widgetType": "%s-widget" % node[0].node_type, "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size}]})  
+          
+          wf_rows.append({  
+            "id": str(uuid.uuid4()),
+            "widgets":[  
+
+            ],
+            "columns":[  
+               {  
+                  "id": str(uuid.uuid4()),
+                  "size": (size / len(node[1])),
+                  "rows":
+                     [{  
+                        "id": str(uuid.uuid4()),
+                        "widgets": col['widgets'] 
+                      } 
+                  ]
+                  ,                  
+                  "klass":"card card-home card-column span%s" % (size / len(node[1]))
+               }
+               for col in _create_layout(node[1], size)
+            ]
+          })
+          
+          wf_rows.append({"widgets":[{"size":size, "name": node[2].name.title(), "id":  uuids[node[2].id], "widgetType": "%s-widget" % node[2].node_type, "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size}]})
+        else:
+          wf_rows += _create_layout(node, size)
+
+    return wf_rows
+  
+  wf_rows = _create_layout(old_nodes[1:-1])      
+
+    
+  if wf_rows:
+    data['layout'][0]['rows'] = [data['layout'][0]['rows'][0]] + wf_rows + [data['layout'][0]['rows'][-1]]
+  
+  # Content
+  def _dig_nodes(nodes):
+    for node in nodes:
+      if type(node) != list:
+        properties = {}
+        if '%s-widget' % node.node_type in NODES and node.node_type != 'kill-widget':
+          properties = dict(NODES['%s-widget' % node.node_type].get_fields())
+        # TODO map and fill up properties
+        
+        wf_nodes.append({
+            "id": uuids[node.id],
+            "name": node.name.title(), # + uuid
+            "type": "%s-widget" % node.node_type,
+            "properties": properties,
+            "children":[{('to' if link.name == 'ok' else link.name): uuids[link.child.get_full_node().id]} for link in node.get_children_links()]
+        })
+      else:
+        _dig_nodes(node)
+  _dig_nodes(old_nodes)
+  
+  data['workflow']['nodes'] = wf_nodes
+
+  return Workflow(data=json.dumps(data))
+  
+
+

+ 4 - 0
apps/oozie/src/oozie/templates/editor/workflow_editor.mako

@@ -137,6 +137,10 @@ ${ commonheader(_("Workflow Editor"), "Oozie", user) | n,unicode }
     <a title="${ _('Gen XML') }" rel="tooltip" data-placement="bottom" data-bind="click: gen_xml, css: {'btn': true}">
       <i class="fa fa-file-code-o"></i>
     </a>
+    <a title="${ _('Import workflows') }" rel="tooltip" data-placement="bottom" data-bind="click: import_workflows, css: {'btn': true}">
+      <i class="fa fa fa-download"></i>
+    </a>
+    &nbsp;&nbsp;&nbsp;
     <a title="${ _('Submit') }" rel="tooltip" data-placement="bottom" data-bind="click: showSubmitPopup, css: {'btn': true}">
       <i class="fa fa-play"></i>
     </a>

+ 1 - 0
apps/oozie/src/oozie/urls.py

@@ -79,6 +79,7 @@ urlpatterns += patterns(
   url(r'^editor/workflow/add_node/$', 'add_node', name='add_node'),
   
   url(r'^editor/workflow/gen_xml/$', 'gen_xml_workflow', name='gen_xml_workflow'), # Temporary
+  url(r'^editor/workflow/import_workflows/$', 'import_hue_3_7_workflows', name='import_hue_3_7_workflows'), # Temporary
 )
 
 

+ 15 - 5
apps/oozie/src/oozie/views/editor2.py

@@ -35,7 +35,7 @@ from liboozie.oozie_api import get_oozie
 from liboozie.submission2 import Submission
 
 from oozie.forms import ParameterForm
-from oozie.models2 import Workflow, NODES, WORKFLOW_NODE_PROPERTIES
+from oozie.models2 import Workflow, NODES, WORKFLOW_NODE_PROPERTIES, import_workflows_from_hue_3_7
 
 
 LOG = logging.getLogger(__name__)
@@ -58,6 +58,7 @@ def edit_workflow(request):
     workflow = Workflow(document=Document2.objects.get(id=workflow_id)) # Todo perms
   else:
     workflow = Workflow()
+    workflow = import_workflows_from_hue_3_7()
   
   workflow_data = workflow.get_data()
 
@@ -153,10 +154,6 @@ def add_node(request):
     _properties.update({
        'workflow': subworkflow['value']
     })
-  _properties.update({
-      'sla': Workflow.SLA_DEFAULT,
-      'credentials': []
-  })
 
   response['status'] = 0
   response['properties'] = _properties
@@ -222,3 +219,16 @@ def _submit_workflow(user, fs, jt, workflow, mapping):
     raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
 
   return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
+
+
+def import_hue_3_7_workflows(request):
+  response = {'status': -1}
+
+  try:
+    response['status'] = 0
+    response['json'] = import_workflows_from_hue_3_7().to_xml()
+  except Exception, e:
+    response['message'] = str(e)
+    
+  return HttpResponse(json.dumps(response), mimetype="application/json") 
+  

+ 14 - 0
apps/oozie/static/js/workflow-editor.ko.js

@@ -940,6 +940,20 @@ var WorkflowEditorViewModel = function (layout_json, workflow_json, credentials_
     });
   };
 
+  self.import_workflows = function () {
+    $.post("/oozie/editor/workflow/import_workflows/", {
+    }, function (data) {
+      if (data.status == 0) {
+        console.log(data.json);
+      }
+      else {
+        $(document).trigger("error", data.message);
+     }
+   }).fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
+  
   self.showSubmitPopup = function () {
     // If self.workflow.id() == null, need to save wf for now