Browse Source

HUE-3878 [editor] Sqoop DB import snippet

Romain Rigaux 9 years ago
parent
commit
17fb36135f

+ 114 - 1
apps/oozie/src/oozie/models2.py

@@ -788,6 +788,15 @@ class Node():
         self.data['properties']['job_properties'] = job_properties
       self.data['properties']['files'] = [{'value': prop} for prop in action['properties']['parameters']]
 
+    elif self.data['type'] == SqoopDocumentAction.TYPE:
+      notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=self.data['properties']['uuid']))
+      action = notebook.get_data()['snippets'][0]
+
+      name = '%s-%s' % (self.data['type'].split('-')[0], self.data['id'][:4])
+      self.data['properties']['command'] = action['statement']
+      self.data['properties']['arguments'] = []
+
+
     data = {
       'node': self.data,
       'mapping': mapping,
@@ -2321,6 +2330,72 @@ class PigDocumentAction(Action):
     return [cls.FIELDS['uuid']]
 
 
+class SqoopDocumentAction(Action):
+  TYPE = 'sqoop-document'
+  FIELDS = {
+    'uuid': {
+        'name': 'uuid',
+        'label': _('Spark program'),
+        'value': '',
+        'help_text': _('Select a saved Spark program you want to schedule.'),
+     },
+     # Common
+     'files': {
+          'name': 'files',
+          'label': _('Files'),
+          'value': [],
+          'help_text': _('Files put in the running directory.'),
+          'type': ''
+     },
+     'archives': {
+          'name': 'archives',
+          'label': _('Archives'),
+          'value': [],
+          'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
+          'type': ''
+     },
+     'job_properties': {
+          'name': 'job_properties',
+          'label': _('Hadoop job properties'),
+          'value': [],
+          'help_text': _('value, e.g. production'),
+          'type': ''
+     },
+     'prepares': {
+          'name': 'prepares',
+          'label': _('Prepares'),
+          'value': [],
+          'help_text': _('Path to manipulate before starting the application.'),
+          'type': ''
+     },
+     'job_xml': {
+          'name': 'job_xml',
+          'label': _('Job XML'),
+          'value': '',
+          'help_text': _('Refer to a Hadoop JobConf job.xml'),
+          'type': ''
+     },
+     'retry_max': {
+          'name': 'retry_max',
+          'label': _('Max retry'),
+          'value': [],
+          'help_text': _('Number of times, default is 3'),
+          'type': ''
+     },
+     'retry_interval': {
+          'name': 'retry_interval',
+          'label': _('Retry interval'),
+          'value': [],
+          'help_text': _('Wait time in minutes, default is 10'),
+          'type': ''
+     }
+  }
+
+  @classmethod
+  def get_mandatory_fields(cls):
+    return [cls.FIELDS['uuid']]
+
+
 class DecisionNode(Action):
   TYPE = 'decision'
   FIELDS = {}
@@ -2355,7 +2430,8 @@ NODES = {
   'hive-document-widget': HiveDocumentAction,
   'java-document-widget': JavaDocumentAction,
   'spark-document-widget': SparkDocumentAction,
-  'pig-document-widget': PigDocumentAction
+  'pig-document-widget': PigDocumentAction,
+  'sqoop-document-widget': SqoopDocumentAction
 }
 
 
@@ -3163,6 +3239,8 @@ class WorkflowBuilder():
         node = self.get_spark_document_node(document, user)
       elif document.type == 'query-pig':
         node = self.get_pig_document_node(document, user)
+      elif document.type == 'query-sqoop1':
+        node = self.get_sqoop_document_node(document, user)
       else:
         raise PopupException(_('Snippet type %s is not supported in batch execution.') % document.type)
 
@@ -3316,6 +3394,41 @@ class WorkflowBuilder():
 
     return node
 
+  def get_sqoop_document_node(self, document, user):
+    node = self._get_sqoop_node(document.uuid, is_document_node=True)
+
+    node['properties']['uuid'] = document.uuid
+
+    return node
+
+  def _get_sqoop_node(self, node_id, credentials=None, is_document_node=False):
+    if credentials is None:
+      credentials = []
+
+    return {
+        "id": node_id,
+        'name': 'sqoop-%s' % node_id[:4],
+        "type": "sqoop-document-widget",
+        "properties":{
+              "command": "",
+              "arguments": [],
+              "retry_max": [],
+              "retry_interval": [],
+              "job_properties": [],
+              "capture_output": False,
+              "prepares": [],
+              "credentials": credentials,
+              "sla": [{"value":False, "key":"enabled"}, {"value":"${nominal_time}", "key":"nominal-time"}, {"value":"", "key":"should-start"}, {"value":"${30 * MINUTES}", "key":"should-end"}, {"value":"", "key":"max-duration"}, {"value":"", "key":"alert-events"}, {"value":"", "key":"alert-contact"}, {"value":"", "key":"notification-msg"}, {"value":"", "key":"upstream-apps"}],
+              "archives": []
+        },
+        "children": [
+            {"to": "33430f0f-ebfa-c3ec-f237-3e77efa03d0a"},
+            {"error": "17c9c895-5a16-7443-bb81-f34b30b21548"}
+        ],
+        "actionParameters": [],
+        "actionParametersFetched": False
+    }
+
   def get_pig_document_node(self, document, user):
     node = self._get_pig_node(document.uuid, is_document_node=True)
 

+ 1 - 0
apps/oozie/src/oozie/templates/editor2/gen/workflow-sqoop-document.xml.mako

@@ -0,0 +1 @@
+workflow-sqoop.xml.mako

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -708,6 +708,10 @@
       name=Spark
       interface=oozie
 
+    [[[sqoop1]]]
+      name=Sqoop1
+      interface=oozie
+
     # [[[mysql]]]
     #   name=MySql JDBC
     #   interface=jdbc

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -712,6 +712,10 @@
       name=Spark
       interface=oozie
 
+    [[[sqoop1]]]
+      name=Sqoop1
+      interface=oozie
+
     # [[[mysql]]]
     #   name=MySql JDBC
     #   interface=jdbc

+ 4 - 0
desktop/libs/notebook/src/notebook/conf.py

@@ -172,6 +172,10 @@ def _default_interpreters():
           'name': 'Java', 'interface': 'oozie', 'options': {}
       })
       ,
+      ('sqoop1', {
+          'name': 'Sqoop 1', 'interface': 'oozie', 'options': {}
+      })
+      ,
       ('spark2', {
           'name': 'Spark', 'interface': 'oozie', 'options': {}
       })

+ 5 - 1
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -303,11 +303,15 @@ ${ hueIcons.symbols() }
                 <img src="${ static('spark/art/icon_spark_48.png') }" class="app-icon" />
                 Spark
               <!-- /ko -->
+              <!-- ko if: editorType() == 'sqoop1' -->
+                <img src="${ static('oozie/art/icon_sqoop_48.png') }" class="app-icon" />
+                Sqoop 1
+              <!-- /ko -->
               <!-- ko if: editorType() == 'beeswax' || editorType() == 'hive' -->
                 <img src="${ static('beeswax/art/icon_beeswax_48.png') }" class="app-icon" />
                 Hive
               <!-- /ko -->
-              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2'].indexOf(editorType()) == -1 -->
+              <!-- ko if: ['impala', 'pig', 'hive', 'beeswax', 'rdbms', 'java', 'spark2', 'sqoop1'].indexOf(editorType()) == -1 -->
                 <img src="${ static('rdbms/art/icon_rdbms_48.png') }" class="app-icon" />
                 SQL
               <!-- /ko -->