浏览代码

HUE-4530 [indexer] Submit notebooks as batch

Romain Rigaux 9 年之前
父节点
当前提交
b6b7636

+ 33 - 0
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -64,6 +64,39 @@ class Indexer(object):
   def run_morphline(self, request, collection_name, morphline, input_path):
     workspace_path = self._upload_workspace(morphline)
 
+#     snippets = [
+#       {
+#         u'type': u'java',
+#         u'files': [
+#             {u'path': u'%s/log4j.properties' % workspace_path, u'type': u'file'},
+#             {u'path': u'%s/morphline.conf' % workspace_path, u'type': u'file'}
+#         ],
+#         u'class': u'org.apache.solr.hadoop.MapReduceIndexerTool',
+#         u'app_jar': CONFIG_INDEXER_LIBS_PATH.get(),
+#         u'arguments': [
+#             u'--morphline-file',
+#             u'morphline.conf',
+#             u'--output-dir',
+#             u'${nameNode}/user/%s/indexer' % self.username,
+#             u'--log4j',
+#             u'log4j.properties',
+#             u'--go-live',
+#             u'--zk-host',
+#             zkensemble(),
+#             u'--collection',
+#             collection_name,
+#             input_path,
+#         ],
+#         u'archives': [],
+#       }
+#     ]
+#
+#     # managed notebook
+#     notebook = make_notebook2(name='Indexer job for %s' % collection_name, snippets=snippets).get_data()
+#     notebook_doc, created = _save_notebook(notebook, self.user)
+#
+#     snippet = {'wasBatchExecuted': True}
+
     snippet_properties =  {
        u'files': [
            {u'path': u'%s/log4j.properties' % workspace_path, u'type': u'file'},

+ 6 - 3
desktop/libs/notebook/src/notebook/connectors/oozie_batch.py

@@ -62,9 +62,12 @@ class OozieApi(Api):
 
     notebook_doc = Document2.objects.get_by_uuid(user=self.user, uuid=notebook['uuid'], perm_type='read')
 
-    # Create a managed workflow from the notebook doc
-    workflow_doc = WorkflowBuilder().create_workflow(document=notebook_doc, user=self.user, managed=True, name=_("Batch job for %s") % (notebook_doc.name or notebook_doc.type))
-    workflow = Workflow(document=workflow_doc, user=self.user)
+    if notebook_doc.type == 'notebook':
+      pass
+    else:
+      # Create a managed workflow from the notebook doc
+      workflow_doc = WorkflowBuilder().create_workflow(document=notebook_doc, user=self.user, managed=True, name=_("Batch job for %s") % (notebook_doc.name or notebook_doc.type))
+      workflow = Workflow(document=workflow_doc, user=self.user)
 
     # Submit workflow
     job_id = _submit_workflow(user=self.user, fs=self.fs, jt=self.jt, workflow=workflow, mapping=None)

+ 58 - 1
desktop/libs/notebook/src/notebook/models.py

@@ -122,7 +122,64 @@ def make_notebook(name='Browse', description='', editor_type='hive', statement='
 
 
 def make_notebook2(name='Browse', description='', is_saved=False, snippets=None):
-  pass
+
+  from notebook.connectors.hiveserver2 import HS2Api
+
+  editor = Notebook()
+
+  _snippets = []
+
+  for snippet in snippets:
+    default_properties = {
+        'files': [],
+        'functions': [],
+        'settings': []
+    }
+
+    if snippet['type'] == 'hive':
+      pass
+    elif snippet['type'] == 'impala':
+      pass
+    elif snippet['type'] == 'java':
+      pass
+
+    _snippets.append(snippet)
+
+  print _snippets
+
+  data = {
+    'name': name,
+    'uuid': str(uuid.uuid4()),
+    'description': description,
+    'sessions': [
+      {
+         'type': _snippet['type'],
+         'properties': HS2Api.get_properties(snippet['type']),
+         'id': None
+      } for _snippet in _snippets # Non unique types currently
+    ],
+    'selectedSnippet': _snippets[0]['type'],
+    'type': 'notebook',
+    'showHistory': False,
+    'isSaved': is_saved,
+    'snippets': [
+      {
+         'status': _snippet.get('status', 'ready'),
+         'id': str(uuid.uuid4()),
+         'statement_raw': _snippet.get('statement', ''),
+         'statement': _snippet.get('statement', ''),
+         'type': _snippet.get('type'),
+         'properties': _snippet.properties,
+         'name': name,
+         'database': _snippet.get('database'),
+         'result': {}
+      } for _snippet in _snippets
+    ]
+  }
+
+  editor.data = json.dumps(data)
+
+  return editor
 
 
 def import_saved_beeswax_query(bquery):