Parcourir la source

HUE-4568 [editor] Export a query to an index

With on/off flag.
Refactor batch submission to be more generic and robust.
Romain Rigaux il y a 9 ans
Parent
commit
0c01184d4b

+ 35 - 18
apps/oozie/src/oozie/models2.py

@@ -2935,7 +2935,7 @@ def _save_workflow(workflow, layout, user, fs=None):
 
   dependencies = \
       [node['properties']['workflow'] for node in workflow['nodes'] if node['type'] == 'subworkflow-widget'] + \
-      [node['properties']['uuid'] for node in workflow['nodes'] if 'document-widget' in node['type']]
+      [node['properties']['uuid'] for node in workflow['nodes'] if 'document-widget' in node['type'] and node['properties'].get('uuid')]
   if dependencies:
     dependency_docs = Document2.objects.filter(uuid__in=dependencies)
     workflow_doc.dependencies.add(*dependency_docs)
@@ -2973,7 +2973,7 @@ class WorkflowBuilder():
       if document.type == 'query-java':
         node = self.get_java_document_node(document, name)
       else:
-        node = self.get_hive_document_node(document, name, user)
+        node = self.get_hive_document_node(document, user)
 
       nodes.append(node)
 
@@ -2990,11 +2990,10 @@ class WorkflowBuilder():
       name = _('Schedule of ') + ','.join([snippet['name'] or snippet['type'] for snippet in notebook['snippets']])
 
     for snippet in notebook['snippets']:
-      print snippet
       if snippet['type'] == 'java':
         node = self.get_java_snippet_node(snippet)
-      elif snippet['type'] == 'java':
-        node = self.get_hive_document_node(snippet, user)
+      elif snippet['type'] == 'query-hive':
+        node = self.get_hive_snippet_node(snippet, user)
       else:
         raise PopupException(_('Snippet type %(type)s is not supported in batch execution.') % snippet)
 
@@ -3004,25 +3003,24 @@ class WorkflowBuilder():
 
     return workflow_doc
 
+  def get_document_parameters(self, document):
+    notebook = Notebook(document=document)
+    parameters = find_dollar_braced_variables(notebook.get_str())
+
+    return [{u'value': u'%s=${%s}' % (p, p)} for p in parameters]
 
-  def get_hive_document_node(self, document, name, user):
+  def _get_hive_node(self, node_id, user, is_document_node=False):
     api = get_oozie(user)
 
     credentials = [HiveDocumentAction.DEFAULT_CREDENTIALS] if api.security_enabled else []
 
-    notebook = Notebook(document=document)
-    parameters = find_dollar_braced_variables(notebook.get_str())
-    parameters = [{u'value': u'%s=${%s}' % (p, p)} for p in parameters]
-
     return {
-        u'name': u'doc-hive-%s' % document.uuid[:4],
-        u'id': str(uuid.uuid4()),
-        u'type': u'hive-document-widget',
+        u'id': node_id,
+        u'name': u'hive-%s' % node_id[:4],
+        u"type": u"hive-document-widget", # if is_document_node else u"hive2-widget",
         u'properties': {
             u'files': [],
             u'job_xml': u'',
-            u'uuid': document.uuid, # + snippet uuid
-            u'parameters': parameters,
             u'retry_interval': [],
             u'retry_max': [],
             u'job_properties': [],
@@ -3050,13 +3048,32 @@ class WorkflowBuilder():
         u'actionParameters': [],
     }
 
+  def get_hive_snippet_node(self, snippet, user):
+    node = self._get_hive_node(snippet['id'], user)
+
+    node['properties']['parameters'] = []
+    node['properties']['statements'] = 'USE %s;\n\n%s' % (snippet['database'], snippet['statement_raw'])
+    node['properties']['parameters'] = []
+
+    return node
+
+  def get_hive_document_node(self, document, user):
+    node = self._get_hive_node(document.uuid, user, is_document_node=True)
+
+    notebook = Notebook(document=document)
+    parameters = find_dollar_braced_variables(notebook.get_str())
+    node['parameters'] = [{u'value': u'%s=${%s}' % (p, p)} for p in parameters] #Todo check if need properties
+    node['properties']['uuid'] = document.uuid
+
+    return node
+
   def _get_java_node(self, node_id, credentials=None, is_document_node=False):
     if credentials is None:
       credentials = []
 
     return {
         "id": node_id,
-        'name': 'doc-hive-%s' % node_id[:4],
+        'name': 'java-%s' % node_id[:4],
         "type": "java-document-widget" if is_document_node else "java-widget",
         "properties":{
               "job_xml": [],
@@ -3082,7 +3099,7 @@ class WorkflowBuilder():
   def get_java_snippet_node(self, snippet):
     credentials = []
 
-    node_id = snippet.get('id') or str(uuid.uuid4())
+    node_id = snippet.get('id', str(uuid.uuid4()))
 
     node = self._get_java_node(node_id, credentials)
     node['properties']['main_class'] = snippet['properties']['class']
@@ -3104,7 +3121,7 @@ class WorkflowBuilder():
     parameters = []
 
     data = {
-      'workflow': {
+      u'workflow': {
       u'name': name,
       u'nodes': [{
           u'name': u'Start',

+ 33 - 11
apps/oozie/src/oozie/models2_tests.py

@@ -36,7 +36,7 @@ from oozie.importlib.workflows import generate_v2_graph_nodes
 from oozie.models2 import Node, Workflow, WorkflowConfiguration, find_dollar_variables, find_dollar_braced_variables, \
     _create_graph_adjaceny_list, _get_hierarchy_from_adj_list, WorkflowBuilder
 from oozie.tests import OozieMockBase, save_temp_workflow, MockOozieApi
-from notebook.models import make_notebook
+from notebook.models import make_notebook, make_notebook2
 from notebook.api import _save_notebook
 
 
@@ -981,19 +981,41 @@ class TestModelAPI(OozieMockBase):
     assert_equal(len(_data['workflow']['nodes']), 4)
 
 
-  def test_gen_workflow_from_documents(self):
-    notebook = make_notebook(name='Browse', editor_type='hive', statement='SHOW TABLES', status='ready')
-    notebook_doc, save_as = _save_notebook(notebook.get_data(), self.user)
-
-    notebook2 = make_notebook(name='Browse', editor_type='hive', statement='SHOW TABLES', status='ready')
-    notebook2_doc, save_as = _save_notebook(notebook2.get_data(), self.user)
-
-    workflow_doc = WorkflowBuilder().create_workflow(documents=[notebook_doc, notebook2_doc], user=self.user, managed=True)
+  def test_gen_workflow_from_notebook(self):
+    snippets = [
+      {
+         'status': 'running',
+         'statement_raw': 'SHOW TABLES',
+         'statement': 'SHOW TABLES',
+         'type': 'query-hive',
+         'properties': {
+         },
+         'database': 'default',
+      },
+      {
+        'type': 'java',
+        'status': 'running',
+        'properties':  {
+          'files': [],
+          'class': 'org.apache.solr.hadoop.MapReduceIndexerTool',
+          'app_jar': '/user/hue/app.jar',
+          'arguments': [
+              '--morphline-file',
+              'morphline.conf',
+          ],
+          'archives': [],
+        }
+      }
+    ]
 
+    notebook = make_notebook2(name='2 actions', snippets=snippets)
+    notebook_data = notebook.get_data()
+    workflow_doc = WorkflowBuilder().create_notebook_workflow(notebook=notebook_data, user=self.user, managed=True)
     workflow = Workflow(document=workflow_doc, user=self.user)
 
     _data = workflow.get_data()
 
     assert_equal(len(_data['workflow']['nodes']), 5)
-    assert_equal(len(re.findall('<ok to="doc-hive-', workflow.to_xml())), 1, workflow.to_xml())
-    assert_equal(len(re.findall('<action name="doc-hive-', workflow.to_xml())), 2, workflow.to_xml())
+    assert_equal(len(re.findall('<ok to="java-', workflow.to_xml())), 1, workflow.to_xml())
+    assert_equal(len(re.findall('<action name="hive-', workflow.to_xml())), 1, workflow.to_xml())
+    assert_equal(len(re.findall('<action name="java-', workflow.to_xml())), 1, workflow.to_xml())

+ 9 - 6
desktop/conf.dist/hue.ini

@@ -600,6 +600,12 @@
   ## Flag to enable the bulk submission of queries as a background task through Oozie.
   # enable_batch_execute=true
 
+  ## Flag to enable the SQL query builder of the table assist.
+  # enable_query_builder=true
+
+  ## Flag to enable the creation of a coordinator for the current SQL query.
+  # enable_query_scheduling=true
+
   ## Flag to enable the Java document in editor and workflow.
   # enable_java_document=true
 
@@ -702,12 +708,6 @@
   ## Main flag to override the automatic starting of the DBProxy server.
   # enable_dbproxy_server=true
 
-  ## Flag to enable the SQL query builder of the table assist.
-  # enable_query_builder=true
-
-  ## Flag to enable the creation of a coordinator for the current SQL query.
-  # enable_query_scheduling=true
-
 
 ###########################################################################
 # Settings to configure your Hadoop cluster.
@@ -1100,6 +1100,9 @@
   # Location of the solrctl binary.
   ## solrctl_path=/usr/bin/solrctl
 
+  # Flag to turn on the morphline based Solr indexer.
+  ## enable_new_indexer=false
+
 
 ###########################################################################
 # Settings to configure Job Designer

+ 9 - 6
desktop/conf/pseudo-distributed.ini.tmpl

@@ -608,6 +608,12 @@
   ## Flag to enable the bulk submission of queries as a background task through Oozie.
   # enable_batch_execute=true
 
+  ## Flag to enable the SQL query builder of the table assist.
+  # enable_query_builder=true
+
+  ## Flag to enable the creation of a coordinator for the current SQL query.
+  # enable_query_scheduling=true
+
   ## Flag to enable the Java document in editor and workflow.
   # enable_java_document=true
 
@@ -710,12 +716,6 @@
   ## Main flag to override the automatic starting of the DBProxy server.
   # enable_dbproxy_server=true
 
-  ## Flag to enable the SQL query builder of the table assist.
-  # enable_query_builder=true
-
-  ## Flag to enable the creation of a coordinator for the current SQL query.
-  # enable_query_scheduling=true
-
 
 ###########################################################################
 # Settings to configure your Hadoop cluster.
@@ -1110,6 +1110,9 @@
   # Location of the solrctl binary.
   ## solrctl_path=/usr/bin/solrctl
 
+  # Flag to turn on the morphline based Solr indexer.
+  ## enable_new_indexer=false
+
 
 ###########################################################################
 # Settings to configure Job Designer

+ 6 - 0
desktop/core/src/desktop/templates/common_header.mako

@@ -663,6 +663,12 @@ if USE_NEW_EDITOR.get():
                  % endif
                  % if 'indexer' in apps:
                  <li><a href="${ url('indexer:collections') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-database" style="vertical-align: middle"></i> ${ _('Indexes') }</a></li>
+                 <%!
+                 from indexer.conf import ENABLE_NEW_INDEXER
+                 %>
+                 % if ENABLE_NEW_INDEXER.get():
+                 <li><a href="${ url('indexer:indexer') }" style="height: 24px; line-height: 24px!important;"><i class="fa fa-plus" style="vertical-align: middle"></i> ${ _('Index') }</a></li>
+                 % endif
                  % endif
                  <li class="divider"></li>
                % endif

+ 27 - 8
desktop/libs/indexer/src/indexer/api3.py

@@ -23,12 +23,14 @@ from django.utils.translation import ugettext as _
 from beeswax.server import dbms
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
-from notebook.connectors.base import get_api
+from desktop.models import Document2
+from notebook.connectors.base import get_api, Notebook
 
-from indexer.smart_indexer import Indexer
 from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
+from indexer.smart_indexer import Indexer
+
 
 LOG = logging.getLogger(__name__)
 
@@ -79,7 +81,7 @@ def guess_format(request):
     else:
       raise PopupException('Hive table format %s is not supported.' % table_metadata.details['properties']['format'])
   elif file_format['inputFormat'] == 'query':
-    format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\t"} # \t --> CTRL+A
+    format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\u0001"}
 
   return JsonResponse(format_)
 
@@ -110,19 +112,35 @@ def guess_field_types(request):
             for col in table_metadata.cols
         ]
     }
-  elif file_format['inputFormat'] == 'query':
+  elif file_format['inputFormat'] == 'query': # Only support open query history
     # TODO get schema from explain query, which is not possible
-    # Only support select * for now and would require a workflow that generates the morphline on the fly based on a temporary CTAS
-    format_ = {u'sample': [[u'00-0000', u'All Occupations', 134354250, 40690], [u'11-0000', u'Management occupations', 6003930, 96150], [u'11-1011', u'Chief executives', 299160, 151370], [u'11-1021', u'General and operations managers', 1655410, 103780]], u'columns': [{u'operations': [], u'name': u'code', u'required': False, u'keep': True, u'unique': False, u'type': u'string'}, {u'operations': [], u'name': u'description', u'required': False, u'keep': True, u'unique': False, u'type': u'string'}, {u'operations': [], u'name': u'total_emp', u'required': False, u'keep': True, u'unique': False, u'type': u'string'}, {u'operations': [], u'name': u'salary', u'required': False, u'keep': True, u'unique': False, u'type': u'string'}]}
+    notebook = Notebook(document=Document2.objects.get(id=file_format['query'])).get_data()
+    snippet = notebook['snippets'][0]
+    sample = get_api(request, snippet).fetch_result(notebook, snippet, 4, start_over=True)
+
+    format_ = {
+        "sample": sample['rows'][:4],
+        "columns": [
+            Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
+            for col in sample.meta
+        ]
+    }
 
   return JsonResponse(format_)
 
+
 def index_file(request):
   file_format = json.loads(request.POST.get('fileFormat', '{}'))
   _convert_format(file_format["format"], inverse=True)
   collection_name = file_format["name"]
 
+  job_handle = _index(request, file_format, collection_name)
+  return JsonResponse(job_handle)
+
+
+def _index(request, file_format, collection_name, query=None):
   indexer = Indexer(request.user, request.fs)
+
   unique_field = indexer.get_unique_field(file_format)
   is_unique_generated = indexer.is_unique_generated(file_format)
 
@@ -142,6 +160,7 @@ def index_file(request):
     input_path = table_metadata.path_location
   elif file_format['inputFormat'] == 'file':
     input_path = '${nameNode}%s' % file_format["path"]
+  else:
+    input_path = None
 
-  job_handle = indexer.run_morphline(request, collection_name, morphline, input_path)
-  return JsonResponse(job_handle)
+  return indexer.run_morphline(request, collection_name, morphline, input_path, query)

+ 7 - 0
desktop/libs/indexer/src/indexer/conf.py

@@ -60,6 +60,13 @@ def zkensemble():
 
 
 
+ENABLE_NEW_INDEXER = Config(
+  key="enable_new_indexer",
+  help=_t("Flag to turn on the morphline based Solr indexer."),
+  type=bool,
+  default=False
+)
+
 # Unused
 BATCH_INDEXER_PATH = Config(
   key="batch_indexer_path",

+ 2 - 1
desktop/libs/indexer/src/indexer/file_format.py

@@ -26,6 +26,7 @@ from indexer.operations import get_operator
 
 LOG = logging.getLogger(__name__)
 
+
 def get_format_types():
   return [
     CSVFormat,
@@ -280,10 +281,10 @@ class CSVFormat(FileFormat):
 
   @staticmethod
   def format_character(string):
-    string = string.replace('\\', '\\\\')
     string = string.replace('"', '\\"')
     string = string.replace('\t', '\\t')
     string = string.replace('\n', '\\n')
+    string = string.replace('\u0001', '\\u0001')
 
     return string
 

+ 40 - 37
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -17,12 +17,15 @@
 import logging
 import os
 
+from collections import deque
+
 from django.contrib.auth.models import User
 from django.utils.translation import ugettext as _
 from mako.lookup import TemplateLookup
 from mako.template import Template
 
-from collections import deque
+
+from desktop.models import Document2
 from notebook.api import _execute_notebook
 from notebook.models import make_notebook2
 from oozie.models2 import Job
@@ -34,6 +37,8 @@ from indexer.conf import CONFIG_INDEXING_TEMPLATES_PATH
 from indexer.conf import CONFIG_INDEXER_LIBS_PATH
 from indexer.conf import zkensemble
 
+from notebook.connectors.base import get_api
+
 
 LOG = logging.getLogger(__name__)
 
@@ -61,11 +66,41 @@ class Indexer(object):
 
     return hdfs_workspace_path
 
-  def run_morphline(self, request, collection_name, morphline, input_path):
+  def run_morphline(self, request, collection_name, morphline, input_path, query=None):
     workspace_path = self._upload_workspace(morphline)
 
-    snippets = [
-      {
+    snippets = []
+
+    if query:
+      from notebook.models import Notebook
+      notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=query))
+      notebook_data = notebook.get_data()
+      snippet = notebook_data['snippets'][0]
+
+      api = get_api(request, snippet)
+
+      destination = '__hue_%s' % notebook_data['uuid'][:4]
+      location = '/user/%s/__hue-%s' % (request.user,  notebook_data['uuid'][:4])
+      sql, success_url = api.export_data_as_table(notebook_data, snippet, destination, is_temporary=True, location=location)
+
+      statement = sql
+      input_path = '${nameNode}%s' % location
+
+      snippets.append({
+         'status': 'running',
+         'statement_raw': statement,
+         'statement': statement,
+         'type': 'query-hive',
+         'properties': {
+#             'files': [] if files is None else files,
+#             'functions': [] if functions is None else functions,
+#             'settings': [] if settings is None else settings
+         },
+         'database': snippet['database'],
+      }
+    )
+
+    snippets.append({
         u'type': u'java',
         u'status': u'running',
         u'properties':  {
@@ -92,7 +127,7 @@ class Indexer(object):
           u'archives': [],
         }
       }
-    ]
+    )
 
     notebook = make_notebook2(name='Indexer job for %s' % collection_name, snippets=snippets).get_data()
     snippet = {'wasBatchExecuted': True, 'type': 'oozie', 'id': notebook['snippets'][0]['id'], 'statement': ''}
@@ -102,25 +137,6 @@ class Indexer(object):
     return job_handle
 
   def guess_format(self, data):
-    """
-    Input:
-    data: {'type': 'file', 'path': '/user/hue/logs.csv'}
-    Output:
-    {'format':
-      {
-        type: 'csv',
-        fieldSeparator : ",",
-        recordSeparator: '\n',
-        quoteChar : "\""
-      },
-      'columns':
-        [
-          {name: business_id, type: string},
-          {name: cool, type: integer},
-          {name: date, type: date}
-          ]
-    }
-    """
     file_format = get_file_format_instance(data['file'])
     return file_format.get_format()
 
@@ -174,19 +190,6 @@ class Indexer(object):
     return field_type.regex.replace('\\', '\\\\')
 
   def generate_morphline_config(self, collection_name, data, uuid_name=None):
-    """
-    Input:
-    data: {
-      'type': {'name': 'My New Collection!' format': 'csv', 'columns': [{'name': business_id, 'included': True', 'type': 'string'}, cool, date], fieldSeparator : ",", recordSeparator: '\n', quoteChar : "\""},
-      'transformation': [
-        'country_code': {'replace': {'FRA': 'FR, 'CAN': 'CA'..}}
-        'ip': {'geoIP': }
-      ]
-    }
-    Output:
-    Morphline content 'SOLR_LOCATOR : { ...}'
-    """
-
     geolite_loc = os.path.join(CONFIG_INDEXER_LIBS_PATH.get(), "GeoLite2-City.mmdb")
     grok_dicts_loc = os.path.join(CONFIG_INDEXER_LIBS_PATH.get(), "grok_dictionaries")
 

+ 8 - 3
desktop/libs/indexer/src/indexer/templates/indexer.mako

@@ -367,9 +367,12 @@ ${ assist.assistPanel() }
       <!-- /ko -->
 
       <span data-bind="visible: createWizard.editorId">
-       <a href="javascript:void(0)" class="btn btn-success" data-bind="attr: {href: '${ url('notebook:editor') }?editor=' + createWizard.editorId() }" target="_blank" title="${ _('Open') }">
-         ${_('View indexing status')}
-       </a>
+        <a href="javascript:void(0)" class="btn btn-success" data-bind="attr: {href: '/oozie/list_oozie_workflow/' + createWizard.jobId() }" target="_blank" title="${ _('Open') }">
+          ${_('Oozie Status')}
+         </a>
+        <a href="javascript:void(0)" class="btn btn-success" data-bind="attr: {href: '${ url('notebook:editor') }?editor=' + createWizard.editorId() }" target="_blank" title="${ _('Open') }">
+          ${_('View indexing status')}
+        </a>
 
         ${ _('View collection') } <a href="javascript:void(0)" data-bind="attr: {href: '${ url("indexer:collections") }' +'#edit/' + createWizard.fileFormat().name() }, text: createWizard.fileFormat().name" target="_blank"></a>
       </span>
@@ -759,6 +762,7 @@ ${ assist.assistPanel() }
       self.sample = ko.observableArray();
 
       self.editorId = ko.observable();
+      self.jobId = ko.observable();
       self.editorVM = null;
 
       self.indexingStarted = ko.observable(false);
@@ -848,6 +852,7 @@ ${ assist.assistPanel() }
         }, function (resp) {
           self.showCreate(true);
           self.editorId(resp.history_id);
+          self.jobId(resp.handle.id);
           $('#notebook').html($('#notebook-progress').html());
           self.editorVM = new EditorViewModel(resp.history_uuid, '', {
             user: '${ user.username }',

+ 10 - 4
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -198,11 +198,17 @@ class Submission(object):
 
           self.job.override_subworkflow_id(action, workflow.id) # For displaying the correct graph
           self.properties['workspace_%s' % workflow.uuid] = workspace # For pointing to the correct workspace
-        elif action.data['type'] == 'hive-document':
-          from notebook.models import Notebook
-          notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid']))
+        elif action.data['type'] == 'hive-document' or action.data['type'] == 'hive2':
+          if action.data['type'] == 'hive-document' and action.data['properties'].get('uuid'):
+            from notebook.models import Notebook
+            notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid']))
+            statements = notebook.get_str()
+          else:
+            statements = action.data['properties'].get('statements')
+
+          if statements is not None:
+            self._create_file(deployment_dir, action.data['name'] + '.sql', statements)
 
-          self._create_file(deployment_dir, action.data['name'] + '.sql', notebook.get_str())
         elif action.data['type'] == 'java-document' or action.data['type'] == 'java':
           if action.data['type'] == 'java-document':
             from notebook.models import Notebook

+ 4 - 0
desktop/libs/notebook/src/notebook/api.py

@@ -666,5 +666,9 @@ def export_result(request):
     notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
     response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=insert_as_query&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
     response['status'] = 0
+  elif data_format == 'search-index':
+    notebook_id = notebook['id'] or request.GET.get('editor', request.GET.get('notebook'))
+    response['watch_url'] = reverse('notebook:execute_and_watch') + '?action=index_query&notebook=' + str(notebook_id) + '&snippet=0&destination=' + destination
+    response['status'] = 0
 
   return JsonResponse(response)

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -175,6 +175,6 @@ class Api(object):
 
   def export_data_as_hdfs_file(self, snippet, target_file, overwrite): raise NotImplementedError()
 
-  def export_data_as_table(self, notebook, snippet, destination): raise NotImplementedError()
+  def export_data_as_table(self, notebook, snippet, destination, is_temporary=False, location=None): raise NotImplementedError()
 
   def export_large_data_to_hdfs(self, notebook, snippet, destination): raise NotImplementedError()

+ 2 - 2
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -414,7 +414,7 @@ class HS2Api(Api):
     return '/filebrowser/view=%s' % target_file
 
 
-  def export_data_as_table(self, notebook, snippet, destination):
+  def export_data_as_table(self, notebook, snippet, destination, is_temporary=False, location=None):
     db = self._get_db(snippet)
 
     response = self._get_current_statement(db, snippet)
@@ -432,7 +432,7 @@ class HS2Api(Api):
 
     db.use(query.database)
 
-    hql = 'CREATE TABLE `%s`.`%s` AS %s' % (database, table, query.hql_query)
+    hql = 'CREATE %sTABLE `%s`.`%s` %sAS %s' % ('TEMPORARY ' if is_temporary else '', database, table, "LOCATION '%s' " % location if location else '', query.hql_query)
     success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
 
     return hql, success_url

+ 3 - 0
desktop/libs/notebook/src/notebook/models.py

@@ -136,6 +136,9 @@ def make_notebook2(name='Browse', description='', is_saved=False, snippets=None)
         'settings': []
     }
 
+    default_properties.update(snippet['properties'])
+    snippet['properties'] = default_properties
+
     if snippet['type'] == 'hive':
       pass
     elif snippet['type'] == 'impala':

+ 19 - 0
desktop/libs/notebook/src/notebook/templates/notebook_ko_components.mako

@@ -26,6 +26,12 @@ try:
 except ImportError, e:
   LOG.warn("Hive app is not enabled")
   DOWNLOAD_CELL_LIMIT = None
+
+try:
+  from indexer.conf import ENABLE_NEW_INDEXER
+except ImportError, e:
+  LOG.warn("Indexer app is not enabled")
+  ENABLE_NEW_INDEXER = None
 %>
 
 <%def name="snippetDbSelection()">
@@ -401,6 +407,19 @@ except ImportError, e:
                 </div>
               </div>
             </div>
+            % if ENABLE_NEW_INDEXER and ENABLE_NEW_INDEXER.get():
+            <div class="control-group">
+              <div class="controls">
+                <label class="radio">
+                  <input data-bind="checked: saveTarget" type="radio" name="save-results-type" value="search-index">
+                  &nbsp;${ _('A search dashboard') }
+                </label>
+                <div data-bind="visible: saveTarget() == 'search-index'" class="inline">
+                  <input data-bind="value: savePath" type="text" name="target_index" class="input-xlarge margin-left-10" placeholder="${_('Search index name')}">
+                </div>
+              </div>
+            </div>
+            % endif
           </fieldset>
         </form>
       </div>

+ 25 - 0
desktop/libs/notebook/src/notebook/views.py

@@ -18,7 +18,9 @@
 import json
 import logging
 
+from django.core.urlresolvers import reverse
 from django.db.models import Q
+from django.shortcuts import redirect
 from django.utils.translation import ugettext as _
 
 from desktop.conf import USE_NEW_EDITOR
@@ -148,6 +150,29 @@ def execute_and_watch(request):
   elif action == 'insert_as_query':
     sql, success_url = api.export_large_data_to_hdfs(notebook, snippet, destination)
     editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute')
+  elif action == 'index_query':
+    sql, success_url = api.export_data_as_table(notebook, snippet, destination, is_temporary=True, location='')
+    editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute')
+
+    sample = get_api(request, snippet).fetch_result(notebook, snippet, 0, start_over=True)
+
+    from indexer.api3 import _index # Will ve moved to the lib in next commit
+    from indexer.file_format import HiveFormat
+    from indexer.fields import Field
+
+    file_format = {
+        'name': 'col',
+        'inputFormat': 'query',
+        'format': {'quoteChar': '"', 'recordSeparator': '\n', 'type': 'csv', 'hasHeader': False, 'fieldSeparator': '\u0001'},
+        "sample": '',
+        "columns": [
+            Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
+            for col in sample['meta']
+        ]
+    }
+
+    job_handle = _index(request, file_format, destination, query=notebook['uuid'])
+    return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_handle['handle']['id']}))
   else:
     raise PopupException(_('Action %s is unknown') % action)