Browse Source

HUE-7 [metastore] Skeleton of create table wizard redesign backend

Romain Rigaux 9 years ago
parent
commit
842ac35

+ 64 - 3
desktop/libs/indexer/src/indexer/api3.py

@@ -20,7 +20,9 @@ import logging
 
 from django.utils.translation import ugettext as _
 
+from beeswax.create_table import _submit_create_and_load # Beware need to protect from blacklisting
 from beeswax.server import dbms
+from desktop.lib import django_mako
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document2
@@ -30,6 +32,8 @@ from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
 from indexer.fields import Field
 from indexer.smart_indexer import Indexer
+from notebook.models import make_notebook
+from notebook.api import _execute_notebook
 
 
 LOG = logging.getLogger(__name__)
@@ -65,11 +69,11 @@ def guess_format(request):
     indexer = Indexer(request.user, request.fs)
     stream = request.fs.open(file_format["path"])
     format_ = indexer.guess_format({
-      "file":{
+      "file": {
         "stream": stream,
         "name": file_format['path']
-        }
-      })
+      }
+    })
     _convert_format(format_)
   elif file_format['inputFormat'] == 'table':
     db = dbms.get(request.user)
@@ -142,6 +146,63 @@ def index_file(request):
   return JsonResponse(job_handle)
 
 
+def importer_submit(request):
+  source = json.loads(request.POST.get('source', '{}'))
+  destination = json.loads(request.POST.get('destination', '{}'))
+
+  if destination['ouputFormat'] == 'index':
+    _convert_format(source["format"], inverse=True)
+    collection_name = source["name"]
+    job_handle = _index(request, source, collection_name)
+  else:
+    job_handle = _create_table(request, source, destination)
+
+  return JsonResponse(job_handle)
+
+
+def _create_table(request, source, destination):
+  # Create table from File  
+  delim = ','
+  table_name = destination['name']
+  load_data = True
+  skip_header = True
+  database = 'default'
+  path = source['path']
+
+  create_hql = django_mako.render_to_string("gen/create_table_statement.mako", {
+      'table': {
+          'name': table_name,
+          'comment': 'comment', # todo
+          'row_format': 'Delimited',
+          'field_terminator': delim,
+          'file_format': 'TextFile',
+          'load_data': load_data,
+          'path': path, 
+          'skip_header': skip_header
+       },
+      'columns': destination['columns'],
+      'partition_columns': [],
+      'database': database
+    }
+  )
+
+  try:
+    if load_data == 'IMPORT':
+        create_hql += "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, table_name)
+
+    #on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
+
+    #query = hql_query(create_hql, database=database)
+    editor_type = 'hive'
+    notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=create_hql, status='ready', database=database)
+    #_execute_notebook(request, notebook, snippet)
+    handle = notebook.execute(request) #, batch=True)
+    print handle
+    return handle
+  except Exception, e:
+    raise PopupException(_('The table could not be created.'), detail=e.message)
+
+
 def _index(request, file_format, collection_name, query=None):
   indexer = Indexer(request.user, request.fs)
 

+ 93 - 0
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -0,0 +1,93 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+def col_type(col):
+  if col["type"] == "array":
+    return "array <%s>" % col["array_type"]
+  elif col["type"] == "map":
+    return "map <%s, %s>" % (col["map_key_type"], col["map_value_type"])
+  elif col["type"] == "char":
+    return "char(%d)" % col["char_length"]
+  elif col["type"] == "varchar":
+    return "varchar(%d)" % col["varchar_length"]
+  return col["type"]
+%>\
+
+<%def name="column_list(columns)">\
+## Returns (foo int, bar string)-like data for columns
+(
+<% first = True %>\
+% for col in columns:
+%   if first:
+<% first = False %>\
+%   else:
+,
+%   endif
+  `${col["name"]|n}` ${col_type(col)|n} \
+%   if col.get("comment"):
+COMMENT "${col["comment"]|n}" \
+%   endif
+% endfor
+) \
+</%def>\
+#########################
+CREATE \
+% if table.get("load_data", "IMPORT") == 'EXTERNAL':
+EXTERNAL \
+% endif
+TABLE `${ '%s.%s' % (database, table["name"]) | n }`
+${column_list(columns)|n}
+% if table["comment"]:
+COMMENT "${table["comment"] | n}"
+% endif
+% if len(partition_columns) > 0:
+PARTITIONED BY ${column_list(partition_columns)|n}
+% endif
+## TODO: CLUSTERED BY here
+## TODO: SORTED BY...INTO...BUCKETS here
+ROW FORMAT \
+% if table.has_key('row_format'):
+%   if table["row_format"] == "Delimited":
+  DELIMITED
+%     if table.has_key('field_terminator'):
+    FIELDS TERMINATED BY '${table["field_terminator"] | n}'
+%     endif
+%     if table.has_key('collection_terminator'):
+    COLLECTION ITEMS TERMINATED BY '${table["collection_terminator"] | n}'
+%     endif
+%     if table.has_key('map_key_terminator'):
+    MAP KEYS TERMINATED BY '${table["map_key_terminator"] | n}'
+%     endif
+%   else:
+  SERDE ${table["serde_name"] | n}
+%     if table["serde_properties"]:
+  WITH SERDEPROPERTIES ${table["serde_properties"] | n}
+%     endif
+%   endif
+% endif
+% if table.has_key('file_format'):
+  STORED AS ${table["file_format"] | n} \
+% endif
+% if table.get("file_format") == "InputFormat":
+INPUTFORMAT ${table["input_format_class"] | n} OUTPUTFORMAT ${table["output_format_class"] | n}
+% endif
+% if table.get("load_data", "IMPORT") == 'EXTERNAL':
+LOCATION "${table["path"] | n}"
+% endif
+% if table.get("skip_header", False):
+TBLPROPERTIES("skip.header.line.count" = "1")
+% endif

+ 34 - 18
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -27,6 +27,13 @@ ${ commonheader(_("Solr Indexes"), "search", user, request, "60px") | n,unicode
 
 <span class="notebook">
 
+# Todo lot of those
+<script src="${ static('desktop/js/autocomplete/sql.js') }"></script>
+<script src="${ static('desktop/js/sqlAutocompleter.js') }"></script>
+<script src="${ static('desktop/js/sqlAutocompleter2.js') }"></script>
+<script src="${ static('desktop/js/hdfsAutocompleter.js') }"></script>
+<script src="${ static('desktop/js/autocompleter.js') }"></script>
+
 <script src="${ static('desktop/js/jquery.hiveautocomplete.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/ext/js/jquery/plugins/jquery-ui-1.10.4.custom.min.js') }"></script>
 <script src="${ static('desktop/js/jquery.huedatatable.js') }"></script>
@@ -271,6 +278,7 @@ ${ assist.assistPanel() }
             </form>
           </div>
 
+          <!-- if: createWizard.source.sampleCols -->
           <h3 class="card-heading simple">${_('Preview')}</h3>
           <div class="card-body">
             <!-- ko if: createWizard.isGuessingFieldTypes -->
@@ -300,6 +308,7 @@ ${ assist.assistPanel() }
               </table>
             </div>
           </div>
+          <!-- /ko -->
 
         </form>
       </div>
@@ -334,7 +343,7 @@ ${ assist.assistPanel() }
         <!-- /ko -->
 
         <!-- ko if: $root.createWizard.destination.ouputFormat() == 'table' -->
-          <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="value: createWizard.destination.name, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
+          <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
 
           <label class="checkbox">
             <input type="checkbox"> ${_('External loc')}
@@ -389,17 +398,14 @@ ${ assist.assistPanel() }
       <!-- /ko -->
 
       <!-- ko if: currentStep() == 2 -->
-        <button href="javascript:void(0)" class="btn btn-primary disable-feedback" data-bind="click: createWizard.indexFile, enable: createWizard.readyToIndex() && !createWizard.indexingStarted()">
-          ${_('Create')} <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.indexingStarted"></i>
+        <button href="javascript:void(0)" class="btn btn-primary disable-feedback" data-bind="click: createWizard.indexFile, enable: createWizard.readyToIndex() && ! createWizard.indexingStarted()">
+          ${ _('Submit') } <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.indexingStarted"></i>
         </button>
       <!-- /ko -->
 
       <span data-bind="visible: createWizard.editorId">
-        <a href="javascript:void(0)" class="btn btn-success" data-bind="attr: {href: '/oozie/list_oozie_workflow/' + createWizard.jobId() }" target="_blank" title="${ _('Open') }">
-          ${_('Oozie Status')}
-         </a>
         <a href="javascript:void(0)" class="btn btn-success" data-bind="attr: {href: '${ url('notebook:editor') }?editor=' + createWizard.editorId() }" target="_blank" title="${ _('Open') }">
-          ${_('View indexing status')}
+          ${_('Status')}
         </a>
 
         ${ _('View collection') } <a href="javascript:void(0)" data-bind="attr: {href: '${ url("indexer:collections") }' +'#edit/' + createWizard.source.name() }, text: createWizard.source.name" target="_blank"></a>
@@ -423,7 +429,7 @@ ${ assist.assistPanel() }
     <input type="text" class="input-large" placeholder="${ _('Field name') }" data-bind="value: name">
   </label>
   <label>${ _('Type') }
-    <select class="input-small" data-bind="options: $root.createWizard.fieldTypes, value: type"></select>
+    <select class="input-small" data-bind="options: $root.createWizard.hiveFieldTypes, value: type"></select>
   </label>
 
   <label data-bind="text: $root.createWizard.source.sample()[0][$index()]"></label>
@@ -439,6 +445,7 @@ ${ assist.assistPanel() }
   </span>
 
   <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Nested')}</a>
+  <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Comment')}</a>
 </script>
 
 
@@ -552,7 +559,7 @@ ${ assist.assistPanel() }
         'progress-warning': progress() > 0 && progress() < 100,
         'progress-success': progress() == 100,
         'progress-danger': progress() == 0 && errors().length > 0}" style="background-color: #FFF; width: 100%; height: 4px">
-        <div class="bar" data-bind="style: {'width': (errors().length > 0 ? 100 : Math.max(2,progress())) + '%'}"></div>
+        <div class="bar" data-bind="style: {'width': (errors().length > 0 ? 100 : Math.max(2, progress())) + '%'}"></div>
       </div>
     <!-- /ko -->
   <!-- /ko -->
@@ -799,7 +806,7 @@ ${ assist.assistPanel() }
 
         if (self.inputFormat() == 'file') {
           if (self.path()) {
-            name = self.path().split('/').pop();
+            name = self.path().split('/').pop().split('.')[0];
           }
         } else if (self.inputFormat() == 'table') {
           if (val && self.table().split('.', 2).length == 2) {
@@ -811,7 +818,7 @@ ${ assist.assistPanel() }
           }
         }
 
-        return name.replace(' ', '_') + '_dashboard';
+        return name.replace(' ', '_');
       });
       self.defaultName.subscribe(function(newVal) {
         vm.createWizard.destination.name(newVal);
@@ -852,7 +859,8 @@ ${ assist.assistPanel() }
 
       self.operationTypes = ${operators_json | n};
 
-      self.fieldTypes = ${fields_json | n};
+      self.fieldTypes = ${fields_json | n}.solr;
+      self.hiveFieldTypes = ${fields_json | n}.hive;
       self.fileTypes = ${file_types_json | n};
 
 
@@ -875,8 +883,7 @@ ${ assist.assistPanel() }
 
       self.readyToIndex = ko.computed(function () {
         var validFields = self.destination.columns().length;
-
-        return self.source.name().length > 0 && validFields;
+        return self.destination.name().length > 0 && validFields;
       });
 
       self.source.format.subscribe(function () {
@@ -950,14 +957,17 @@ ${ assist.assistPanel() }
       self.indexingError = ko.observable(false);
       self.indexingSuccess = ko.observable(false);
       self.indexFile = function () {
-        if (!self.readyToIndex()) return;
+        if (! self.readyToIndex()) {
+          return;
+        }
 
         self.indexingStarted(true);
         viewModel.isLoading(true);
         self.isIndexing(true);
 
-        $.post("${ url('indexer:index_file') }", {
-          "fileFormat": ko.mapping.toJSON(self.source)
+        $.post("${ url('indexer:importer_submit') }", {
+          "source": ko.mapping.toJSON(self.source),
+          "destination": ko.mapping.toJSON(self.destination)
         }, function (resp) {
           self.showCreate(true);
           self.editorId(resp.history_id);
@@ -966,10 +976,16 @@ ${ assist.assistPanel() }
           self.editorVM = new EditorViewModel(resp.history_uuid, '', {
             user: '${ user.username }',
             userId: ${ user.id },
-            languages: [{name: "Java SQL", type: "java"}],
+            languages: [{name: "Java", type: "java"}, {name: "Hive SQL", type: "hive"}], // TODO reuse
             snippetViewSettings: {
               java : {
                 snippetIcon: 'fa-file-archive-o '
+              },
+              hive: {
+                placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
+                aceMode: 'ace/mode/hive',
+                snippetImage: '${ static("beeswax/art/icon_beeswax_48.png") }',
+                sqlDialect: true
               }
             }
           });

+ 3 - 1
desktop/libs/indexer/src/indexer/urls.py

@@ -26,7 +26,7 @@ urlpatterns = patterns('indexer.views',
 
   # V3
   url(r'^indexer/$', 'indexer', name='indexer'),
-  url(r'^importer/$', 'importer', name='importer')
+  url(r'^importer/$', 'importer', name='importer'),
 )
 
 urlpatterns += patterns('indexer.api',
@@ -57,4 +57,6 @@ urlpatterns += patterns('indexer.api3',
   url(r'^api/indexer/guess_format/$', 'guess_format', name='guess_format'),
   url(r'^api/indexer/index_file/$', 'index_file', name='index_file'),
   url(r'^api/indexer/guess_field_types/$', 'guess_field_types', name='guess_field_types'),
+
+  url(r'^api/importer/submit', 'importer_submit', name='importer_submit')
 )

+ 6 - 1
desktop/libs/indexer/src/indexer/views.py

@@ -65,6 +65,11 @@ def indexer(request):
       'default_field_type' : json.dumps(Field().to_dict())
   })
 
+HIVE_PRIMITIVE_TYPES = \
+    ("string", "tinyint", "smallint", "int", "bigint", "boolean",
+      "float", "double", "timestamp", "date", "char", "varchar")
+HIVE_TYPES = HIVE_PRIMITIVE_TYPES + ("array", "map", "struct")
+
 
 def importer(request):
   searcher = IndexController(request.user)
@@ -75,7 +80,7 @@ def importer(request):
 
   return render('importer.mako', request, {
       'indexes_json': json.dumps(indexes),
-      'fields_json' : json.dumps([field.name for field in FIELD_TYPES]),
+      'fields_json' : json.dumps({'solr': [field.name for field in FIELD_TYPES], 'hive': HIVE_PRIMITIVE_TYPES}),
       'operators_json' : json.dumps([operator.to_dict() for operator in OPERATORS]),
       'file_types_json' : json.dumps([format_.format_info() for format_ in get_file_indexable_format_types()]),
       'default_field_type' : json.dumps(Field().to_dict())

+ 1 - 1
desktop/libs/notebook/src/notebook/api.py

@@ -438,7 +438,7 @@ def get_history(request):
         'type': doc.type,
         'data': {
             'statement': statement[:1001] if statement else '',
-            'lastExecuted': notebook['snippets'][0]['lastExecuted'],
+            'lastExecuted': notebook['snippets'][0].get('lastExecuted', -1),
             'status':  notebook['snippets'][0]['status'],
             'parentSavedQueryUuid': notebook.get('parentSavedQueryUuid', '')
         } if notebook['snippets'] else {},

+ 3 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -179,7 +179,9 @@ class Notebook(object):
     from notebook.api import _execute_notebook # Cyclic dependency
 
     notebook_data = self.get_data()
-    snippet = {'wasBatchExecuted': batch, 'type': 'oozie', 'id': notebook_data['snippets'][0]['id'], 'statement': ''}
+    #snippet = {'wasBatchExecuted': batch, 'type': 'oozie', 'id': notebook_data['snippets'][0]['id'], 'statement': ''}
+    snippet = notebook_data['snippets'][0]
+    snippet['wasBatchExecuted'] = batch
 
     return _execute_notebook(request, notebook_data, snippet)
 

+ 2 - 2
desktop/libs/notebook/src/notebook/models.py

@@ -110,7 +110,7 @@ def make_notebook(name='Browse', description='', editor_type='hive', statement='
          },
          'name': name,
          'database': database,
-         'result': {},
+         'result': {'handle':{}},
          'variables': []
       }
     ]
@@ -169,7 +169,7 @@ def make_notebook2(name='Browse', description='', is_saved=False, snippets=None)
          'properties': _snippet['properties'],
          'name': name,
          'database': _snippet.get('database'),
-         'result': {},
+         'result': {'handle':{}},
          'variables': []
       } for _snippet in _snippets
     ]