浏览代码

HUE-7 [metastore] Make create table wizard genric

Romain Rigaux 9 年之前
父节点
当前提交
1858ecd

+ 46 - 19
desktop/libs/indexer/src/indexer/api3.py

@@ -18,10 +18,9 @@
 import json
 import logging
 
+from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
 
-from beeswax.create_table import _submit_create_and_load # Beware need to protect from blacklisting
-from beeswax.server import dbms
 from desktop.lib import django_mako
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
@@ -33,11 +32,15 @@ from indexer.file_format import HiveFormat
 from indexer.fields import Field
 from indexer.smart_indexer import Indexer
 from notebook.models import make_notebook
-from notebook.api import _execute_notebook
 
 
 LOG = logging.getLogger(__name__)
 
+try:
+  from beeswax.server import dbms
+except ImportError, e:
+  LOG.warn('Hive and HiveServer2 interfaces are not enabled')
+
 
 def _escape_white_space_characters(s, inverse = False):
   MAPPINGS = {
@@ -153,6 +156,7 @@ def importer_submit(request):
   if destination['ouputFormat'] == 'index':
     _convert_format(source["format"], inverse=True)
     collection_name = source["name"]
+    source['columns'] = destination['columns']
     job_handle = _index(request, source, collection_name)
   else:
     job_handle = _create_table(request, source, destination)
@@ -163,20 +167,38 @@ def importer_submit(request):
 def _create_table(request, source, destination):
   # Create table from File  
   delim = ','
-  table_name = destination['name']
+  table_name = final_table_name = destination['name']
+  comment = 'comment'
+  external = True
   load_data = True
   skip_header = True
   database = 'default'
   path = source['path']
-
-  create_hql = django_mako.render_to_string("gen/create_table_statement.mako", {
+  table_format = 'parquet' #'text'
+
+  file_format = 'TextFile'
+  sql = ''
+  
+  # if external and non text and load_data, both bath !=
+  
+  
+  if table_format == 'parquet':
+    table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name # Or tmp table?
+    
+  if external and not request.fs.isdir(path):
+    path = request.fs.split(path)[0]
+    # If dir not empty, create data dir %(filename)_table and move file there...
+    
+    # Guess should accept a directory too
+
+  sql += django_mako.render_to_string("gen/create_table_statement.mako", {
       'table': {
           'name': table_name,
-          'comment': 'comment', # todo
+          'comment': comment,
           'row_format': 'Delimited',
           'field_terminator': delim,
-          'file_format': 'TextFile',
-          'load_data': load_data,
+          'file_format': file_format,
+          'external': external,
           'path': path, 
           'skip_header': skip_header
        },
@@ -186,19 +208,24 @@ def _create_table(request, source, destination):
     }
   )
 
-  try:
-    if load_data == 'IMPORT':
-        create_hql += "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, table_name)
+  if not external and load_data:
+    sql += "\n\nLOAD DATA INPATH '%s' INTO TABLE `%s`.`%s`;" % (path, database, table_name)
 
-    #on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
+  if table_format == 'parquet':
+    sql += '\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s` STORED AS %(file_format)s AS SELECT * FROM `%(database)s`.`%(table_name)s`;' % {
+        'database': database,
+        'final_table_name': final_table_name,
+        'table_name': table_name,
+        'file_format': table_format
+    }
+    sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;' % {'database': database, 'table_name': table_name}
 
-    #query = hql_query(create_hql, database=database)
+  try:
     editor_type = 'hive'
-    notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=create_hql, status='ready', database=database)
-    #_execute_notebook(request, notebook, snippet)
-    handle = notebook.execute(request) #, batch=True)
-    print handle
-    return handle
+    # on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
+    notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready', database=database)
+
+    return notebook.execute(request, batch=False)
   except Exception, e:
     raise PopupException(_('The table could not be created.'), detail=e.message)
 

+ 3 - 2
desktop/libs/indexer/src/indexer/templates/gen/create_table_statement.mako

@@ -46,7 +46,7 @@ COMMENT "${col["comment"]|n}" \
 </%def>\
 #########################
 CREATE \
-% if table.get("load_data", "IMPORT") == 'EXTERNAL':
+% if table.get("external", False):
 EXTERNAL \
 % endif
 TABLE `${ '%s.%s' % (database, table["name"]) | n }`
@@ -85,9 +85,10 @@ ROW FORMAT \
 % if table.get("file_format") == "InputFormat":
 INPUTFORMAT ${table["input_format_class"] | n} OUTPUTFORMAT ${table["output_format_class"] | n}
 % endif
-% if table.get("load_data", "IMPORT") == 'EXTERNAL':
+% if table.get("external", False):
 LOCATION "${table["path"] | n}"
 % endif
 % if table.get("skip_header", False):
 TBLPROPERTIES("skip.header.line.count" = "1")
 % endif
+;

+ 22 - 18
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -33,6 +33,8 @@ ${ commonheader(_("Solr Indexes"), "search", user, request, "60px") | n,unicode
 <script src="${ static('desktop/js/sqlAutocompleter2.js') }"></script>
 <script src="${ static('desktop/js/hdfsAutocompleter.js') }"></script>
 <script src="${ static('desktop/js/autocompleter.js') }"></script>
+<script src="${ static('desktop/js/hue.json.js') }"></script>
+
 
 <script src="${ static('desktop/js/jquery.hiveautocomplete.js') }" type="text/javascript" charset="utf-8"></script>
 <script src="${ static('desktop/ext/js/jquery/plugins/jquery-ui-1.10.4.custom.min.js') }"></script>
@@ -213,7 +215,7 @@ ${ assist.assistPanel() }
           <span class="fa fa-check"></span>
           <!-- /ko -->
         </div>
-        <div class="caption">${ _('Pick data from') }</div>
+        <div class="caption">${ _('Pick data from ') }<span data-bind="text: createWizard.source.inputFormat"></span></div>
       </li>
 
       <li data-bind="css: { 'inactive': currentStep() == 1, 'active': currentStep() == 2, 'complete': currentStep() == 3 }, click: function() { currentStep(2) }">
@@ -227,7 +229,7 @@ ${ assist.assistPanel() }
             <!-- /ko -->
           <!-- /ko -->
         </div>
-        <div class="caption">${ _('Move it to') }</div>
+        <div class="caption">${ _('Move it to ') }<span data-bind="text: createWizard.destination.ouputFormat"></span></div>
       </li>
     </ol>
 
@@ -249,9 +251,6 @@ ${ assist.assistPanel() }
               <label for="path" class="control-label"><div>${ _('Path') }</div>
                 <input type="text" class="form-control path input-xxlarge" data-bind="value: createWizard.source.path, filechooser: createWizard.source.path, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }">
               </label>
-              <label class="checkbox">
-                <input type="checkbox" checked -bind=""> ${_('Import')}
-              </label>
             </div>
 
             <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'table'">
@@ -267,18 +266,16 @@ ${ assist.assistPanel() }
             </div>
           </div>
 
+          <!-- ko if: createWizard.source.show -->
           <h3 class="card-heading simple">${_('Format')}</h3>
           <div class="card-body">
-            <form class="form-inline">
               <label>${_('File Type')} <select data-bind="options: $root.createWizard.fileTypes, optionsText: 'description', value: $root.createWizard.fileType"></select></label>
 
               <span data-bind="with: createWizard.source.format, visible: createWizard.source.show">
-                <!-- ko template: {name: 'format-settings'}--><!-- /ko -->
+                <!-- ko template: {name: 'format-settings'} --> <!-- /ko -->
               </span>
-            </form>
           </div>
 
-          <!-- if: createWizard.source.sampleCols -->
           <h3 class="card-heading simple">${_('Preview')}</h3>
           <div class="card-body">
             <!-- ko if: createWizard.isGuessingFieldTypes -->
@@ -346,18 +343,21 @@ ${ assist.assistPanel() }
           <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
 
           <label class="checkbox">
-            <input type="checkbox"> ${_('External loc')}
+            <input type="checkbox" checked -bind=""> ${_('Import data')}
+          </label>
+          <label class="checkbox">
+            <input type="checkbox" checked> ${_('Default location')}
           </label>
           <label for="path" class="control-label"><div>${ _('Path') }</div>
             <input type="text" class="form-control path input-xxlarge" data-bind="value: createWizard.source.path, filechooser: createWizard.source.path, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }">
           </label>
           <label class="checkbox">
-            <input type="checkbox"> ${_('Delimiters')}
+            <input type="checkbox"> ${_('Custom delimiters')}
           </label>
           ## field, coll map delimieters
 
           <label class="checkbox">
-              <input type="checkbox" checked> ${_('Has headers')}
+              <input type="checkbox" checked> ${_('Use headers')}
             </label>
           <label class="checkbox">
               <input type="checkbox" checked> ${_('Bulk edit col names')}
@@ -445,6 +445,7 @@ ${ assist.assistPanel() }
   </span>
 
   <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Nested')}</a>
+  <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
   <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Comment')}</a>
 </script>
 
@@ -487,7 +488,7 @@ ${ assist.assistPanel() }
 
 <script type="text/html" id="operation-template">
   <div class="operation">
-    <select data-bind="options: $root.createWizard.operationTypes.map(function(o){return o.name});, value: operation.type"></select>
+    <select data-bind="options: $root.createWizard.operationTypes.map(function(o){return o.name}), value: operation.type"></select>
     <!-- ko template: "args-template" --><!-- /ko -->
     <!-- ko if: operation.settings().outputType() == "custom_fields" -->
       <label> ${ _('Number of expected fields') }
@@ -737,7 +738,7 @@ ${ assist.assistPanel() }
           self.getDocuments();
         }
       });
-      self.inputFormats = ko.observableArray(['file', 'table', 'query', 'manual']);
+      self.inputFormats = ko.observableArray(['file', 'text', 'table', 'query', 'dbms', 'nothing']);
 
       // File
       self.path = ko.observable('');
@@ -809,7 +810,7 @@ ${ assist.assistPanel() }
             name = self.path().split('/').pop().split('.')[0];
           }
         } else if (self.inputFormat() == 'table') {
-          if (val && self.table().split('.', 2).length == 2) {
+          if (self.table().split('.', 2).length == 2) {
             name = self.tableName();
           }
         } else if (self.inputFormat() == 'query') {
@@ -831,14 +832,14 @@ ${ assist.assistPanel() }
       self.name = ko.observable('');
 
       self.ouputFormat = ko.observable('table');
-      self.ouputFormats = ko.observableArray(['table', 'index']);
+      self.ouputFormats = ko.observableArray(['table', 'index', 'file']);
 
       self.format = ko.observable();
       self.columns = ko.observableArray();
 
       // Table
       self.tableFormat = ko.observable('text');
-      self.tableFormats = ko.observableArray(['text', 'parquet', 'json', 'kudu']);
+      self.tableFormats = ko.observableArray(['text', 'parquet', 'json', 'orc', 'kudu']);
       self.hasHeader = ko.observable(false); // ?
       self.bulkEditColumns = ko.observable(false);
       self.partitionColumns = ko.observableArray();
@@ -919,7 +920,6 @@ ${ assist.assistPanel() }
 
           self.isGuessingFormat(false);
           viewModel.wizardEnabled(true);
-          //viewModel.currentStep(2);
         }).fail(function (xhr, textStatus, errorThrown) {
           $(document).trigger("error", xhr.responseText);
           viewModel.isLoading(false);
@@ -973,6 +973,7 @@ ${ assist.assistPanel() }
           self.editorId(resp.history_id);
           self.jobId(resp.handle.id);
           $('#notebook').html($('#notebook-progress').html());
+
           self.editorVM = new EditorViewModel(resp.history_uuid, '', {
             user: '${ user.username }',
             userId: ${ user.id },
@@ -989,8 +990,10 @@ ${ assist.assistPanel() }
               }
             }
           });
+          self.editorVM.editorMode(true);
           ko.cleanNode($("#notebook")[0]);
           ko.applyBindings(self.editorVM, $("#notebook")[0]);
+
           self.editorVM.openNotebook(resp.history_uuid, null, true, function(){
             self.editorVM.selectedNotebook().snippets()[0].progress.subscribe(function(val){
               if (val == 100){
@@ -1006,6 +1009,7 @@ ${ assist.assistPanel() }
                 self.indexingError(true);
               }
             });
+            self.editorVM.selectedNotebook().snippets()[0].checkStatus();
           });
           viewModel.isLoading(false);
         }).fail(function (xhr, textStatus, errorThrown) {

+ 2 - 3
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -32,11 +32,10 @@ from desktop.lib.exceptions import StructuredException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 from desktop.lib.rest.http_client import RestException
-from desktop.models import DefaultConfiguration, Document2
+from desktop.models import DefaultConfiguration
 from metadata.optimizer_client import OptimizerApi
 
-from notebook.connectors.base import Api, QueryError, QueryExpired, OperationTimeout, OperationNotSupported,\
-  Notebook
+from notebook.connectors.base import Api, QueryError, QueryExpired, OperationTimeout, OperationNotSupported
 
 
 LOG = logging.getLogger(__name__)