Browse Source

HUE-7 [metastore] Connect frontend to backend for table creation wizard

Romain Rigaux 9 years ago
parent
commit
a73e10e

+ 1 - 0
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -1032,6 +1032,7 @@ class HiveServerTableCompatible(HiveServerTable):
 
     self.describe = HiveServerTTableSchema(self.desc_results, self.desc_schema).cols()
     self._details = None
+    self.is_impala_only = False
 
   @property
   def cols(self):

+ 3 - 2
desktop/core/src/desktop/templates/responsive.mako

@@ -298,8 +298,9 @@ ${ hueIcons.symbols() }
         <li><a data-bind="click: function () { onePageViewModel.changeEditorType('hive'); onePageViewModel.currentApp('editor') }">Hive</a></li>
         <li><a data-bind="click: function () { onePageViewModel.changeEditorType('impala'); onePageViewModel.currentApp('editor') }">Impala</a></li>
         <li><a data-bind="click: function () { onePageViewModel.currentApp('search') }">Dashboard</a></li>
-        <li><a href="javascript: void(0);">Report</a></li>
+        <li><a data-bind="click: function () { onePageViewModel.currentApp('notebook') }"Report</a></li>
         <li><a data-bind="click: function () { onePageViewModel.currentApp('oozie_workflow') }">Oozie</a></li>
+        <li><a href="/indexer/importer/">Importer</a></li>
         <li><a href="javascript: void(0);">Custom App 1</a></li>
         <li><a href="javascript: void(0);">Custom App 2</a></li>
         <li><a href="javascript: void(0);">Custom App 3</a></li>
@@ -605,7 +606,7 @@ ${ assist.assistPanel() }
           self.currentApp(window.location.getParameter('app'));
         }
         else {
-          self.currentApp('home');
+          self.currentApp('hive');
         }
 
         huePubSub.subscribe('switch.app', function (name) {

+ 60 - 29
desktop/libs/indexer/src/indexer/api3.py

@@ -155,7 +155,7 @@ def importer_submit(request):
 
   if destination['ouputFormat'] == 'index':
     _convert_format(source["format"], inverse=True)
-    collection_name = source["name"]
+    collection_name = destination["name"]
     source['columns'] = destination['columns']
     job_handle = _index(request, source, collection_name)
   else:
@@ -165,44 +165,72 @@ def importer_submit(request):
 
 
 def _create_table(request, source, destination):
-  # Create table from File  
-  delim = ','
+  return _create_table_from_a_file(request, source, destination)
+
+
+def _create_table_from_a_file(request, source, destination):
   table_name = final_table_name = destination['name']
-  comment = 'comment'
-  external = True
-  load_data = True
-  skip_header = True
-  database = 'default'
-  path = source['path']
   table_format = destination['tableFormat']
+
+  database = destination['database']
   columns = destination['columns']
-  primary_keys = ['id']
+  partition_columns = destination['partitionColumns']
+
+  comment = destination['description']
+
+  external = not destination['useDefaultLocation']
+  external_path = not destination['nonDefaultLocation']
+
+  load_data = destination['importData']
+  skip_header = destination['hasHeader']
+
+  primary_keys = destination['primaryKeys']
+
+  if destination['useCustomDelimiters']:
+    field_delimiter = destination['customFieldDelimiter']
+    collection_delimiter = destination['customCollectionDelimiter']
+    map_delimiter = destination['customMapDelimiter']
+    regexp_delimiter = destination['customRegexp']
+  else:
+    field_delimiter = ','
+    collection_delimiter = r'\\002'
+    map_delimiter = r'\\003'
+    regexp_delimiter = '.*'
+
+  source_path = source['path']
+
 
   file_format = 'TextFile'
+  extra_create_properties = ''
   sql = ''
-  
+
   # if external and non text and load_data, both bath !=
-  
+
   if load_data:
     if table_format in ('parquet', 'kudu'):
       table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name # Or tmp table?
-    
-  if external and not request.fs.isdir(path):
-    path = request.fs.split(path)[0]
+
+  if external:
+    if not request.fs.isdir(source_path): # File selected
+      source_path = request.fs.split(source_path)[0]
     # If dir not empty, create data dir %(filename)_table and move file there...
-    
+
     # Guess format should accept a directory too
   # Kudu external table has extra prop
+  sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
+      'database': database,
+      'table_name': table_name
+  }
 
   sql += django_mako.render_to_string("gen/create_table_statement.mako", {
       'table': {
           'name': table_name,
           'comment': comment,
           'row_format': 'Delimited',
-          'field_terminator': delim,
+          'field_terminator': field_delimiter,
           'file_format': file_format,
           'external': external,
-          'path': path, 
+          'path': source_path,
           'skip_header': skip_header,
           'primary_keys': primary_keys if table_format == 'kudu' and not load_data else []
        },
@@ -213,31 +241,34 @@ def _create_table(request, source, destination):
   )
 
   if table_format == 'text' and not external and load_data:
-    sql += "\n\nLOAD DATA INPATH '%s' INTO TABLE `%s`.`%s`;" % (path, database, table_name)
+    sql += "\n\nLOAD DATA INPATH '%s' INTO TABLE `%s`.`%s`;" % (source_path, database, table_name)
 
   if table_format in ('parquet', 'kudu'):
     file_format = table_format
     if table_format == 'kudu':
-      columns_list = primary_keys + [col['name'] for col in destination['columns'] if col['name'] not in primary_keys]
-    else:
-      columns_list = ['*']
-    sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`
-      PRIMARY KEY (%(primary_keys)s)
+      columns_list = ['`%s`' % col for col in primary_keys + [col['name'] for col in destination['columns'] if col['name'] not in primary_keys]]
+      extra_create_properties = """PRIMARY KEY (%(primary_keys)s)
       DISTRIBUTE BY HASH INTO 16 BUCKETS
       STORED AS %(file_format)s
       TBLPROPERTIES(
       'kudu.num_tablet_replicas' = '1'
-      )
+      )""" % {
+        'file_format': file_format,
+        'primary_keys': ', '.join(primary_keys)
+      }
+    else:
+      columns_list = ['*']
+    sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`
+      %(extra_create_properties)s
       AS SELECT %(columns_list)s
       FROM `%(database)s`.`%(table_name)s`;''' % {
         'database': database,
         'final_table_name': final_table_name,
         'table_name': table_name,
-        'file_format': file_format,
-        'columns_list': ', '.join(['`%s`' % col for col in columns_list]),
-        'primary_keys': ', '.join(primary_keys)
+        'extra_create_properties': extra_create_properties,
+        'columns_list': ', '.join(columns_list),
     }
-    sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;' % {
+    sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
         'database': database,
         'table_name': table_name
     }

+ 1 - 0
desktop/libs/indexer/src/indexer/conf.py

@@ -67,6 +67,7 @@ ENABLE_NEW_INDEXER = Config(
   default=False
 )
 
+
 # Unused
 BATCH_INDEXER_PATH = Config(
   key="batch_indexer_path",

+ 116 - 40
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -243,7 +243,7 @@ ${ assist.assistPanel() }
           <div>
             <div class="control-group">
               <label for="collectionType" class="control-label"><div>${ _('Type') }</div>
-                <select id="collectionType" data-bind="options: createWizard.source.inputFormats, value: createWizard.source.inputFormat"></select>
+                <select id="collectionType" data-bind="options: createWizard.source.inputFormats, value: createWizard.source.inputFormat, optionsText: 'name', optionsValue: 'value'"></select>
               </label>
             </div>
 
@@ -316,73 +316,106 @@ ${ assist.assistPanel() }
     <!-- ko if: currentStep() == 2 -->
     <div class="card step2">
 
+      <!-- ko with: createWizard.destination -->
       <div class="control-group">
         <label for="collectionType" class="control-label"><div>${ _('Type') }</div>
-          <select id="collectionType" data-bind="options: createWizard.destination.ouputFormats, value: createWizard.destination.ouputFormat"></select>
+          <select id="collectionType" data-bind="options: ouputFormats, value: ouputFormat, optionsValue: 'value', optionsText: 'name'"></select>
         </label>
-      </div>
 
-      <div class="control-group>
         <label for="collectionName" class="control-label"><div>${ _('Name') }</div>
-          <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="value: createWizard.destination.name, valueUpdate: 'afterkeydown'" placeholder="${ _('Collection name') }">
-          <span class="help-inline muted" data-bind="visible: createWizard.isNameAvailable()">${ _('A new collection will be created') }</span>
-          <span class="help-inline muted" data-bind="visible: ! createWizard.isNameAvailable() && createWizard.destination.name().length > 0">
-          ${ _('Adding data to this existing collection') }
-          <a href="javascript:void(0)" data-bind="attr: {href: '${ url("indexer:collections") }' +'#edit/' + createWizard.destination.name() }, text: createWizard.destination.name" target="_blank"></a>
+          <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="value: name, valueUpdate: 'afterkeydown'" placeholder="${ _('Name') }">
+          <span class="help-inline muted" data-bind="visible: $root.createWizard.isNameAvailable()">
+            ${ _('Create a new ') } <span data-bind="text: ouputFormat"></span>
+          </span>
+          <span class="help-inline muted" data-bind="visible: ! $root.createWizard.isNameAvailable() && name().length > 0">
+            ${ _('Adding data to this existing ') } <span data-bind="text: ouputFormat"></span>
+            <a href="javascript:void(0)" data-bind="attr: {href: '${ url("indexer:collections") }' +'#edit/' + name() }, text: name" target="_blank"></a>
           </span>
         </label>
       </div>
 
-      <h3 class="card-heading simple">${_('Fields')}</h3>
       <div class="card-body">
-        <!-- ko if: createWizard.isGuessingFieldTypes -->
-          <i class="fa fa-spinner fa-spin"></i>
-        <!-- /ko -->
+        ##<!-- ko if: createWizard.isGuessingFieldTypes -->
+        ##  <i class="fa fa-spinner fa-spin"></i>
+        ##<!-- /ko -->
 
-        <!-- ko if: $root.createWizard.destination.ouputFormat() == 'table' -->
-          <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
+        <!-- ko if: ouputFormat() == 'table' -->
+        <h3 class="card-heading simple">${_('Properties')}</h3>
+          <input type="text" class="form-control input-xlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
 
+          <div class="control-group">
+            <label for="collectionType" class="control-label"><div>${ _('Format') }</div>
+              <select id="collectionType" data-bind="options: tableFormats, value: tableFormat, optionsValue: 'value', optionsText: 'name'"></select>
+            </label>
+          </div>
+
+          ${_('Database')}
           <label class="checkbox">
-            <input type="checkbox" checked -bind=""> ${_('Import data')}
-          </label>
-          <label class="checkbox">
-            <input type="checkbox" checked> ${_('Default location')}
-          </label>
-          <label for="path" class="control-label"><div>${ _('Path') }</div>
-            <input type="text" class="form-control path input-xxlarge" data-bind="value: createWizard.source.path, filechooser: createWizard.source.path, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }">
+            <input type="text" data-bind="value: database">
           </label>
+
+
           <label class="checkbox">
-            <input type="checkbox"> ${_('Custom delimiters')}
+            <input type="checkbox" data-bind="checked: importData, disable: ! useDefaultLocation() && $parent.createWizard.source.path() == nonDefaultLocation();"> ${_('Import data')}
           </label>
-          ## field, coll map delimieters
-          ## regexp
 
           <label class="checkbox">
-              <input type="checkbox" checked> ${_('Use headers')}
+            <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Default location')}
+          </label>
+          <span data-bind="visible: ! useDefaultLocation()">
+            <label for="path" class="control-label"><div>${ _('External location') }</div>
+              <input type="text" class="form-control path input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
             </label>
+          </span>
+
           <label class="checkbox">
-              <input type="checkbox" checked> ${_('Bulk edit col names')}
-            </label>
+            <input type="checkbox" data-bind="checked: useCustomDelimiters"> ${_('Custom delimiters')}
+          </label>
+          <span data-bind="visible: useCustomDelimiters">
+            <div class="control-group">
+              <label for="fieldDelimiter" class="control-label"><div>${ _('Field') }</div>
+                <select id="fieldDelimiter" data-bind="options: $root.createWizard.customDelimiters, value: customFieldDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
+              </label>
+            </div>
+            <div class="control-group">
+              <label for="collectionDelimiter" class="control-label"><div>${ _('Array, Map') }</div>
+                <select id="collectionDelimiter" data-bind="options: $root.createWizard.customDelimiters, value: customCollectionDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
+              </label>
+            </div>
+            <div class="control-group">
+              <label for="structDelimiter" class="control-label"><div>${ _('Struct') }</div>
+                <select id="structDelimiter" data-bind="options: $root.createWizard.customDelimiters, value: customMapDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
+              </label>
+            </div>
+            <input type="text" data-bind="value: customRegexp"> ${_('Regexp')}
+          </span>
 
-          <div class="control-group">
-            <label for="collectionType" class="control-label"><div>${ _('Format') }</div>
-              <select id="collectionType" data-bind="options: $root.createWizard.destination.tableFormats, value: $root.createWizard.destination.tableFormat"></select>
-            </label>
+          <div class="row" style="margin-left: 8px">
+            <div class="span3">
+              <input type="checkbox" data-bind="checked: hasHeader">
+              ${_('Use first row as column names')} <a class="btn disable-feedback"><i class="fa fa-outdent"></i></a>
+            </div>
+            <div class="span3" data-bind="click: function() { alert('Hello'); }">
+              ${ _('Bulk edit column names') }<a class="btn"><i class="fa fa-edit"></i></a>
+            </div>
           </div>
 
           <a class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Add partition')}</a>
+          ## partitionColumns
         <!-- /ko -->
 
-        <form class="form-inline" data-bind="foreach: createWizard.destination.columns">
-          <!-- ko if: $root.createWizard.destination.ouputFormat() == 'table' -->
+         <h3 class="card-heading simple">${_('Fields')}</h3>
+        <form class="form-inline" data-bind="foreach: columns">
+          <!-- ko if: $parent.ouputFormat() == 'table' -->
             <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field"></div>
           <!-- /ko -->
 
-          <!-- ko if: $root.createWizard.destination.ouputFormat() == 'index' -->
+          <!-- ko if: $parent.ouputFormat() == 'index' -->
             <div data-bind="template: { name: 'index-field-template', data: $data }" class="margin-top-10 field"></div>
           <!-- /ko -->
         </form>
       </div>
+      <!-- /ko -->
     </div>
     <!-- /ko -->
 
@@ -739,13 +772,21 @@ ${ assist.assistPanel() }
           self.getDocuments();
         }
       });
-      self.inputFormats = ko.observableArray(['file', 'text', 'table', 'query', 'dbms', 'manual']);
+      self.inputFormats = ko.observableArray([
+          {'value': 'file', 'name': 'File'},
+          {'value': 'table', 'name': 'Table'},
+          {'value': 'text', 'name': 'Copy paste text'},
+          {'value': 'query', 'name': 'SQL Query'},
+          {'value': 'dbms', 'name': 'DBMS'},
+          {'value': 'manual', 'name': 'Manual with no input'},
+      ]);
 
       // File
       self.path = ko.observable('');
       self.path.subscribe(function(val) {
         if (val) {
           vm.createWizard.guessFormat();
+          vm.createWizard.destination.nonDefaultLocation(val);
         }
       })
 
@@ -831,19 +872,45 @@ ${ assist.assistPanel() }
       var self = this;
 
       self.name = ko.observable('');
+      self.description = ko.observable('');
 
       self.ouputFormat = ko.observable('table');
-      self.ouputFormats = ko.observableArray(['table', 'index', 'file']);
+      self.ouputFormats = ko.observableArray([
+          {'name': 'Table', 'value': 'table'},
+          {'name': 'Solr index', 'value': 'index'},
+          {'name': 'File', 'value': 'file'}
+      ]);
 
       self.format = ko.observable();
       self.columns = ko.observableArray();
 
       // Table
+      self.database = ko.observable('default');
       self.tableFormat = ko.observable('text');
-      self.tableFormats = ko.observableArray(['text', 'parquet', 'json', 'orc', 'kudu']);
-      self.hasHeader = ko.observable(false); // ?
-      self.bulkEditColumns = ko.observable(false);
+      self.tableFormats = ko.observableArray([
+          {'value': 'text', 'name': 'Text'},
+          {'value': 'parquet', 'name': 'Parquet'},
+          {'value': 'json', 'name': 'Json'},
+          {'value': 'orc', 'name': 'ORC'},
+          {'value': 'kudu', 'name': 'Kudu'}
+      ]);
+      self.ouputFormat = ko.observable('table');
+
       self.partitionColumns = ko.observableArray();
+      self.primaryKeys = ko.observableArray();
+
+      self.importData = ko.observable(true);
+      self.useDefaultLocation = ko.observable(true);
+      self.nonDefaultLocation = ko.observable('');
+
+      self.useCustomDelimiters = ko.observable(false);
+      self.customFieldDelimiter = ko.observable(',');
+      self.customCollectionDelimiter = ko.observable('\\002');
+      self.customMapDelimiter = ko.observable('\\003');
+      self.customRegexp = ko.observable('');
+
+      self.kuduDistribute = ko.observableArray([]);
+      self.hasHeader = ko.observable(true);
 
       // Index
     };
@@ -872,6 +939,15 @@ ${ assist.assistPanel() }
       self.source = new Source(vm);
       self.destination = new Destination(vm);
 
+      self.customDelimiters = ko.observable([
+        {'value': '\\001', 'name': '^A (\\001)', 'ascii': 1},
+        {'value': '\\002', 'name': '^B (\\002)', 'ascii': 2},
+        {'value': '\\003', 'name': '^C (\\003)', 'ascii': 3},
+        {'value': '\\t', 'name': '^Tab (\\t)', 'ascii': 9},
+        {'value': ',', 'name': 'Comma (,)', 'ascii': 44},
+        {'value': ' ', 'name': 'Space', 'ascii': 32}
+      ]);
+
       self.editorId = ko.observable();
       self.jobId = ko.observable();
       self.editorVM = null;

+ 2 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -966,7 +966,7 @@ var EditorViewModel = (function() {
         self.statusForButtons('executed');
         stopLongOperationTimeout();
 
-        if (vm.editorMode() && data.history_id) {
+        if (vm.editorMode() && data.history_id && window.location.pathname != '/indexer/importer/') {
           var url = '/notebook/editor' + (vm.isMobile() ? '_m' : '') + '?editor=' + data.history_id;
           if (vm.isResponsive()){
             url = vm.URLS.responsive + '&editor=' + data.history_id;
@@ -2433,6 +2433,7 @@ var EditorViewModel = (function() {
         if (self.editorMode()) {
           self.selectedNotebook().newSnippet(self.editorType());
           huePubSub.publish('detach.scrolls', self.selectedNotebook().snippets()[0]);
+
           if (window.location.getParameter('new') == '') {
             self.selectedNotebook().snippets()[0].statement_raw($.totalStorage('hue.notebook.lastWrittenSnippet.' + self.user + '.' + window.location.getParameter('type')));
             $.totalStorage('hue.notebook.lastWrittenSnippet.' + self.user +  '.' + window.location.getParameter('type'), '');