瀏覽代碼

HUE-1746 [metastore] Read header from file

Currently the header is not truncated. Will be added separately.
Current tests are good, this is mostly front end.
Romain Rigaux 12 年之前
父節點
當前提交
2c79c08

+ 14 - 9
apps/beeswax/src/beeswax/create_table.py

@@ -21,6 +21,7 @@ Views & controls for creating tables
 
 import logging
 import gzip
+import json
 
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
@@ -87,7 +88,7 @@ def create_table(request, database='default'):
   })
 
 
-IMPORT_PEEK_SIZE = 8192
+IMPORT_PEEK_SIZE = 5 * 1024**2
 IMPORT_PEEK_NLINES = 10
 DELIMITERS = [ hive_val for hive_val, desc, ascii in TERMINATORS ]
 DELIMITER_READABLE = {'\\001' : _('ctrl-As'),
@@ -190,17 +191,19 @@ def import_wizard(request, database='default'):
         if s3_col_formset is None:
           columns = []
           for i in range(n_cols):
-            columns.append(dict(
-                column_name='col_%s' % (i,),
-                column_type='string',
-            ))
+            columns.append({
+                'column_name': 'col_%s' % (i,),
+                'column_type': 'string',
+            })
           s3_col_formset = ColumnTypeFormSet(prefix='cols', initial=columns)
+
         return render('define_columns.mako', request, {
           'action': reverse(app_name + ':import_wizard', kwargs={'database': database}),
           'file_form': s1_file_form,
           'delim_form': s2_delim_form,
           'column_formset': s3_col_formset,
           'fields_list': fields_list,
+          'fields_list_json': json.dumps(fields_list),
           'n_cols': n_cols,
           'database': database,
         })
@@ -212,10 +215,12 @@ def import_wizard(request, database='default'):
         delim = s2_delim_form.cleaned_data['delimiter']
         table_name = s1_file_form.cleaned_data['name']
         proposed_query = django_mako.render_to_string("create_table_statement.mako", {
-            'table': dict(name=table_name,
-                          comment=s1_file_form.cleaned_data['comment'],
-                          row_format='Delimited',
-                          field_terminator=delim),
+            'table': {
+                'name': table_name,
+                'comment': s1_file_form.cleaned_data['comment'],
+                'row_format': 'Delimited',
+                'field_terminator': delim
+             },
             'columns': [ f.cleaned_data for f in s3_col_formset.forms ],
             'partition_columns': [],
             'database': database,

+ 1 - 1
apps/beeswax/src/beeswax/forms.py

@@ -343,7 +343,7 @@ PartitionTypeFormSet = simple_formset_factory(PartitionTypeForm, add_label=_t("A
 
 def _clean_databasename(name):
   try:
-    if name in db.get_databases():
+    if name in db.get_databases(): # Will always fail
       raise forms.ValidationError(_('Database "%(name)s" already exists.') % {'name': name})
   except Exception:
     return name

+ 94 - 38
apps/beeswax/src/beeswax/templates/define_columns.mako

@@ -59,11 +59,23 @@ ${ layout.metastore_menubar() }
                 <fieldset>
                     <div class="alert alert-info"><h3>${_('Define your columns')}</h3></div>
                     <div class="control-group">
+
+                    <div class="control-group" id="use-header">
+                        <div class="controls">
+                            ${_('Use first line as column names')}
+                            <input id="use_header" type="checkbox" name="use_header"/>
+                        </div>
+                        <div class="controls">
+                            ${_('Bulk edit column names')}
+                            <i id="editColumns" class="fa fa-edit" rel="tooltip" data-placement="right" title="${ _('Bulk edit names') }"></i>
+                        </div>
+                    </div>
+
                         <div class="controls">
                             <div class="scrollable">
                                 <table class="table table-striped">
                                     <thead>
-                                      <th id="editColumns">${ _('Column name') } &nbsp;<i class="fa fa-edit" rel="tooltip" data-placement="right" title="${ _('Bulk edit names') }"></i></th>
+                                      <th id="column_names">${ _('Column name') }</th>
                                       <th>${ _('Column Type') }</th>
                                       % for i in range(0, n_rows):
                                         <th><em>${_('Sample Row')} #${i + 1}</em></th>
@@ -73,19 +85,14 @@ ${ layout.metastore_menubar() }
                                       % for col, form in zip(range(len(column_formset.forms)), column_formset.forms):
                                       <tr>
                                         <td class="cols">
-                                          ${comps.field(form["column_name"],
-                                              render_default=False,
-                                              placeholder=_("Column name")
-                                            )}
+                                          ${ comps.field(form["column_name"], render_default=False, placeholder=_("Column name")) }
                                         </td>
                                         <td>
-                                          ${comps.field(form["column_type"],
-                                              render_default=True
-                                            )}
+                                          ${ comps.field(form["column_type"], render_default=True) }
                                           ${unicode(form["_exists"]) | n}
                                         </td>
                                         % for row in fields_list[:n_rows]:
-                                          ${ comps.getEllipsifiedCell(row[col], "bottom", "dataSample") }
+                                          ${ comps.getEllipsifiedCell(row[col], "bottom", "dataSample cols-%s" % (loop.index + 1)) }
                                         % endfor
                                       </tr>
                                       %endfor
@@ -187,7 +194,7 @@ ${ layout.metastore_menubar() }
       $(".cols input[type='text']").each(function (cnt, item) {
         _newVal += $(item).val() + (cnt < $(".cols input[type='text']").length - 1 ? ", " : "");
       });
-      $("#columnNamesPopover").show().css("left", $("#editColumns i").position().left + 16).css("top", $("#editColumns i").position().top - ($("#columnNamesPopover").height() / 2));
+      $("#columnNamesPopover").show().css("left", $("#column_names").position().left + 16).css("top", $("#column_names").position().top - ($("#columnNamesPopover").height() / 2));
       $(".editable-input input").val(_newVal).focus();
     });
 
@@ -212,43 +219,92 @@ ${ layout.metastore_menubar() }
       $("#columnNamesPopover").hide();
     });
 
-    $(".dataSample").each(function () {
-      var _val = $.trim($(this).text());
-      var _field = $(this).siblings().find("select[id^=id_cols-]");
-      var _foundType = "string";
-      if ($.isNumeric(_val)) {
-        _val = _val * 1;
-        if (isInt(_val)) {
-          // it's an int
-          _foundType = "int";
+    function guessColumnTypes() {
+      // Pick from 2nd column only
+      $(".dataSample").each(function () {
+        var _val = $.trim($(this).text());
+        var _field = $(this).siblings().find("select[id^=id_cols-]");
+        var _foundType = "string";
+
+        if ($.isNumeric(_val)) {
+          if (isInt(_val)) {
+            // it's an int
+            _foundType = "int";
+          }
+          else {
+            // it's possibly a float
+            _foundType = "float";
+          }
         }
         else {
-          // it's possibly a float
-          _foundType = "float";
+          if (_val.toLowerCase().indexOf("true") > -1 || _val.toLowerCase().indexOf("false") > -1) {
+            // it's a boolean
+            _foundType = "boolean";
+          }
+          else {
+            // it's most probably a string
+            _foundType = "string";
+          }
         }
-      }
-      else {
-        if (_val.toLowerCase().indexOf("true") > -1 || _val.toLowerCase().indexOf("false") > -1) {
-          // it's a boolean
-          _foundType = "boolean";
+
+        _field.data("possibleType", _foundType);
+        $(this).data("possibleType", _foundType);
+      });
+
+      $("select[id^=id_cols-]").each(function () {
+        $(this).val($(this).data("possibleType"));
+      });
+    }
+
+    guessColumnTypes();
+
+    $("#use_header").change(function (e) {
+      var input = this;
+
+      $(".cols input[type='text']").each(function (cnt, item) {
+        if (input.checked) {
+          $(item).data('previous', $(item).val());
+          $(item).val($.trim(${ fields_list_json | n,unicode }[0][cnt]));
+        } else {
+          $(item).val($(item).data('previous'));
         }
-        else {
-          // it's most probably a string
-          _foundType = "string";
+      });
+
+      $(".cols-1").each(function (cnt, item) {
+        if (input.checked) {
+          $(item).data('previous', $(item).text());
+          $(item).text($.trim(${ fields_list_json | n,unicode }[1][cnt]));
+        } else {
+          $(item).text($(item).data('previous'));
         }
-      }
-      if (_field.data("possibleType") != null && _field.data("possibleType") != _foundType) {
-        _field.data("possibleType", "string");
-      }
-      else {
-        _field.data("possibleType", _foundType);
-      }
+      });
+
+      $(".cols-2").each(function (cnt, item) {
+        if (input.checked) {
+          $(item).data('previous', $(item).text());
+          $(item).text($.trim(${ fields_list_json | n,unicode }[2][cnt]));
+        } else {
+          $(item).text($(item).data('previous'));
+        }
+      });
+
+      guessColumnTypes();
     });
 
-    $("select[id^=id_cols-]").each(function () {
-      $(this).val($(this).data("possibleType"));
+    // Really basic heuristic to detect if first row is a header.
+    var isString = 0;
+    $(".cols-1").each(function (cnt, item) {
+      if ($(".cols-1").data("possibleType") == 'string') {
+        isString += 1;
+      }
     });
 
+    if (isString > $(".cols-1").length - 1) {
+      $("#use_header").prop('checked', true);
+      $("#use_header").change();
+    }
+
+
     function parseJSON(val) {
       try {
         if (val.indexOf("\"") == -1 && val.indexOf("'") == -1) {