Pārlūkot izejas kodu

HUE-1746 [metastore] Read header from file

Currently the header is not truncated. Will be added separately.
Current tests are good, this is mostly front end.
Romain Rigaux 12 gadi atpakaļ
vecāks
revīzija
2c79c08

+ 14 - 9
apps/beeswax/src/beeswax/create_table.py

@@ -21,6 +21,7 @@ Views & controls for creating tables
 
 import logging
 import gzip
+import json
 
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
@@ -87,7 +88,7 @@ def create_table(request, database='default'):
   })
 
 
-IMPORT_PEEK_SIZE = 8192
+IMPORT_PEEK_SIZE = 5 * 1024**2
 IMPORT_PEEK_NLINES = 10
 DELIMITERS = [ hive_val for hive_val, desc, ascii in TERMINATORS ]
 DELIMITER_READABLE = {'\\001' : _('ctrl-As'),
@@ -190,17 +191,19 @@ def import_wizard(request, database='default'):
         if s3_col_formset is None:
           columns = []
           for i in range(n_cols):
-            columns.append(dict(
-                column_name='col_%s' % (i,),
-                column_type='string',
-            ))
+            columns.append({
+                'column_name': 'col_%s' % (i,),
+                'column_type': 'string',
+            })
           s3_col_formset = ColumnTypeFormSet(prefix='cols', initial=columns)
+
         return render('define_columns.mako', request, {
           'action': reverse(app_name + ':import_wizard', kwargs={'database': database}),
           'file_form': s1_file_form,
           'delim_form': s2_delim_form,
           'column_formset': s3_col_formset,
           'fields_list': fields_list,
+          'fields_list_json': json.dumps(fields_list),
           'n_cols': n_cols,
           'database': database,
         })
@@ -212,10 +215,12 @@ def import_wizard(request, database='default'):
         delim = s2_delim_form.cleaned_data['delimiter']
         table_name = s1_file_form.cleaned_data['name']
         proposed_query = django_mako.render_to_string("create_table_statement.mako", {
-            'table': dict(name=table_name,
-                          comment=s1_file_form.cleaned_data['comment'],
-                          row_format='Delimited',
-                          field_terminator=delim),
+            'table': {
+                'name': table_name,
+                'comment': s1_file_form.cleaned_data['comment'],
+                'row_format': 'Delimited',
+                'field_terminator': delim
+             },
             'columns': [ f.cleaned_data for f in s3_col_formset.forms ],
             'partition_columns': [],
             'database': database,

+ 1 - 1
apps/beeswax/src/beeswax/forms.py

@@ -343,7 +343,7 @@ PartitionTypeFormSet = simple_formset_factory(PartitionTypeForm, add_label=_t("A
 
 def _clean_databasename(name):
   try:
-    if name in db.get_databases():
+    if name in db.get_databases(): # Will always fail
       raise forms.ValidationError(_('Database "%(name)s" already exists.') % {'name': name})
   except Exception:
     return name

+ 94 - 38
apps/beeswax/src/beeswax/templates/define_columns.mako

@@ -59,11 +59,23 @@ ${ layout.metastore_menubar() }
                 <fieldset>
                     <div class="alert alert-info"><h3>${_('Define your columns')}</h3></div>
                     <div class="control-group">
+
+                    <div class="control-group" id="use-header">
+                        <div class="controls">
+                            ${_('Use first line as column names')}
+                            <input id="use_header" type="checkbox" name="use_header"/>
+                        </div>
+                        <div class="controls">
+                            ${_('Bulk edit column names')}
+                            <i id="editColumns" class="fa fa-edit" rel="tooltip" data-placement="right" title="${ _('Bulk edit names') }"></i>
+                        </div>
+                    </div>
+
                         <div class="controls">
                             <div class="scrollable">
                                 <table class="table table-striped">
                                     <thead>
-                                      <th id="editColumns">${ _('Column name') } &nbsp;<i class="fa fa-edit" rel="tooltip" data-placement="right" title="${ _('Bulk edit names') }"></i></th>
+                                      <th id="column_names">${ _('Column name') }</th>
                                       <th>${ _('Column Type') }</th>
                                       % for i in range(0, n_rows):
                                         <th><em>${_('Sample Row')} #${i + 1}</em></th>
@@ -73,19 +85,14 @@ ${ layout.metastore_menubar() }
                                       % for col, form in zip(range(len(column_formset.forms)), column_formset.forms):
                                       <tr>
                                         <td class="cols">
-                                          ${comps.field(form["column_name"],
-                                              render_default=False,
-                                              placeholder=_("Column name")
-                                            )}
+                                          ${ comps.field(form["column_name"], render_default=False, placeholder=_("Column name")) }
                                         </td>
                                         <td>
-                                          ${comps.field(form["column_type"],
-                                              render_default=True
-                                            )}
+                                          ${ comps.field(form["column_type"], render_default=True) }
                                           ${unicode(form["_exists"]) | n}
                                         </td>
                                         % for row in fields_list[:n_rows]:
-                                          ${ comps.getEllipsifiedCell(row[col], "bottom", "dataSample") }
+                                          ${ comps.getEllipsifiedCell(row[col], "bottom", "dataSample cols-%s" % (loop.index + 1)) }
                                         % endfor
                                       </tr>
                                       %endfor
@@ -187,7 +194,7 @@ ${ layout.metastore_menubar() }
       $(".cols input[type='text']").each(function (cnt, item) {
         _newVal += $(item).val() + (cnt < $(".cols input[type='text']").length - 1 ? ", " : "");
       });
-      $("#columnNamesPopover").show().css("left", $("#editColumns i").position().left + 16).css("top", $("#editColumns i").position().top - ($("#columnNamesPopover").height() / 2));
+      $("#columnNamesPopover").show().css("left", $("#column_names").position().left + 16).css("top", $("#column_names").position().top - ($("#columnNamesPopover").height() / 2));
       $(".editable-input input").val(_newVal).focus();
     });
 
@@ -212,43 +219,92 @@ ${ layout.metastore_menubar() }
       $("#columnNamesPopover").hide();
     });
 
-    $(".dataSample").each(function () {
-      var _val = $.trim($(this).text());
-      var _field = $(this).siblings().find("select[id^=id_cols-]");
-      var _foundType = "string";
-      if ($.isNumeric(_val)) {
-        _val = _val * 1;
-        if (isInt(_val)) {
-          // it's an int
-          _foundType = "int";
+    function guessColumnTypes() {
+      // Pick from 2nd column only
+      $(".dataSample").each(function () {
+        var _val = $.trim($(this).text());
+        var _field = $(this).siblings().find("select[id^=id_cols-]");
+        var _foundType = "string";
+
+        if ($.isNumeric(_val)) {
+          if (isInt(_val)) {
+            // it's an int
+            _foundType = "int";
+          }
+          else {
+            // it's possibly a float
+            _foundType = "float";
+          }
         }
         else {
-          // it's possibly a float
-          _foundType = "float";
+          if (_val.toLowerCase().indexOf("true") > -1 || _val.toLowerCase().indexOf("false") > -1) {
+            // it's a boolean
+            _foundType = "boolean";
+          }
+          else {
+            // it's most probably a string
+            _foundType = "string";
+          }
         }
-      }
-      else {
-        if (_val.toLowerCase().indexOf("true") > -1 || _val.toLowerCase().indexOf("false") > -1) {
-          // it's a boolean
-          _foundType = "boolean";
+
+        _field.data("possibleType", _foundType);
+        $(this).data("possibleType", _foundType);
+      });
+
+      $("select[id^=id_cols-]").each(function () {
+        $(this).val($(this).data("possibleType"));
+      });
+    }
+
+    guessColumnTypes();
+
+    $("#use_header").change(function (e) {
+      var input = this;
+
+      $(".cols input[type='text']").each(function (cnt, item) {
+        if (input.checked) {
+          $(item).data('previous', $(item).val());
+          $(item).val($.trim(${ fields_list_json | n,unicode }[0][cnt]));
+        } else {
+          $(item).val($(item).data('previous'));
         }
-        else {
-          // it's most probably a string
-          _foundType = "string";
+      });
+
+      $(".cols-1").each(function (cnt, item) {
+        if (input.checked) {
+          $(item).data('previous', $(item).text());
+          $(item).text($.trim(${ fields_list_json | n,unicode }[1][cnt]));
+        } else {
+          $(item).text($(item).data('previous'));
         }
-      }
-      if (_field.data("possibleType") != null && _field.data("possibleType") != _foundType) {
-        _field.data("possibleType", "string");
-      }
-      else {
-        _field.data("possibleType", _foundType);
-      }
+      });
+
+      $(".cols-2").each(function (cnt, item) {
+        if (input.checked) {
+          $(item).data('previous', $(item).text());
+          $(item).text($.trim(${ fields_list_json | n,unicode }[2][cnt]));
+        } else {
+          $(item).text($(item).data('previous'));
+        }
+      });
+
+      guessColumnTypes();
     });
 
-    $("select[id^=id_cols-]").each(function () {
-      $(this).val($(this).data("possibleType"));
+    // Really basic heuristic to detect if first row is a header.
+    var isString = 0;
+    $(".cols-1").each(function (cnt, item) {
+      if ($(".cols-1").data("possibleType") == 'string') {
+        isString += 1;
+      }
     });
 
+    if (isString > $(".cols-1").length - 1) {
+      $("#use_header").prop('checked', true);
+      $("#use_header").change();
+    }
+
+
     function parseJSON(val) {
       try {
         if (val.indexOf("\"") == -1 && val.indexOf("'") == -1) {