Browse Source

HUE-4670 [metastore] Allow headers to be ignored/removed when creating external table

Jenny Kim 9 years ago
parent
commit
3be53cd

+ 1 - 11
apps/beeswax/src/beeswax/create_table.py

@@ -250,6 +250,7 @@ def import_wizard(request, database='default'):
                 'file_format': 'TextFile',
                 'load_data': load_data,
                 'path': path,
+                'skip_header': request.REQUEST.get('removeHeader', 'off').lower() == 'on'
              },
             'columns': [ f.cleaned_data for f in s3_col_formset.forms ],
             'partition_columns': [],
@@ -282,7 +283,6 @@ def _submit_create_and_load(request, create_hql, table_name, path, load_data, da
   if load_data == 'IMPORT':
     on_success_params['table'] = table_name
     on_success_params['path'] = path
-    on_success_params['removeHeader'] = request.POST.get('removeHeader')
     on_success_url = reverse(app_name + ':load_after_create', kwargs={'database': database}) + '?' + on_success_params.urlencode()
   else:
     on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table_name})
@@ -482,22 +482,12 @@ def load_after_create(request, database):
   """
   tablename = request.REQUEST.get('table')
   path = request.REQUEST.get('path')
-  is_remove_header = request.REQUEST.get('removeHeader').lower() == 'on' and not path.endswith('gz')
 
   if not tablename or not path:
     msg = _('Internal error: Missing needed parameter to load data into table.')
     LOG.error(msg)
     raise PopupException(msg)
 
-  if is_remove_header:
-    try:
-      path = path.rstrip('/')  # need to remove trailing slash before overwrite
-      remove_header(request.fs, path)
-    except Exception, e:
-      msg = "The headers of the file could not be removed."
-      LOG.exception(msg)
-      raise PopupException(_(msg), detail=e)
-
   LOG.debug("Auto loading data from %s into table %s" % (path, tablename))
   hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, tablename)
   query = hql_query(hql)

+ 3 - 0
apps/beeswax/src/beeswax/forms.py

@@ -267,6 +267,9 @@ class CreateTableForm(DependencyAwareForm):
   use_default_location = forms.BooleanField(required=False, initial=True, label=_t("Use default location."))
   external_location = forms.CharField(required=False, help_text=_t("Path to HDFS directory or file of table data."))
 
+  # Table Properties
+  skip_header = forms.BooleanField(required=False, initial=False, label=_t("Use header row for column names?"))
+
   dependencies += [
     ("use_default_location", False, "external_location")
   ]

+ 11 - 0
apps/beeswax/src/beeswax/templates/create_table_manually.mako

@@ -343,6 +343,17 @@ ${ assist.assistPanel() }
                     <fieldset>
                         <div class="alert alert-info"><h3>${_('Configure Table Columns')}</h3>
                         </div>
+                        <div class="control-group">
+                            <label class="control-label">${_('Table Properties')}</label>
+                            <div class="controls">
+                                <label class="checkbox">
+                                    ${comps.field(table_form["skip_header"],
+                                    render_default=True
+                                    )}
+                                    ${_('Skip Header Row?')}
+                                </label>
+                            </div>
+                        </div>
                         % for form in columns_form.forms:
                             ${render_column(form)}
                         %endfor

+ 3 - 0
apps/beeswax/src/beeswax/templates/create_table_statement.mako

@@ -93,3 +93,6 @@ INPUTFORMAT ${table["input_format_class"] | n} OUTPUTFORMAT ${table["output_form
 % if table.get("load_data", "IMPORT") == 'EXTERNAL':
 LOCATION "${table["path"] | n}"
 % endif
+% if table.get("skip_header", False):
+TBLPROPERTIES("skip.header.line.count" = "1")
+% endif

+ 2 - 0
apps/beeswax/src/beeswax/tests.py

@@ -1281,6 +1281,7 @@ for x in sys.stdin:
       'table-map_key_terminator_0': r'\003',
       'table-file_format': 'TextFile',
       'table-use_default_location': 'False',
+      'table-skip_header': 'False',
       'table-external_location': '/tmp/foo',
       'columns-0-column_name': 'my_col',
       'columns-0-column_type': 'string',
@@ -1336,6 +1337,7 @@ for x in sys.stdin:
       'table-map_key_terminator_0': r'\003',
       'table-file_format': 'TextFile',
       'table-use_default_location': 'True',
+      'table-skip_header': 'False',
       'columns-0-column_name': 'my_col',
       'columns-0-column_type': 'string',
       'columns-0-_exists': 'True',