浏览代码

HUE-1747 [metastore] Create table from quoted csv file

Make a test independent of the Yarn ports
Romain Rigaux 12 年之前
父节点
当前提交
3d9e2b623d
共有 2 个文件被更改,包括 56 次插入27 次删除
  1. 28 21
      apps/beeswax/src/beeswax/create_table.py
  2. 28 6
      apps/beeswax/src/beeswax/tests.py

+ 28 - 21
apps/beeswax/src/beeswax/create_table.py

@@ -15,13 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""
-Views & controls for creating tables
-"""
 
-import logging
+import csv
 import gzip
 import json
+import logging
 
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
@@ -196,17 +194,19 @@ def import_wizard(request, database='default'):
                 'column_type': 'string',
             })
           s3_col_formset = ColumnTypeFormSet(prefix='cols', initial=columns)
-
-        return render('define_columns.mako', request, {
-          'action': reverse(app_name + ':import_wizard', kwargs={'database': database}),
-          'file_form': s1_file_form,
-          'delim_form': s2_delim_form,
-          'column_formset': s3_col_formset,
-          'fields_list': fields_list,
-          'fields_list_json': json.dumps(fields_list),
-          'n_cols': n_cols,
-          'database': database,
-        })
+        try:
+          return render('define_columns.mako', request, {
+            'action': reverse(app_name + ':import_wizard', kwargs={'database': database}),
+            'file_form': s1_file_form,
+            'delim_form': s2_delim_form,
+            'column_formset': s3_col_formset,
+            'fields_list': fields_list,
+            'fields_list_json': json.dumps(fields_list),
+            'n_cols': n_cols,
+            'database': database,
+          })
+        except Exception, e:
+          raise PopupException(_("The selected delimiter is creating an un-even number of columns. Please make sure you don't have empty columns."), detail=e)
 
       #
       # Final: Execute
@@ -364,11 +364,13 @@ def _readfields(lines, delimiters):
   res = (None, None)
 
   for delim in delimiters:
-    fields_list = [ ]
-    for line in lines:
-      if line:
-        # Unescape the delimiter back to its character value
-        fields_list.append(line.split(delim.decode('string_escape')))
+    # Unescape the delimiter back to its character value
+    delimiter = delim.decode('string_escape')
+    try:
+      fields_list = _get_rows(lines, delimiter)
+    except:
+      fields_list = [line.split(delimiter) for line in lines if line]
+
     score = score_delim(fields_list)
     LOG.debug("'%s' gives score of %s" % (delim, score))
     if score > max_score:
@@ -377,6 +379,11 @@ def _readfields(lines, delimiters):
   return res
 
 
+def _get_rows(lines, delimiter):
+  column_reader = csv.reader(lines, delimiter=delimiter)
+  return [row for row in column_reader if row]
+
+
 def _peek_file(fs, file_form):
   """_peek_file(fs, file_form) -> (path, initial data)"""
   try:
@@ -445,7 +452,7 @@ def load_after_create(request, database):
   LOG.debug("Auto loading data from %s into table %s" % (path, tablename))
   hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, tablename)
   query = hql_query(hql)
-  app_name = get_app_name(request)
+
   on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': tablename})
 
   return execute_directly(request, query, on_success_url=on_success_url)

+ 28 - 6
apps/beeswax/src/beeswax/tests.py

@@ -998,7 +998,13 @@ for x in sys.stdin:
     RAW_FIELDS = [
       ['ta\tb', 'nada', 'sp ace'],
       ['f\too', 'bar', 'fred'],
-      ['a\ta', 'bb', 'cc'] ]
+      ['a\ta', 'bb', 'cc'],
+    ]
+    CSV_FIELDS = [
+      ['a', 'b', 'c'],
+      ['"a,a"', '"b,b"', '"c,c"'],
+      ['"a,\"\"a"', '"b,\"\"b"', '"c,\"\"c"'],
+    ]
 
     def write_file(filename, raw_fields, delim, do_gzip=False):
       lines = [ delim.join(row) for row in raw_fields ]
@@ -1018,6 +1024,7 @@ for x in sys.stdin:
     write_file('/tmp/comma.dat', RAW_FIELDS, ',')
     write_file('/tmp/pipes.dat', RAW_FIELDS, '|')
     write_file('/tmp/comma.dat.gz', RAW_FIELDS, ',', do_gzip=True)
+    write_file('/tmp/comma.csv', CSV_FIELDS, ',')
 
     # Test auto delim selection
     resp = self.client.post('/beeswax/create/import_wizard/default', {
@@ -1057,6 +1064,21 @@ for x in sys.stdin:
     })
     assert_equal(len(resp.context['fields_list'][0]), 3)
 
+    # Make sure quoted CSV works
+    resp = self.client.post('/beeswax/create/import_wizard/default', {
+      'submit_preview': 'on',
+      'path': '/tmp/comma.csv',
+      'name': 'test_create_import_csv',
+      'delimiter_0': '__other__',
+      'delimiter_1': ',',
+      'file_type': 'text',
+    })
+    assert_equal(resp.context['fields_list'], [
+      ['a', 'b', 'c'],
+      ['a,a', 'b,b', 'c,c'],
+      ['a,"a', 'b,"b', 'c,"c'],
+    ] )
+
     # Test column definition
     resp = self.client.post('/beeswax/create/import_wizard/default', {
       'submit_delim': 'on',
@@ -1595,11 +1617,11 @@ class TestDesign():
         {'type': 'FILE', 'path': 's3://host/my_s3_file'}
     ]
 
-    assert_equal([
-        u'\nADD FILE hdfs://localhost:8020my_file\n', # Expected
-        u'\nADD FILE hdfs://localhost:8020/my_path/my_file\n',
-        u'\nADD FILE s3://host/my_s3_file\n'
-    ], design.get_configuration_statements())
+    statements = design.get_configuration_statements()
+    assert_true(re.match('\nADD FILE hdfs://localhost:(\d+)my_file\n', statements[0]), statements[0])
+    assert_true(re.match('\nADD FILE hdfs://localhost:(\d+)/my_path/my_file\n', statements[1]), statements[1])
+    assert_equal('\nADD FILE s3://host/my_s3_file\n', statements[2])
+
 
 def search_log_line(component, expected_log, all_logs):
   """Checks if 'expected_log' can be found in one line of 'all_logs' outputed by the logging component 'component'."""