瀏覽代碼

[importer] auto formatting table and header name

ayush.goyal 4 年之前
父節點
當前提交
62e7a7400e

+ 1 - 1
apps/beeswax/data/tables/flights.csv

@@ -1,4 +1,4 @@
-"date","hour","minute","dep","arr","dep_delay","arr_delay","carrier","flight","dest","plane","cancelled","time","dist"
+"date(1)","hour.1","minute","dep","arr","dep_delay","arr_delay","carrier","flight","dest","plane","cancelled","time","dist"
 2011-12-14 12:00:00,13,4,1304,1704,24,14,"WN",3085,"PHL","N524SW",true,159,1336
 2011-12-14 12:00:00,17,52,1752,1943,12,8,"WN",39,"PHX","N503SW",true,155,1020
 2011-12-14 12:00:00,7,9,709,853,-1,-12,"WN",424,"PHX","N761RR",true,152,1020

+ 3 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -25,6 +25,7 @@ import json
 import logging
 import urllib.error
 import openpyxl
+import re
 import sys
 import tempfile
 import uuid
@@ -239,7 +240,7 @@ def guess_field_types(request):
 
       if file_format['format']['hasHeader']:
         sample = csv_data[1:5]
-        column_row = csv_data[0]
+        column_row = [re.sub('[^0-9a-zA-Z]+', '_', col) for col in csv_data[0]]
       else:
         sample = csv_data[:4]
         column_row = ['field_' + str(count+1) for count, col in enumerate(sample[0])]
@@ -731,7 +732,7 @@ def upload_local_file_drag_and_drop(request):
 def upload_local_file(request):
   upload_file = request.FILES['file']
   username = request.user.username
-  filename = "%s_%s:%s;" % (username, uuid.uuid4(), upload_file.name)
+  filename = "%s_%s:%s;" % (username, uuid.uuid4(), re.sub('[^0-9a-zA-Z]+', '_', upload_file.name))
   file_format = upload_file.name.split(".")[-1]
 
   if file_format == "xlsx":

+ 26 - 3
desktop/libs/indexer/src/indexer/api3_tests.py

@@ -17,12 +17,12 @@
 
 import json
 import sys
-from nose.tools import assert_equal
+from nose.tools import assert_equal, assert_true
 from django.utils.datastructures import MultiValueDict
 from django.core.files.uploadhandler import InMemoryUploadedFile
 
 from desktop.settings import BASE_DIR
-from indexer.api3 import upload_local_file
+from indexer.api3 import upload_local_file, guess_field_types
 
 if sys.version_info[0] > 2:
   from urllib.parse import unquote as urllib_unquote
@@ -69,7 +69,7 @@ scattered,,,scattered
 ,scattered,,
 '''
   with open(BASE_DIR + '/apps/beeswax/data/tables/book.xlsx', 'rb') as file:
-    uploaded_file = InMemoryUploadedFile(file=file, field_name='test', name='book.xlsx', content_type='xlsx', size=100, charset='utf-8')
+    uploaded_file = InMemoryUploadedFile(file=file, field_name='test', name='book(1).xlsx', content_type='xlsx', size=100, charset='utf-8')
     request = Mock()
     request.user = Mock()
     request.FILES = MultiValueDict({'file': [uploaded_file]})
@@ -81,3 +81,26 @@ scattered,,,scattered
     test_file = _test_file.read().replace('\r\n', '\n')
 
     assert_equal(csv_file, test_file)
+    assert_true("book_1_xlsx" in path)
+
+
+def test_col_names():
+  file_format = {
+    'inputFormat': 'localfile',
+    'path': BASE_DIR + '/apps/beeswax/data/tables/flights.csv',
+    'format': {
+      'hasHeader': True
+    }
+  }
+  file_format = json.dumps(file_format)
+  request = Mock()
+  request.POST = {'fileFormat': file_format}
+
+  response = guess_field_types(request)
+  response = json.loads(response.content)
+
+  columns_name = [col['name'] for col in response['columns']]
+
+  assert_true('date_1_' in columns_name)
+  assert_true('hour_1' in columns_name)
+  assert_true('minute' in columns_name)