|
@@ -71,6 +71,7 @@ if sys.version_info[0] > 2:
|
|
|
from io import StringIO as string_io
|
|
from io import StringIO as string_io
|
|
|
from urllib.parse import urlparse, unquote as urllib_unquote
|
|
from urllib.parse import urlparse, unquote as urllib_unquote
|
|
|
from django.utils.translation import gettext as _
|
|
from django.utils.translation import gettext as _
|
|
|
|
|
+ import pandas as pd
|
|
|
else:
|
|
else:
|
|
|
from StringIO import StringIO as string_io
|
|
from StringIO import StringIO as string_io
|
|
|
from urllib import unquote as urllib_unquote
|
|
from urllib import unquote as urllib_unquote
|
|
@@ -127,7 +128,7 @@ def guess_format(request):
|
|
|
path = urllib_unquote(file_format["path"])
|
|
path = urllib_unquote(file_format["path"])
|
|
|
if 'xlsx' in path:
|
|
if 'xlsx' in path:
|
|
|
format_ = {
|
|
format_ = {
|
|
|
- "type": "xlsx",
|
|
|
|
|
|
|
+ "type": "excel",
|
|
|
"hasHeader": True
|
|
"hasHeader": True
|
|
|
}
|
|
}
|
|
|
else:
|
|
else:
|
|
@@ -141,6 +142,20 @@ def guess_format(request):
|
|
|
|
|
|
|
|
elif file_format['inputFormat'] == 'file':
|
|
elif file_format['inputFormat'] == 'file':
|
|
|
path = urllib_unquote(file_format["path"])
|
|
path = urllib_unquote(file_format["path"])
|
|
|
|
|
+ if path[-3:] == 'xls' or path[-4:] == 'xlsx':
|
|
|
|
|
+ if sys.version_info[0] > 2:
|
|
|
|
|
+ file_obj = request.fs.open(path)
|
|
|
|
|
+ if path[-3:] == 'xls':
|
|
|
|
|
+ df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='xlrd')
|
|
|
|
|
+ else:
|
|
|
|
|
+ df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='openpyxl')
|
|
|
|
|
+ _csv_data = df.to_csv(index=False)
|
|
|
|
|
+
|
|
|
|
|
+ path = excel_to_csv_file_name_change(path)
|
|
|
|
|
+ request.fs.create(path, overwrite=True, data=_csv_data)
|
|
|
|
|
+ else:
|
|
|
|
|
+ return JsonResponse({'status': -1, 'message': 'Python2 based Hue does not support Excel file importer'})
|
|
|
|
|
+
|
|
|
indexer = MorphlineIndexer(request.user, request.fs)
|
|
indexer = MorphlineIndexer(request.user, request.fs)
|
|
|
if not request.fs.isfile(path):
|
|
if not request.fs.isfile(path):
|
|
|
raise PopupException(_('Path %(path)s is not a file') % file_format)
|
|
raise PopupException(_('Path %(path)s is not a file') % file_format)
|
|
@@ -153,6 +168,16 @@ def guess_format(request):
|
|
|
}
|
|
}
|
|
|
})
|
|
})
|
|
|
_convert_format(format_)
|
|
_convert_format(format_)
|
|
|
|
|
+
|
|
|
|
|
+ if file_format["path"][-3:] == 'xls' or file_format["path"][-4:] == 'xlsx':
|
|
|
|
|
+ format_ = {
|
|
|
|
|
+ "quoteChar": "\"",
|
|
|
|
|
+ "recordSeparator": '\\n',
|
|
|
|
|
+ "type": "excel",
|
|
|
|
|
+ "hasHeader": True,
|
|
|
|
|
+ "fieldSeparator": ","
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
elif file_format['inputFormat'] == 'table':
|
|
elif file_format['inputFormat'] == 'table':
|
|
|
db = dbms.get(request.user)
|
|
db = dbms.get(request.user)
|
|
|
try:
|
|
try:
|
|
@@ -271,6 +296,8 @@ def guess_field_types(request):
|
|
|
elif file_format['inputFormat'] == 'file':
|
|
elif file_format['inputFormat'] == 'file':
|
|
|
indexer = MorphlineIndexer(request.user, request.fs)
|
|
indexer = MorphlineIndexer(request.user, request.fs)
|
|
|
path = urllib_unquote(file_format["path"])
|
|
path = urllib_unquote(file_format["path"])
|
|
|
|
|
+ if path[-3:] == 'xls' or path[-4:] == 'xlsx':
|
|
|
|
|
+ path = excel_to_csv_file_name_change(path)
|
|
|
stream = request.fs.open(path)
|
|
stream = request.fs.open(path)
|
|
|
encoding = check_encoding(stream.read(10000))
|
|
encoding = check_encoding(stream.read(10000))
|
|
|
LOG.debug('File %s encoding is %s' % (path, encoding))
|
|
LOG.debug('File %s encoding is %s' % (path, encoding))
|
|
@@ -432,6 +459,8 @@ def importer_submit(request):
|
|
|
if source['inputFormat'] == 'file':
|
|
if source['inputFormat'] == 'file':
|
|
|
if source['path']:
|
|
if source['path']:
|
|
|
path = urllib_unquote(source['path'])
|
|
path = urllib_unquote(source['path'])
|
|
|
|
|
+ if path[-3:] == 'xls' or path[-4:] == 'xlsx':
|
|
|
|
|
+ path = excel_to_csv_file_name_change(path)
|
|
|
source['path'] = request.fs.netnormpath(path)
|
|
source['path'] = request.fs.netnormpath(path)
|
|
|
stream = request.fs.open(path)
|
|
stream = request.fs.open(path)
|
|
|
file_encoding = check_encoding(stream.read(10000))
|
|
file_encoding = check_encoding(stream.read(10000))
|
|
@@ -721,6 +750,14 @@ def save_pipeline(request):
|
|
|
return JsonResponse(response)
|
|
return JsonResponse(response)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+def excel_to_csv_file_name_change(path):
|
|
|
|
|
+ if path[-4:] == 'xlsx':
|
|
|
|
|
+ path = path[:-4] + 'csv'
|
|
|
|
|
+ elif path[-3:] == 'xls':
|
|
|
|
|
+ path = path[:-3] + 'csv'
|
|
|
|
|
+ return path
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def upload_local_file_drag_and_drop(request):
|
|
def upload_local_file_drag_and_drop(request):
|
|
|
response = {'status': -1, 'data': ''}
|
|
response = {'status': -1, 'data': ''}
|
|
|
form = UploadLocalFileForm(request.POST, request.FILES)
|
|
form = UploadLocalFileForm(request.POST, request.FILES)
|