Browse Source

[importer] importer supporting xlsx file for local small file option

ayush.goyal 4 years ago
parent
commit
538c61ed7c

BIN
apps/beeswax/data/tables/book.xlsx


+ 14 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -24,6 +24,7 @@ import csv
 import json
 import logging
 import urllib.error
+import openpyxl
 import sys
 import tempfile
 import uuid
@@ -731,9 +732,20 @@ def upload_local_file(request):
   upload_file = request.FILES['file']
   username = request.user.username
   filename = "%s_%s:%s;" % (username, uuid.uuid4(), upload_file.name)
+  file_format = upload_file.name.split(".")[-1]
+
+  if file_format == "xlsx":
+    workbook = openpyxl.load_workbook(upload_file)
+    sheet = workbook.active
+    temp_file = tempfile.NamedTemporaryFile(mode='w', prefix=filename, suffix='.csv', delete=False)
+    csv_file = csv.writer(temp_file)
+    for row in sheet.rows:
+      csv_file.writerow([cell.value for cell in row])
+
+  else: 
+    temp_file = tempfile.NamedTemporaryFile(prefix=filename, suffix='.csv', delete=False)
+    temp_file.write(upload_file.read())
 
-  temp_file = tempfile.NamedTemporaryFile(prefix=filename, suffix='.csv', delete=False)
-  temp_file.write(upload_file.read())
   local_file_url = temp_file.name
   temp_file.close()
 

+ 83 - 0
desktop/libs/indexer/src/indexer/api3_tests.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import sys
+from nose.tools import assert_equal
+from django.utils.datastructures import MultiValueDict
+from django.core.files.uploadhandler import InMemoryUploadedFile
+
+from desktop.settings import BASE_DIR
+from indexer.api3 import upload_local_file
+
+if sys.version_info[0] > 2:
+  from urllib.parse import unquote as urllib_unquote
+  from unittest.mock import patch, Mock, MagicMock
+else:
+  from urllib import unquote as urllib_unquote
+  from mock import patch, Mock, MagicMock
+
+
+def test_xlsx_local_file_upload():
+
+  csv_file = '''test 1,test.2,test_3,test_4
+2010-10-10 00:00:00,2012-10-11 01:00:00,30,
+2021-10-10 00:00:00,2012-10-11 03:00:00,0.12,
+2010-10-10 00:00:00,2012-10-11 09:00:00,0.99,
+2010-10-10 00:00:00,2012-10-11 04:00:00,500,
+2010-10-10 00:00:00,2012-10-11 09:00:00,29830,
+2010-10-10 00:00:00,2012-10-11 08:00:00,50,
+2010-10-10 00:00:00,2012-10-11 07:00:00,,
+2010-10-10 00:00:00,2012-10-11 03:00:00,,
+2010-10-10 00:00:00,2012-10-11 09:00:00,,
+2010-10-10 00:00:00,2022-10-10 14:00:00,,
+2010-10-10 00:00:00,2021-10-10 15:00:00,,
+2010-10-10 00:00:00,,,
+2010-10-10 00:00:00,,,
+2010-10-10 00:00:00,scattered,,
+2010-10-10 00:00:00,,scattered,
+2010-10-10 00:00:00,,,
+,,,
+,,,
+scattered,,,
+,,scattered,
+,,,
+,,,
+,scattered,,
+,,,
+,,,
+,,,
+scattered,,,scattered
+,,,
+,,,
+,,,
+,,,
+,scattered,,
+'''
+  with open(BASE_DIR + '/apps/beeswax/data/tables/book.xlsx', 'rb') as file:
+    uploaded_file = InMemoryUploadedFile(file=file, field_name='test', name='book.xlsx', content_type='xlsx', size=100, charset='utf-8')
+    request = Mock()
+    request.user = Mock()
+    request.FILES = MultiValueDict({'file': [uploaded_file]})
+
+    response = upload_local_file(request)
+
+  path = urllib_unquote(json.loads(response.content.decode('utf-8'))['local_file_url'])
+  with open(path, 'r') as _test_file:
+    test_file = _test_file.read().replace('\r\n', '\n')
+
+    assert_equal(csv_file, test_file)

+ 1 - 1
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -237,7 +237,7 @@ ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
           <div data-bind="visible: createWizard.source.inputFormat() == 'localfile'">
               <form method="post" action="" enctype="multipart/form-data" id="uploadform">
                 <div >
-                    <input type="file" id="inputfile" name="inputfile" style="margin-left: 130px" accept=".csv">
+                    <input type="file" id="inputfile" name="inputfile" style="margin-left: 130px" accept=".csv, .xlsx">
                 </div>
             </form>
           </div>