Kaynağa Gözat

[libs] upgrading pandas and openpyxl (#2807)

* [libs] upgrading pandas and openpyxl

* supporting xls file type in local importer
Ayush Goyal 3 yıl önce
ebeveyn
işleme
9bb6c67b2d

+ 2 - 2
desktop/core/requirements.txt

@@ -41,8 +41,8 @@ lxml==4.6.5
 Mako==1.1.4
 Markdown==3.1
 nose==1.3.7
-openpyxl==2.6.2
-pandas==1.1.5
+openpyxl==3.0.9
+pandas==1.4.2
 phoenixdb==1.1.0
 prompt-toolkit==2.0.10
 protobuf==3.17.0

+ 24 - 21
desktop/libs/indexer/src/indexer/api3.py

@@ -123,10 +123,14 @@ def _convert_format(format_dict, inverse=False):
 @api_error_handler
 def guess_format(request):
   file_format = json.loads(request.POST.get('fileFormat', '{}'))
+  file_type = file_format['file_type']
+  path = urllib_unquote(file_format["path"])
+  
+  if sys.version_info[0] < 3 and (file_type == 'excel' or path[-3:] == 'xls' or path[-4:] == 'xlsx'):
+    return JsonResponse({'status': -1, 'message': 'Python2 based Hue does not support Excel file importer'})
 
   if file_format['inputFormat'] == 'localfile':
-    path = urllib_unquote(file_format["path"])
-    if 'xlsx' in path:
+    if file_type == 'excel':
       format_ = {
         "type": "excel",
         "hasHeader": True
@@ -141,20 +145,16 @@ def guess_format(request):
       }
 
   elif file_format['inputFormat'] == 'file':
-    path = urllib_unquote(file_format["path"])
     if path[-3:] == 'xls' or path[-4:] == 'xlsx':
-      if sys.version_info[0] > 2:
-        file_obj = request.fs.open(path)
-        if path[-3:] == 'xls':
-          df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='xlrd')
-        else:
-          df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='openpyxl')
-        _csv_data = df.to_csv(index=False)
-
-        path = excel_to_csv_file_name_change(path)
-        request.fs.create(path, overwrite=True, data=_csv_data)
+      file_obj = request.fs.open(path)
+      if path[-3:] == 'xls':
+        df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='xlrd')
       else:
-        return JsonResponse({'status': -1, 'message': 'Python2 based Hue does not support Excel file importer'})
+        df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='openpyxl')
+      _csv_data = df.to_csv(index=False)
+
+      path = excel_to_csv_file_name_change(path)
+      request.fs.create(path, overwrite=True, data=_csv_data)
 
     indexer = MorphlineIndexer(request.user, request.fs)
     if not request.fs.isfile(path):
@@ -778,14 +778,17 @@ def upload_local_file(request):
   username = request.user.username
   filename = "%s_%s:%s;" % (username, uuid.uuid4(), re.sub('[^0-9a-zA-Z]+', '_', upload_file.name))
   file_format = upload_file.name.split(".")[-1]
+  file_type = 'csv'
 
-  if file_format == "xlsx":
-    workbook = openpyxl.load_workbook(upload_file)
-    sheet = workbook.active
+  if file_format in ("xlsx", "xls"):
+    if file_format == "xlsx":
+      read_file = pd.read_excel(upload_file)
+    else:
+      read_file = pd.read_excel(upload_file, engine='xlrd')
+  
     temp_file = tempfile.NamedTemporaryFile(mode='w', prefix=filename, suffix='.csv', delete=False)
-    csv_file = csv.writer(temp_file)
-    for row in sheet.rows:
-      csv_file.writerow([cell.value for cell in row])
+    read_file.to_csv(temp_file, index=False)
+    file_type = 'excel'
 
   else: 
     temp_file = tempfile.NamedTemporaryFile(prefix=filename, suffix='.csv', delete=False)
@@ -794,4 +797,4 @@ def upload_local_file(request):
   local_file_url = temp_file.name
   temp_file.close()
 
-  return JsonResponse({'local_file_url': local_file_url})
+  return JsonResponse({'local_file_url': local_file_url, 'file_type': file_type})

+ 4 - 1
desktop/libs/indexer/src/indexer/api3_tests.py

@@ -34,6 +34,8 @@ else:
 
 
 def test_xlsx_local_file_upload():
+  if sys.version_info[0] < 3:
+    raise SkipTest
 
   csv_file = '''test 1,test.2,test_3,test_4
 2010-10-10 00:00:00,2012-10-11 01:00:00,30,
@@ -113,7 +115,8 @@ def test_guess_format_excel_remote_file():
       with patch('indexer.api3.MorphlineIndexer') as MorphlineIndexer:
         file_format = {
           'inputFormat': 'file',
-          'path': 's3a://gethue/example1.xlsx'
+          'path': 's3a://gethue/example1.xlsx',
+          'file_type': ''
         }
         file_format = json.dumps(file_format)
         request = Mock(

+ 3 - 1
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -237,7 +237,7 @@ ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
           <div data-bind="visible: createWizard.source.inputFormat() == 'localfile'">
               <form method="post" action="" enctype="multipart/form-data" id="uploadform">
                 <div >
-                    <input type="file" id="inputfile" name="inputfile" style="margin-left: 130px" accept=".csv, .xlsx">
+                    <input type="file" id="inputfile" name="inputfile" style="margin-left: 130px" accept=".csv, .xlsx, .xls">
                 </div>
             </form>
           </div>
@@ -1836,6 +1836,7 @@ ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
 
       // File
       self.path = ko.observable('');
+      self.file_type = ko.observable('');
       self.path.subscribe(function(val) {
         if (val) {
           wizard.guessFormat();
@@ -3245,6 +3246,7 @@ ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
             processData:false,
             success:function (response) {
               viewModel.createWizard.source.path(response['local_file_url']);
+              viewModel.createWizard.source.file_type(response['file_type']);
             }
           });
         }