Эх сурвалжийг харах

HUE-8207 [indexer] Detect the encoding of the imported file

Romain Rigaux 7 жил өмнө
parent
commit
fa5128b2b9

+ 2 - 2
desktop/core/src/desktop/lib/i18n.py

@@ -54,13 +54,13 @@ def validate_encoding(encoding):
   except LookupError:
     return False
 
-def smart_unicode(s, strings_only=False, errors='strict'):
+def smart_unicode(s, strings_only=False, errors='strict', encoding=None):
   """
   Wrapper around Django's version, while supplying our configured encoding.
   Decode char array to unicode.
   """
   return django.utils.encoding.smart_unicode(
-        s, get_site_encoding(), strings_only, errors)
+        s, encoding if encoding is not None else get_site_encoding(), strings_only, errors)
 
 def force_unicode(s, strings_only=False, errors='strict'):
   """

+ 6 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import chardet
 import json
 import logging
 
@@ -124,6 +125,8 @@ def guess_field_types(request):
   if file_format['inputFormat'] == 'file':
     indexer = MorphlineIndexer(request.user, request.fs)
     stream = request.fs.open(file_format["path"])
+    encoding = chardet.detect(stream.read(10000)).get('encoding')
+    stream.seek(0)
     _convert_format(file_format["format"], inverse=True)
 
     format_ = indexer.guess_field_types({
@@ -134,10 +137,11 @@ def guess_field_types(request):
       "format": file_format['format']
     })
 
+    # Note: Would also need to set charset to table (only supported in Hive)
     if 'sample' in format_:
-      format_['sample'] = escape_rows(format_['sample'], nulls_only=True)
+      format_['sample'] = escape_rows(format_['sample'], nulls_only=True, encoding=encoding)
     for col in format_['columns']:
-      col['name'] = smart_unicode(col['name'], errors='replace')
+      col['name'] = smart_unicode(col['name'], errors='replace', encoding=encoding)
 
   elif file_format['inputFormat'] == 'table':
     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])

+ 2 - 2
desktop/libs/notebook/src/notebook/models.py

@@ -32,7 +32,7 @@ LOG = logging.getLogger(__name__)
 
 
 # Materialize and HTML escape results
-def escape_rows(rows, nulls_only=False):
+def escape_rows(rows, nulls_only=False, encoding=None):
   data = []
 
   for row in rows:
@@ -46,7 +46,7 @@ def escape_rows(rows, nulls_only=False):
       elif field is None:
         escaped_field = 'NULL'
       else:
-        escaped_field = smart_unicode(field, errors='replace') # Prevent error when getting back non utf8 like charset=iso-8859-1
+        escaped_field = smart_unicode(field, errors='replace', encoding=encoding) # Prevent error when getting back non utf8 like charset=iso-8859-1
         if not nulls_only:
           escaped_field = escape(escaped_field).replace(' ', ' ')
       escaped_row.append(escaped_field)