Ver código fonte

HUE-8207 [indexer] Detect the encoding of the imported file

Romain Rigaux 7 anos atrás
pai
commit
fa5128b2b9

+ 2 - 2
desktop/core/src/desktop/lib/i18n.py

@@ -54,13 +54,13 @@ def validate_encoding(encoding):
   except LookupError:
   except LookupError:
     return False
     return False
 
 
-def smart_unicode(s, strings_only=False, errors='strict'):
+def smart_unicode(s, strings_only=False, errors='strict', encoding=None):
   """
   """
   Wrapper around Django's version, while supplying our configured encoding.
   Wrapper around Django's version, while supplying our configured encoding.
   Decode char array to unicode.
   Decode char array to unicode.
   """
   """
   return django.utils.encoding.smart_unicode(
   return django.utils.encoding.smart_unicode(
-        s, get_site_encoding(), strings_only, errors)
+        s, encoding if encoding is not None else get_site_encoding(), strings_only, errors)
 
 
 def force_unicode(s, strings_only=False, errors='strict'):
 def force_unicode(s, strings_only=False, errors='strict'):
   """
   """

+ 6 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
+import chardet
 import json
 import json
 import logging
 import logging
 
 
@@ -124,6 +125,8 @@ def guess_field_types(request):
   if file_format['inputFormat'] == 'file':
   if file_format['inputFormat'] == 'file':
     indexer = MorphlineIndexer(request.user, request.fs)
     indexer = MorphlineIndexer(request.user, request.fs)
     stream = request.fs.open(file_format["path"])
     stream = request.fs.open(file_format["path"])
+    encoding = chardet.detect(stream.read(10000)).get('encoding')
+    stream.seek(0)
     _convert_format(file_format["format"], inverse=True)
     _convert_format(file_format["format"], inverse=True)
 
 
     format_ = indexer.guess_field_types({
     format_ = indexer.guess_field_types({
@@ -134,10 +137,11 @@ def guess_field_types(request):
       "format": file_format['format']
       "format": file_format['format']
     })
     })
 
 
+    # Note: Would also need to set charset to table (only supported in Hive)
     if 'sample' in format_:
     if 'sample' in format_:
-      format_['sample'] = escape_rows(format_['sample'], nulls_only=True)
+      format_['sample'] = escape_rows(format_['sample'], nulls_only=True, encoding=encoding)
     for col in format_['columns']:
     for col in format_['columns']:
-      col['name'] = smart_unicode(col['name'], errors='replace')
+      col['name'] = smart_unicode(col['name'], errors='replace', encoding=encoding)
 
 
   elif file_format['inputFormat'] == 'table':
   elif file_format['inputFormat'] == 'table':
     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])

+ 2 - 2
desktop/libs/notebook/src/notebook/models.py

@@ -32,7 +32,7 @@ LOG = logging.getLogger(__name__)
 
 
 
 
 # Materialize and HTML escape results
 # Materialize and HTML escape results
-def escape_rows(rows, nulls_only=False):
+def escape_rows(rows, nulls_only=False, encoding=None):
   data = []
   data = []
 
 
   for row in rows:
   for row in rows:
@@ -46,7 +46,7 @@ def escape_rows(rows, nulls_only=False):
       elif field is None:
       elif field is None:
         escaped_field = 'NULL'
         escaped_field = 'NULL'
       else:
       else:
-        escaped_field = smart_unicode(field, errors='replace') # Prevent error when getting back non utf8 like charset=iso-8859-1
+        escaped_field = smart_unicode(field, errors='replace', encoding=encoding) # Prevent error when getting back non utf8 like charset=iso-8859-1
         if not nulls_only:
         if not nulls_only:
           escaped_field = escape(escaped_field).replace(' ', ' ')
           escaped_field = escape(escaped_field).replace(' ', ' ')
       escaped_row.append(escaped_field)
       escaped_row.append(escaped_field)