ソースを参照

HUE-4530 [indexer] Get full table metadata to get the correct field types

Romain Rigaux 9 年 前
コミット
dbc5f4e

+ 1 - 0
apps/beeswax/src/beeswax/api.py

@@ -645,6 +645,7 @@ def _get_sample_data(db, database, table, column):
 
     response['status'] = 0
     response['headers'] = sample_data.cols()
+    response['full_headers'] = sample_data.full_cols()
     response['rows'] = sample
   else:
     response['message'] = _('Failed to get sample data.')

+ 3 - 0
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -977,6 +977,9 @@ class ResultCompatible:
   def cols(self):
     return [col.name for col in self.data_table.cols()]
 
+  def full_cols(self):
+    return [{'name': col.name, 'type': col.type, 'comment': col.comment} for col in self.data_table.cols()]
+
 
 class PartitionKeyCompatible:
 

+ 16 - 8
desktop/libs/indexer/src/indexer/api3.py

@@ -24,8 +24,8 @@ from desktop.lib.django_util import JsonResponse
 
 from indexer.smart_indexer import Indexer
 from indexer.controller import CollectionManagerController
-from notebook.api import get_sample_data
 from notebook.connectors.base import get_api
+from indexer.file_format import HiveFormat
 
 LOG = logging.getLogger(__name__)
 
@@ -80,19 +80,27 @@ def guess_field_types(request):
     _convert_format(file_format["format"], inverse=True)
 
     format_ = indexer.guess_field_types({
-      "file":{
-        "stream":stream,
-        "name":file_format['path']
+      "file": {
+        "stream": stream,
+        "name": file_format['path']
         },
-      "format":file_format['format']
+      "format": file_format['format']
     })
   elif file_format['inputFormat'] == 'table':
-    # TODO get type metadata
-    sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database='default', table='sample_07')
+    if '.' in file_format["table"]:
+      database, table = file_format["table"].lsplit('.', 1)
+    else:
+      database = 'default'
+      table = file_format["table"]
+
+    sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=database, table=table)
+
     format_ = {
         "sample": sample['rows'][:4],
         "columns": [
-            {"operations": [], "name": col, "required": False, "keep": True, "unique": False, "type": "string"} for col in sample['headers']
+            {"operations": [], "name": col['name'], "required": False, "keep": True, "unique": False,
+             "type": HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')
+         } for col in sample['full_headers']
         ]
     }
   elif file_format['inputFormat'] == 'query':

+ 2 - 1
desktop/libs/indexer/src/indexer/file_format.py

@@ -463,6 +463,7 @@ class CSVFormat(FileFormat):
 
     return fields
 
+
 class HiveFormat(CSVFormat):
   FIELD_TYPE_TRANSLATE = {
     "BOOLEAN_TYPE": "string",
@@ -489,7 +490,7 @@ class HiveFormat(CSVFormat):
       fields.append(Field(
         name=field["name"],
         field_type_name=cls.FIELD_TYPE_TRANSLATE.get(field['type'], 'string')
-        ))
+      ))
 
     return cls(**{
       "delimiter":',',