Эх сурвалжийг харах

HUE-4530 [indexer] Add parquet format and full column types for nested types

Romain Rigaux 9 жил өмнө
parent
commit
bb2c54b832

+ 5 - 3
desktop/libs/indexer/src/indexer/api3.py

@@ -100,13 +100,15 @@ def guess_field_types(request):
     })
   elif file_format['inputFormat'] == 'table':
     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
+    db = dbms.get(request.user)
+    table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
 
     format_ = {
         "sample": sample['rows'][:4],
         "columns": [
-            {"operations": [], "name": col['name'], "required": False, "keep": True, "unique": False,
-             "type": HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')
-         } for col in sample['full_headers']
+            {"operations": [], "name": col.name, "required": False, "keep": True, "unique": False,
+             "type": HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')
+         } for col in table_metadata.cols
         ]
     }
   elif file_format['inputFormat'] == 'query':

+ 4 - 2
desktop/libs/indexer/src/indexer/file_format.py

@@ -33,7 +33,7 @@ def get_format_types():
     ApacheCombinedFormat,
     RubyLogFormat,
     SyslogFormat,
-    ParquetTableFormat
+    ParquetFormat
   ]
 
 def get_file_indexable_format_types():
@@ -261,8 +261,10 @@ class SyslogFormat(GrokLineFormat):
       Field("message", "text_en"),
     ]
 
-class ParquetTableFormat(FileFormat):
+
+class ParquetFormat(FileFormat):
   _name = "parquet"
+  _description = _("Parquet Table")
 
 
 class CSVFormat(FileFormat):