Przeglądaj źródła

HUE-4544 [indexer] Extend Parquet morphline to support generic fields

peddle 9 lat temu
rodzic
commit
f134a3c

+ 1 - 1
desktop/libs/indexer/src/data/oozie_workspace/morphline_template.conf

@@ -42,7 +42,7 @@ morphlines : [
         }
       }
 
-      <%include file="${'parse_%s.conf' % format_class.get_parse_type()}", args="format_settings=format_settings, format_class=format_class"/>
+      <%include file="${'parse_%s.conf' % format_class.get_parse_type()}", args="format_settings=format_settings, format_class=format_class, fields=fields"/>
 
       % if uuid_name:
         {

+ 4 - 13
desktop/libs/indexer/src/data/oozie_workspace/parse_parquet.conf

@@ -13,15 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.from nose.tools import assert_equal
-<%page args="format_settings, format_class"/>
+<%page args="format_settings, format_class, fields"/>
 
 {
   readAvroParquetFile {
-  # Optionally, use this Avro schema in JSON format inline for projection:
-  # projectionSchemaString : """<json can go here>"""
-
-  # Optionally, use this Avro schema file in JSON format for projection:
-  # projectionSchemaFile : /path/to/syslog.avsc
   }
 }
 
@@ -29,13 +24,9 @@
   extractAvroPaths {
   flatten : false
   paths : {
-    "customers.id" : /id
-    "customers.name" : /name
-    "customers.email_preferences" : "/email_preferences"
-    "customers.addresses" : "/addresses"
-    "customers.orders" : "/orders"
+    %for field in fields:
+      "${field['name']}" : "/${field['name']}"
+    %endfor
   }
 }
 }
-
-{ logInfo { format : "Output from unpack {}", args : ["@{}"] } }

+ 3 - 4
desktop/libs/indexer/src/indexer/api3.py

@@ -28,7 +28,7 @@ from notebook.connectors.base import get_api
 from indexer.smart_indexer import Indexer
 from indexer.controller import CollectionManagerController
 from indexer.file_format import HiveFormat
-
+from indexer.fields import Field
 
 LOG = logging.getLogger(__name__)
 
@@ -106,9 +106,8 @@ def guess_field_types(request):
     format_ = {
         "sample": sample['rows'][:4],
         "columns": [
-            {"operations": [], "name": col.name, "required": False, "keep": True, "unique": False,
-             "type": HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')
-         } for col in table_metadata.cols
+            Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict()
+            for col in table_metadata.cols
         ]
     }
   elif file_format['inputFormat'] == 'query':