Ver Fonte

HUE-4530 [indexer] Add hardcoded parquet format

peddle há 9 anos atrás
pai
commit
25dbd6d197

+ 41 - 0
desktop/libs/indexer/src/data/oozie_workspace/parse_parquet.conf

@@ -0,0 +1,41 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.from nose.tools import assert_equal
+<%page args="format_settings, format_class"/>
+
+{
+  readAvroParquetFile {
+  # Optionally, use this Avro schema in JSON format inline for projection:
+  # projectionSchemaString : """<json can go here>"""
+
+  # Optionally, use this Avro schema file in JSON format for projection:
+  # projectionSchemaFile : /path/to/syslog.avsc
+  }
+}
+
+{
+  extractAvroPaths {
+  flatten : false
+  paths : {
+    "customers.id" : /id
+    "customers.name" : /name
+    "customers.email_preferences" : "/email_preferences"
+    "customers.addresses" : "/addresses"
+    "customers.orders" : "/orders"
+  }
+}
+}
+
+{ logInfo { format : "Output from unpack {}", args : ["@{}"] } }

+ 4 - 2
desktop/libs/indexer/src/indexer/fields.py

@@ -39,13 +39,14 @@ class FieldType():
     return pattern.match(field)
 
 class Field(object):
-  def __init__(self, name="new_field", field_type_name="string", operations=None):
+  def __init__(self, name="new_field", field_type_name="string", operations=None, multi_valued=False):
     self.name = name
     self.field_type_name = field_type_name
     self.keep = True
     self.operations = operations if operations else []
     self.required = False
     self.unique = False
+    self.multi_valued = multi_valued
 
   def to_dict(self):
     return {'name': self.name,
@@ -53,7 +54,8 @@ class Field(object):
     'unique': self.unique,
     'keep': self.keep,
     'operations': self.operations,
-    'required': self.required}
+    'required': self.required,
+    'multiValued': self.multi_valued}
 
 FIELD_TYPES = [
   FieldType('text_en', "^[\\s\\S]*$", heuristic_regex="^[\\s\\S]{101,}$"),

+ 15 - 3
desktop/libs/indexer/src/indexer/file_format.py

@@ -32,9 +32,13 @@ def get_format_types():
     HueLogFormat,
     ApacheCombinedFormat,
     RubyLogFormat,
-    SyslogFormat
+    SyslogFormat,
+    ParquetTableFormat
   ]
 
+def get_file_indexable_format_types():
+  return [format_ for format_ in get_format_types() if format_.is_file_indexable]
+
 def _get_format_mapping():
   return dict([(format_.get_name(), format_) for format_ in get_format_types()])
 
@@ -67,6 +71,11 @@ class FileFormat(object):
   _args = []
   _extensions = []
   _parse_type = None
+  _file_indexable = True
+
+  @classmethod
+  def is_file_indexable(cls):
+    return cls._file_indexable
 
   @classmethod
   def get_extensions(cls):
@@ -252,6 +261,10 @@ class SyslogFormat(GrokLineFormat):
       Field("message", "text_en"),
     ]
 
+class ParquetTableFormat(FileFormat):
+  _name = "parquet"
+
+
 class CSVFormat(FileFormat):
   _name = "csv"
   _description = _("CSV File")
@@ -463,7 +476,6 @@ class CSVFormat(FileFormat):
 
     return fields
 
-
 class HiveFormat(CSVFormat):
   FIELD_TYPE_TRANSLATE = {
     "BOOLEAN_TYPE": "string",
@@ -490,7 +502,7 @@ class HiveFormat(CSVFormat):
       fields.append(Field(
         name=field["name"],
         field_type_name=cls.FIELD_TYPE_TRANSLATE.get(field['type'], 'string')
-      ))
+        ))
 
     return cls(**{
       "delimiter":',',

+ 3 - 2
desktop/libs/indexer/src/indexer/utils.py

@@ -41,13 +41,14 @@ from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYP
 
 LOG = logging.getLogger(__name__)
 TIMESTAMP_PATTERN = '\[([\w\d\s\-\/\:\+]*?)\]'
-FIELD_XML_TEMPLATE = '<field name="%(name)s" type="%(type)s" indexed="%(indexed)s" stored="%(stored)s" required="%(required)s" />'
+FIELD_XML_TEMPLATE = '<field name="%(name)s" type="%(type)s" indexed="%(indexed)s" stored="%(stored)s" required="%(required)s" multiValued="%(multiValued)s" />'
 DEFAULT_FIELD = {
   'name': None,
   'type': 'text',
   'indexed': 'true',
   'stored': 'true',
-  'required': 'true'
+  'required': 'true',
+  'multiValued': 'false'
 }
 
 

+ 2 - 2
desktop/libs/indexer/src/indexer/views.py

@@ -26,7 +26,7 @@ from indexer.controller2 import IndexController
 from indexer.management.commands import indexer_setup
 from indexer.fields import FIELD_TYPES, Field
 from indexer.operations import OPERATORS
-from indexer.file_format import get_format_types
+from indexer.file_format import get_file_indexable_format_types
 
 LOG = logging.getLogger(__name__)
 
@@ -57,7 +57,7 @@ def indexer(request):
       'indexes_json': json.dumps(indexes),
       'fields_json' : json.dumps([field.name for field in FIELD_TYPES]),
       'operators_json' : json.dumps([operator.to_dict() for operator in OPERATORS]),
-      'file_types_json' : json.dumps([format_.format_info() for format_ in get_format_types()]),
+      'file_types_json' : json.dumps([format_.format_info() for format_ in get_file_indexable_format_types()]),
       'default_field_type' : json.dumps(Field().to_dict())
   })