|
@@ -32,9 +32,13 @@ def get_format_types():
|
|
|
HueLogFormat,
|
|
HueLogFormat,
|
|
|
ApacheCombinedFormat,
|
|
ApacheCombinedFormat,
|
|
|
RubyLogFormat,
|
|
RubyLogFormat,
|
|
|
- SyslogFormat
|
|
|
|
|
|
|
+ SyslogFormat,
|
|
|
|
|
+ ParquetTableFormat
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
|
|
+def get_file_indexable_format_types():
|
|
|
|
|
+ return [format_ for format_ in get_format_types() if format_.is_file_indexable]
|
|
|
|
|
+
|
|
|
def _get_format_mapping():
|
|
def _get_format_mapping():
|
|
|
return dict([(format_.get_name(), format_) for format_ in get_format_types()])
|
|
return dict([(format_.get_name(), format_) for format_ in get_format_types()])
|
|
|
|
|
|
|
@@ -67,6 +71,11 @@ class FileFormat(object):
|
|
|
_args = []
|
|
_args = []
|
|
|
_extensions = []
|
|
_extensions = []
|
|
|
_parse_type = None
|
|
_parse_type = None
|
|
|
|
|
+ _file_indexable = True
|
|
|
|
|
+
|
|
|
|
|
+ @classmethod
|
|
|
|
|
+ def is_file_indexable(cls):
|
|
|
|
|
+ return cls._file_indexable
|
|
|
|
|
|
|
|
@classmethod
|
|
@classmethod
|
|
|
def get_extensions(cls):
|
|
def get_extensions(cls):
|
|
@@ -252,6 +261,10 @@ class SyslogFormat(GrokLineFormat):
|
|
|
Field("message", "text_en"),
|
|
Field("message", "text_en"),
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
|
|
+class ParquetTableFormat(FileFormat):
|
|
|
|
|
+ _name = "parquet"
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
class CSVFormat(FileFormat):
|
|
class CSVFormat(FileFormat):
|
|
|
_name = "csv"
|
|
_name = "csv"
|
|
|
_description = _("CSV File")
|
|
_description = _("CSV File")
|
|
@@ -463,7 +476,6 @@ class CSVFormat(FileFormat):
|
|
|
|
|
|
|
|
return fields
|
|
return fields
|
|
|
|
|
|
|
|
-
|
|
|
|
|
class HiveFormat(CSVFormat):
|
|
class HiveFormat(CSVFormat):
|
|
|
FIELD_TYPE_TRANSLATE = {
|
|
FIELD_TYPE_TRANSLATE = {
|
|
|
"BOOLEAN_TYPE": "string",
|
|
"BOOLEAN_TYPE": "string",
|
|
@@ -490,7 +502,7 @@ class HiveFormat(CSVFormat):
|
|
|
fields.append(Field(
|
|
fields.append(Field(
|
|
|
name=field["name"],
|
|
name=field["name"],
|
|
|
field_type_name=cls.FIELD_TYPE_TRANSLATE.get(field['type'], 'string')
|
|
field_type_name=cls.FIELD_TYPE_TRANSLATE.get(field['type'], 'string')
|
|
|
- ))
|
|
|
|
|
|
|
+ ))
|
|
|
|
|
|
|
|
return cls(**{
|
|
return cls(**{
|
|
|
"delimiter":',',
|
|
"delimiter":',',
|