|
@@ -279,6 +279,20 @@ class CSVFormat(FileFormat):
|
|
|
]
|
|
]
|
|
|
_extensions = ["csv", "tsv"]
|
|
_extensions = ["csv", "tsv"]
|
|
|
|
|
|
|
|
|
|
+ def __init__(self, delimiter=',', line_terminator='\n', quote_char='"', has_header=False, sample="", fields=None):
|
|
|
|
|
+ self._delimiter = delimiter
|
|
|
|
|
+ self._line_terminator = line_terminator
|
|
|
|
|
+ self._quote_char = quote_char
|
|
|
|
|
+ self._has_header = has_header
|
|
|
|
|
+
|
|
|
|
|
+ # sniffer insists on \r\n even when \n. This is safer and good enough for a preview
|
|
|
|
|
+ self._line_terminator = self._line_terminator.replace("\r\n", "\n")
|
|
|
|
|
+ self._sample_rows = self._get_sample_rows(sample)
|
|
|
|
|
+ self._num_columns = self._guess_num_columns(self._sample_rows)
|
|
|
|
|
+ self._fields = fields if fields else self._guess_fields(sample)
|
|
|
|
|
+
|
|
|
|
|
+ super(CSVFormat, self).__init__()
|
|
|
|
|
+
|
|
|
@staticmethod
|
|
@staticmethod
|
|
|
def format_character(string):
|
|
def format_character(string):
|
|
|
string = string.replace('"', '\\"')
|
|
string = string.replace('"', '\\"')
|
|
@@ -347,7 +361,7 @@ class CSVFormat(FileFormat):
|
|
|
quote_char = format_["quoteChar"].encode('utf-8')
|
|
quote_char = format_["quoteChar"].encode('utf-8')
|
|
|
has_header = format_["hasHeader"]
|
|
has_header = format_["hasHeader"]
|
|
|
return cls(**{
|
|
return cls(**{
|
|
|
- "delimiter":delimiter,
|
|
|
|
|
|
|
+ "delimiter": delimiter,
|
|
|
"line_terminator": line_terminator,
|
|
"line_terminator": line_terminator,
|
|
|
"quote_char": quote_char,
|
|
"quote_char": quote_char,
|
|
|
"has_header": has_header,
|
|
"has_header": has_header,
|
|
@@ -361,23 +375,6 @@ class CSVFormat(FileFormat):
|
|
|
else:
|
|
else:
|
|
|
return cls._guess_from_file_stream(file_stream)
|
|
return cls._guess_from_file_stream(file_stream)
|
|
|
|
|
|
|
|
- def __init__(self, delimiter=',', line_terminator='\n', quote_char='"', has_header=False, sample="", fields=None):
|
|
|
|
|
- self._delimiter = delimiter
|
|
|
|
|
- self._line_terminator = line_terminator
|
|
|
|
|
- self._quote_char = quote_char
|
|
|
|
|
- self._has_header = has_header
|
|
|
|
|
-
|
|
|
|
|
- # sniffer insists on \r\n even when \n. This is safer and good enough for a preview
|
|
|
|
|
- self._line_terminator = self._line_terminator.replace("\r\n", "\n")
|
|
|
|
|
-
|
|
|
|
|
- self._sample_rows = self._get_sample_rows(sample)
|
|
|
|
|
-
|
|
|
|
|
- self._num_columns = self._guess_num_columns(self._sample_rows)
|
|
|
|
|
-
|
|
|
|
|
- self._fields = fields if fields else self._guess_fields(sample)
|
|
|
|
|
-
|
|
|
|
|
- super(CSVFormat, self).__init__()
|
|
|
|
|
-
|
|
|
|
|
@property
|
|
@property
|
|
|
def sample(self):
|
|
def sample(self):
|
|
|
return self._sample_rows
|
|
return self._sample_rows
|
|
@@ -401,10 +398,10 @@ class CSVFormat(FileFormat):
|
|
|
def get_format(self):
|
|
def get_format(self):
|
|
|
format_ = super(CSVFormat, self).get_format()
|
|
format_ = super(CSVFormat, self).get_format()
|
|
|
specific_format = {
|
|
specific_format = {
|
|
|
- "fieldSeparator":self.delimiter,
|
|
|
|
|
- "recordSeparator":self.line_terminator,
|
|
|
|
|
- "quoteChar":self.quote_char,
|
|
|
|
|
- "hasHeader":self._has_header
|
|
|
|
|
|
|
+ "fieldSeparator": self.delimiter,
|
|
|
|
|
+ "recordSeparator": self.line_terminator,
|
|
|
|
|
+ "quoteChar": self.quote_char,
|
|
|
|
|
+ "hasHeader": self._has_header
|
|
|
}
|
|
}
|
|
|
format_.update(specific_format)
|
|
format_.update(specific_format)
|
|
|
|
|
|
|
@@ -452,7 +449,7 @@ class CSVFormat(FileFormat):
|
|
|
if self._has_header:
|
|
if self._has_header:
|
|
|
header = first_row
|
|
header = first_row
|
|
|
else:
|
|
else:
|
|
|
- header = ["field_%d" % (i+1) for i in range(self._num_columns)]
|
|
|
|
|
|
|
+ header = ["field_%d" % (i + 1) for i in range(self._num_columns)]
|
|
|
|
|
|
|
|
return header
|
|
return header
|
|
|
|
|
|
|
@@ -479,6 +476,7 @@ class CSVFormat(FileFormat):
|
|
|
|
|
|
|
|
return fields
|
|
return fields
|
|
|
|
|
|
|
|
|
|
+
|
|
|
class HiveFormat(CSVFormat):
|
|
class HiveFormat(CSVFormat):
|
|
|
FIELD_TYPE_TRANSLATE = {
|
|
FIELD_TYPE_TRANSLATE = {
|
|
|
"BOOLEAN_TYPE": "string",
|
|
"BOOLEAN_TYPE": "string",
|
|
@@ -517,7 +515,7 @@ class HiveFormat(CSVFormat):
|
|
|
fields.append(Field(
|
|
fields.append(Field(
|
|
|
name=field["name"],
|
|
name=field["name"],
|
|
|
field_type_name=cls.FIELD_TYPE_TRANSLATE.get(field['type'], 'string')
|
|
field_type_name=cls.FIELD_TYPE_TRANSLATE.get(field['type'], 'string')
|
|
|
- ))
|
|
|
|
|
|
|
+ ))
|
|
|
|
|
|
|
|
return cls(**{
|
|
return cls(**{
|
|
|
"delimiter":',',
|
|
"delimiter":',',
|