|
@@ -0,0 +1,1424 @@
|
|
|
|
|
+#
|
|
|
|
|
+# Autogenerated by Thrift Compiler (0.9.0)
|
|
|
|
|
+#
|
|
|
|
|
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
|
|
|
|
+#
|
|
|
|
|
+# options string: py
|
|
|
|
|
+#
|
|
|
|
|
+
|
|
|
|
|
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
|
|
|
|
|
+
|
|
|
|
|
+from thrift.transport import TTransport
|
|
|
|
|
+from thrift.protocol import TBinaryProtocol, TProtocol
|
|
|
|
|
+try:
|
|
|
|
|
+ from thrift.protocol import fastbinary
|
|
|
|
|
+except:
|
|
|
|
|
+ fastbinary = None
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class Type:
|
|
|
|
|
+ """
|
|
|
|
|
+ Types supported by Parquet. These types are intended to be used in combination
|
|
|
|
|
+ with the encodings to control the on disk storage format.
|
|
|
|
|
+ For example INT16 is not included as a type since a good encoding of INT32
|
|
|
|
|
+ would handle this.
|
|
|
|
|
+ """
|
|
|
|
|
+ BOOLEAN = 0
|
|
|
|
|
+ INT32 = 1
|
|
|
|
|
+ INT64 = 2
|
|
|
|
|
+ INT96 = 3
|
|
|
|
|
+ FLOAT = 4
|
|
|
|
|
+ DOUBLE = 5
|
|
|
|
|
+ BYTE_ARRAY = 6
|
|
|
|
|
+ FIXED_LEN_BYTE_ARRAY = 7
|
|
|
|
|
+
|
|
|
|
|
+ _VALUES_TO_NAMES = {
|
|
|
|
|
+ 0: "BOOLEAN",
|
|
|
|
|
+ 1: "INT32",
|
|
|
|
|
+ 2: "INT64",
|
|
|
|
|
+ 3: "INT96",
|
|
|
|
|
+ 4: "FLOAT",
|
|
|
|
|
+ 5: "DOUBLE",
|
|
|
|
|
+ 6: "BYTE_ARRAY",
|
|
|
|
|
+ 7: "FIXED_LEN_BYTE_ARRAY",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ _NAMES_TO_VALUES = {
|
|
|
|
|
+ "BOOLEAN": 0,
|
|
|
|
|
+ "INT32": 1,
|
|
|
|
|
+ "INT64": 2,
|
|
|
|
|
+ "INT96": 3,
|
|
|
|
|
+ "FLOAT": 4,
|
|
|
|
|
+ "DOUBLE": 5,
|
|
|
|
|
+ "BYTE_ARRAY": 6,
|
|
|
|
|
+ "FIXED_LEN_BYTE_ARRAY": 7,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+class ConvertedType:
|
|
|
|
|
+ """
|
|
|
|
|
+ Common types used by frameworks(e.g. hive, pig) using parquet. This helps map
|
|
|
|
|
+ between types in those frameworks to the base types in parquet. This is only
|
|
|
|
|
+ metadata and not needed to read or write the data.
|
|
|
|
|
+ """
|
|
|
|
|
+ UTF8 = 0
|
|
|
|
|
+ MAP = 1
|
|
|
|
|
+ MAP_KEY_VALUE = 2
|
|
|
|
|
+ LIST = 3
|
|
|
|
|
+
|
|
|
|
|
+ _VALUES_TO_NAMES = {
|
|
|
|
|
+ 0: "UTF8",
|
|
|
|
|
+ 1: "MAP",
|
|
|
|
|
+ 2: "MAP_KEY_VALUE",
|
|
|
|
|
+ 3: "LIST",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ _NAMES_TO_VALUES = {
|
|
|
|
|
+ "UTF8": 0,
|
|
|
|
|
+ "MAP": 1,
|
|
|
|
|
+ "MAP_KEY_VALUE": 2,
|
|
|
|
|
+ "LIST": 3,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+class FieldRepetitionType:
|
|
|
|
|
+ """
|
|
|
|
|
+ Representation of Schemas
|
|
|
|
|
+ """
|
|
|
|
|
+ REQUIRED = 0
|
|
|
|
|
+ OPTIONAL = 1
|
|
|
|
|
+ REPEATED = 2
|
|
|
|
|
+
|
|
|
|
|
+ _VALUES_TO_NAMES = {
|
|
|
|
|
+ 0: "REQUIRED",
|
|
|
|
|
+ 1: "OPTIONAL",
|
|
|
|
|
+ 2: "REPEATED",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ _NAMES_TO_VALUES = {
|
|
|
|
|
+ "REQUIRED": 0,
|
|
|
|
|
+ "OPTIONAL": 1,
|
|
|
|
|
+ "REPEATED": 2,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+class Encoding:
|
|
|
|
|
+ """
|
|
|
|
|
+ Encodings supported by Parquet. Not all encodings are valid for all types. These
|
|
|
|
|
+ enums are also used to specify the encoding of definition and repetition levels.
|
|
|
|
|
+ See the accompanying doc for the details of the more complicated encodings.
|
|
|
|
|
+ """
|
|
|
|
|
+ PLAIN = 0
|
|
|
|
|
+ GROUP_VAR_INT = 1
|
|
|
|
|
+ PLAIN_DICTIONARY = 2
|
|
|
|
|
+ RLE = 3
|
|
|
|
|
+ BIT_PACKED = 4
|
|
|
|
|
+
|
|
|
|
|
+ _VALUES_TO_NAMES = {
|
|
|
|
|
+ 0: "PLAIN",
|
|
|
|
|
+ 1: "GROUP_VAR_INT",
|
|
|
|
|
+ 2: "PLAIN_DICTIONARY",
|
|
|
|
|
+ 3: "RLE",
|
|
|
|
|
+ 4: "BIT_PACKED",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ _NAMES_TO_VALUES = {
|
|
|
|
|
+ "PLAIN": 0,
|
|
|
|
|
+ "GROUP_VAR_INT": 1,
|
|
|
|
|
+ "PLAIN_DICTIONARY": 2,
|
|
|
|
|
+ "RLE": 3,
|
|
|
|
|
+ "BIT_PACKED": 4,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+class CompressionCodec:
|
|
|
|
|
+ """
|
|
|
|
|
+ Supported compression algorithms.
|
|
|
|
|
+ """
|
|
|
|
|
+ UNCOMPRESSED = 0
|
|
|
|
|
+ SNAPPY = 1
|
|
|
|
|
+ GZIP = 2
|
|
|
|
|
+ LZO = 3
|
|
|
|
|
+
|
|
|
|
|
+ _VALUES_TO_NAMES = {
|
|
|
|
|
+ 0: "UNCOMPRESSED",
|
|
|
|
|
+ 1: "SNAPPY",
|
|
|
|
|
+ 2: "GZIP",
|
|
|
|
|
+ 3: "LZO",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ _NAMES_TO_VALUES = {
|
|
|
|
|
+ "UNCOMPRESSED": 0,
|
|
|
|
|
+ "SNAPPY": 1,
|
|
|
|
|
+ "GZIP": 2,
|
|
|
|
|
+ "LZO": 3,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+class PageType:
|
|
|
|
|
+ DATA_PAGE = 0
|
|
|
|
|
+ INDEX_PAGE = 1
|
|
|
|
|
+ DICTIONARY_PAGE = 2
|
|
|
|
|
+
|
|
|
|
|
+ _VALUES_TO_NAMES = {
|
|
|
|
|
+ 0: "DATA_PAGE",
|
|
|
|
|
+ 1: "INDEX_PAGE",
|
|
|
|
|
+ 2: "DICTIONARY_PAGE",
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ _NAMES_TO_VALUES = {
|
|
|
|
|
+ "DATA_PAGE": 0,
|
|
|
|
|
+ "INDEX_PAGE": 1,
|
|
|
|
|
+ "DICTIONARY_PAGE": 2,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class SchemaElement:
|
|
|
|
|
+ """
|
|
|
|
|
+ Represents a element inside a schema definition.
|
|
|
|
|
+ - if it is a group (inner node) then type is undefined and num_children is defined
|
|
|
|
|
+ - if it is a primitive type (leaf) then type is defined and num_children is undefined
|
|
|
|
|
+ the nodes are listed in depth first traversal order.
|
|
|
|
|
+
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - type: Data type for this field. Not set if the current element is a non-leaf node
|
|
|
|
|
+ - type_length: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
|
|
|
|
|
+ Otherwise, if specified, this is the maximum bit length to store any of the values.
|
|
|
|
|
+ (e.g. a low cardinality INT col could have this set to 3). Note that this is
|
|
|
|
|
+ in the schema, and therefore fixed for the entire file.
|
|
|
|
|
+ - repetition_type: repetition of the field. The root of the schema does not have a repetition_type.
|
|
|
|
|
+ All other nodes must have one
|
|
|
|
|
+ - name: Name of the field in the schema
|
|
|
|
|
+ - num_children: Nested fields. Since thrift does not support nested fields,
|
|
|
|
|
+ the nesting is flattened to a single list by a depth-first traversal.
|
|
|
|
|
+ The children count is used to construct the nested relationship.
|
|
|
|
|
+ This field is not set when the element is a primitive type
|
|
|
|
|
+ - converted_type: When the schema is the result of a conversion from another model
|
|
|
|
|
+ Used to record the original type to help with cross conversion.
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.I32, 'type', None, None, ), # 1
|
|
|
|
|
+ (2, TType.I32, 'type_length', None, None, ), # 2
|
|
|
|
|
+ (3, TType.I32, 'repetition_type', None, None, ), # 3
|
|
|
|
|
+ (4, TType.STRING, 'name', None, None, ), # 4
|
|
|
|
|
+ (5, TType.I32, 'num_children', None, None, ), # 5
|
|
|
|
|
+ (6, TType.I32, 'converted_type', None, None, ), # 6
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, type=None, type_length=None, repetition_type=None, name=None, num_children=None, converted_type=None,):
|
|
|
|
|
+ self.type = type
|
|
|
|
|
+ self.type_length = type_length
|
|
|
|
|
+ self.repetition_type = repetition_type
|
|
|
|
|
+ self.name = name
|
|
|
|
|
+ self.num_children = num_children
|
|
|
|
|
+ self.converted_type = converted_type
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.type = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.type_length = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 3:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.repetition_type = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 4:
|
|
|
|
|
+ if ftype == TType.STRING:
|
|
|
|
|
+ self.name = iprot.readString();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 5:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.num_children = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 6:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.converted_type = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('SchemaElement')
|
|
|
|
|
+ if self.type is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('type', TType.I32, 1)
|
|
|
|
|
+ oprot.writeI32(self.type)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.type_length is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('type_length', TType.I32, 2)
|
|
|
|
|
+ oprot.writeI32(self.type_length)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.repetition_type is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('repetition_type', TType.I32, 3)
|
|
|
|
|
+ oprot.writeI32(self.repetition_type)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.name is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('name', TType.STRING, 4)
|
|
|
|
|
+ oprot.writeString(self.name)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.num_children is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('num_children', TType.I32, 5)
|
|
|
|
|
+ oprot.writeI32(self.num_children)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.converted_type is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('converted_type', TType.I32, 6)
|
|
|
|
|
+ oprot.writeI32(self.converted_type)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.name is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field name is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class DataPageHeader:
|
|
|
|
|
+ """
|
|
|
|
|
+ Data page header
|
|
|
|
|
+
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - num_values: Number of values, including NULLs, in this data page. *
|
|
|
|
|
+ - encoding: Encoding used for this data page *
|
|
|
|
|
+ - definition_level_encoding: Encoding used for definition levels *
|
|
|
|
|
+ - repetition_level_encoding: Encoding used for repetition levels *
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.I32, 'num_values', None, None, ), # 1
|
|
|
|
|
+ (2, TType.I32, 'encoding', None, None, ), # 2
|
|
|
|
|
+ (3, TType.I32, 'definition_level_encoding', None, None, ), # 3
|
|
|
|
|
+ (4, TType.I32, 'repetition_level_encoding', None, None, ), # 4
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, num_values=None, encoding=None, definition_level_encoding=None, repetition_level_encoding=None,):
|
|
|
|
|
+ self.num_values = num_values
|
|
|
|
|
+ self.encoding = encoding
|
|
|
|
|
+ self.definition_level_encoding = definition_level_encoding
|
|
|
|
|
+ self.repetition_level_encoding = repetition_level_encoding
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.num_values = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.encoding = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 3:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.definition_level_encoding = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 4:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.repetition_level_encoding = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('DataPageHeader')
|
|
|
|
|
+ if self.num_values is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('num_values', TType.I32, 1)
|
|
|
|
|
+ oprot.writeI32(self.num_values)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.encoding is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('encoding', TType.I32, 2)
|
|
|
|
|
+ oprot.writeI32(self.encoding)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.definition_level_encoding is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('definition_level_encoding', TType.I32, 3)
|
|
|
|
|
+ oprot.writeI32(self.definition_level_encoding)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.repetition_level_encoding is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('repetition_level_encoding', TType.I32, 4)
|
|
|
|
|
+ oprot.writeI32(self.repetition_level_encoding)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.num_values is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field num_values is unset!')
|
|
|
|
|
+ if self.encoding is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field encoding is unset!')
|
|
|
|
|
+ if self.definition_level_encoding is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field definition_level_encoding is unset!')
|
|
|
|
|
+ if self.repetition_level_encoding is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field repetition_level_encoding is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class IndexPageHeader:
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('IndexPageHeader')
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class DictionaryPageHeader:
|
|
|
|
|
+ """
|
|
|
|
|
+ TODO: *
|
|
|
|
|
+
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - num_values: Number of values in the dictionary *
|
|
|
|
|
+ - encoding: Encoding using this dictionary page *
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.I32, 'num_values', None, None, ), # 1
|
|
|
|
|
+ (2, TType.I32, 'encoding', None, None, ), # 2
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, num_values=None, encoding=None,):
|
|
|
|
|
+ self.num_values = num_values
|
|
|
|
|
+ self.encoding = encoding
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.num_values = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.encoding = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('DictionaryPageHeader')
|
|
|
|
|
+ if self.num_values is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('num_values', TType.I32, 1)
|
|
|
|
|
+ oprot.writeI32(self.num_values)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.encoding is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('encoding', TType.I32, 2)
|
|
|
|
|
+ oprot.writeI32(self.encoding)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.num_values is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field num_values is unset!')
|
|
|
|
|
+ if self.encoding is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field encoding is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class PageHeader:
|
|
|
|
|
+ """
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - type: the type of the page: indicates which of the *_header fields is set *
|
|
|
|
|
+ - uncompressed_page_size: Uncompressed page size in bytes (not including this header) *
|
|
|
|
|
+ - compressed_page_size: Compressed page size in bytes (not including this header) *
|
|
|
|
|
+ - crc: 32bit crc for the data below. This allows for disabling checksumming in HDFS
|
|
|
|
|
+ if only a few pages needs to be read
|
|
|
|
|
+
|
|
|
|
|
+ - data_page_header
|
|
|
|
|
+ - index_page_header
|
|
|
|
|
+ - dictionary_page_header
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.I32, 'type', None, None, ), # 1
|
|
|
|
|
+ (2, TType.I32, 'uncompressed_page_size', None, None, ), # 2
|
|
|
|
|
+ (3, TType.I32, 'compressed_page_size', None, None, ), # 3
|
|
|
|
|
+ (4, TType.I32, 'crc', None, None, ), # 4
|
|
|
|
|
+ (5, TType.STRUCT, 'data_page_header', (DataPageHeader, DataPageHeader.thrift_spec), None, ), # 5
|
|
|
|
|
+ (6, TType.STRUCT, 'index_page_header', (IndexPageHeader, IndexPageHeader.thrift_spec), None, ), # 6
|
|
|
|
|
+ (7, TType.STRUCT, 'dictionary_page_header', (DictionaryPageHeader, DictionaryPageHeader.thrift_spec), None, ), # 7
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, type=None, uncompressed_page_size=None, compressed_page_size=None, crc=None, data_page_header=None, index_page_header=None, dictionary_page_header=None,):
|
|
|
|
|
+ self.type = type
|
|
|
|
|
+ self.uncompressed_page_size = uncompressed_page_size
|
|
|
|
|
+ self.compressed_page_size = compressed_page_size
|
|
|
|
|
+ self.crc = crc
|
|
|
|
|
+ self.data_page_header = data_page_header
|
|
|
|
|
+ self.index_page_header = index_page_header
|
|
|
|
|
+ self.dictionary_page_header = dictionary_page_header
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.type = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.uncompressed_page_size = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 3:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.compressed_page_size = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 4:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.crc = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 5:
|
|
|
|
|
+ if ftype == TType.STRUCT:
|
|
|
|
|
+ self.data_page_header = DataPageHeader()
|
|
|
|
|
+ self.data_page_header.read(iprot)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 6:
|
|
|
|
|
+ if ftype == TType.STRUCT:
|
|
|
|
|
+ self.index_page_header = IndexPageHeader()
|
|
|
|
|
+ self.index_page_header.read(iprot)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 7:
|
|
|
|
|
+ if ftype == TType.STRUCT:
|
|
|
|
|
+ self.dictionary_page_header = DictionaryPageHeader()
|
|
|
|
|
+ self.dictionary_page_header.read(iprot)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('PageHeader')
|
|
|
|
|
+ if self.type is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('type', TType.I32, 1)
|
|
|
|
|
+ oprot.writeI32(self.type)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.uncompressed_page_size is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('uncompressed_page_size', TType.I32, 2)
|
|
|
|
|
+ oprot.writeI32(self.uncompressed_page_size)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.compressed_page_size is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('compressed_page_size', TType.I32, 3)
|
|
|
|
|
+ oprot.writeI32(self.compressed_page_size)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.crc is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('crc', TType.I32, 4)
|
|
|
|
|
+ oprot.writeI32(self.crc)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.data_page_header is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('data_page_header', TType.STRUCT, 5)
|
|
|
|
|
+ self.data_page_header.write(oprot)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.index_page_header is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('index_page_header', TType.STRUCT, 6)
|
|
|
|
|
+ self.index_page_header.write(oprot)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.dictionary_page_header is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('dictionary_page_header', TType.STRUCT, 7)
|
|
|
|
|
+ self.dictionary_page_header.write(oprot)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.type is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field type is unset!')
|
|
|
|
|
+ if self.uncompressed_page_size is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field uncompressed_page_size is unset!')
|
|
|
|
|
+ if self.compressed_page_size is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field compressed_page_size is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class KeyValue:
|
|
|
|
|
+ """
|
|
|
|
|
+ Wrapper struct to store key values
|
|
|
|
|
+
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - key
|
|
|
|
|
+ - value
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.STRING, 'key', None, None, ), # 1
|
|
|
|
|
+ (2, TType.STRING, 'value', None, None, ), # 2
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, key=None, value=None,):
|
|
|
|
|
+ self.key = key
|
|
|
|
|
+ self.value = value
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.STRING:
|
|
|
|
|
+ self.key = iprot.readString();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.STRING:
|
|
|
|
|
+ self.value = iprot.readString();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('KeyValue')
|
|
|
|
|
+ if self.key is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('key', TType.STRING, 1)
|
|
|
|
|
+ oprot.writeString(self.key)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.value is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('value', TType.STRING, 2)
|
|
|
|
|
+ oprot.writeString(self.value)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.key is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field key is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class SortingColumn:
|
|
|
|
|
+ """
|
|
|
|
|
+ Wrapper struct to specify sort order
|
|
|
|
|
+
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - nulls_first: The column index (in this row group)
|
|
|
|
|
+ 1: required i32 column_idx
|
|
|
|
|
+
|
|
|
|
|
+ /** If true, indicates this column is sorted in descending order.
|
|
|
|
|
+ 2: required bool descending
|
|
|
|
|
+
|
|
|
|
|
+ /** If true, nulls will come before non-null values, otherwise,
|
|
|
|
|
+ * nulls go at the end.
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ None, # 1
|
|
|
|
|
+ None, # 2
|
|
|
|
|
+ (3, TType.BOOL, 'nulls_first', None, None, ), # 3
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, nulls_first=None,):
|
|
|
|
|
+ self.nulls_first = nulls_first
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 3:
|
|
|
|
|
+ if ftype == TType.BOOL:
|
|
|
|
|
+ self.nulls_first = iprot.readBool();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('SortingColumn')
|
|
|
|
|
+ if self.nulls_first is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('nulls_first', TType.BOOL, 3)
|
|
|
|
|
+ oprot.writeBool(self.nulls_first)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.nulls_first is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field nulls_first is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class ColumnMetaData:
|
|
|
|
|
+ """
|
|
|
|
|
+ Description for column metadata
|
|
|
|
|
+
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - type: Type of this column *
|
|
|
|
|
+ - encodings: Set of all encodings used for this column. The purpose is to validate
|
|
|
|
|
+ whether we can decode those pages. *
|
|
|
|
|
+ - path_in_schema: Path in schema *
|
|
|
|
|
+ - codec: Compression codec *
|
|
|
|
|
+ - num_values: Number of values in this column *
|
|
|
|
|
+ - total_uncompressed_size: total byte size of all uncompressed pages in this column chunk (including the headers) *
|
|
|
|
|
+ - total_compressed_size: total byte size of all compressed pages in this column chunk (including the headers) *
|
|
|
|
|
+ - key_value_metadata: Optional key/value metadata *
|
|
|
|
|
+ - data_page_offset: Byte offset from beginning of file to first data page *
|
|
|
|
|
+ - index_page_offset: Byte offset from beginning of file to root index page *
|
|
|
|
|
+ - dictionary_page_offset: Byte offset from the beginning of file to first (only) dictionary page *
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.I32, 'type', None, None, ), # 1
|
|
|
|
|
+ (2, TType.LIST, 'encodings', (TType.I32,None), None, ), # 2
|
|
|
|
|
+ (3, TType.LIST, 'path_in_schema', (TType.STRING,None), None, ), # 3
|
|
|
|
|
+ (4, TType.I32, 'codec', None, None, ), # 4
|
|
|
|
|
+ (5, TType.I64, 'num_values', None, None, ), # 5
|
|
|
|
|
+ (6, TType.I64, 'total_uncompressed_size', None, None, ), # 6
|
|
|
|
|
+ (7, TType.I64, 'total_compressed_size', None, None, ), # 7
|
|
|
|
|
+ (8, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 8
|
|
|
|
|
+ (9, TType.I64, 'data_page_offset', None, None, ), # 9
|
|
|
|
|
+ (10, TType.I64, 'index_page_offset', None, None, ), # 10
|
|
|
|
|
+ (11, TType.I64, 'dictionary_page_offset', None, None, ), # 11
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, type=None, encodings=None, path_in_schema=None, codec=None, num_values=None, total_uncompressed_size=None, total_compressed_size=None, key_value_metadata=None, data_page_offset=None, index_page_offset=None, dictionary_page_offset=None,):
|
|
|
|
|
+ self.type = type
|
|
|
|
|
+ self.encodings = encodings
|
|
|
|
|
+ self.path_in_schema = path_in_schema
|
|
|
|
|
+ self.codec = codec
|
|
|
|
|
+ self.num_values = num_values
|
|
|
|
|
+ self.total_uncompressed_size = total_uncompressed_size
|
|
|
|
|
+ self.total_compressed_size = total_compressed_size
|
|
|
|
|
+ self.key_value_metadata = key_value_metadata
|
|
|
|
|
+ self.data_page_offset = data_page_offset
|
|
|
|
|
+ self.index_page_offset = index_page_offset
|
|
|
|
|
+ self.dictionary_page_offset = dictionary_page_offset
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.type = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.encodings = []
|
|
|
|
|
+ (_etype3, _size0) = iprot.readListBegin()
|
|
|
|
|
+ for _i4 in xrange(_size0):
|
|
|
|
|
+ _elem5 = iprot.readI32();
|
|
|
|
|
+ self.encodings.append(_elem5)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 3:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.path_in_schema = []
|
|
|
|
|
+ (_etype9, _size6) = iprot.readListBegin()
|
|
|
|
|
+ for _i10 in xrange(_size6):
|
|
|
|
|
+ _elem11 = iprot.readString();
|
|
|
|
|
+ self.path_in_schema.append(_elem11)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 4:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.codec = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 5:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.num_values = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 6:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.total_uncompressed_size = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 7:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.total_compressed_size = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 8:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.key_value_metadata = []
|
|
|
|
|
+ (_etype15, _size12) = iprot.readListBegin()
|
|
|
|
|
+ for _i16 in xrange(_size12):
|
|
|
|
|
+ _elem17 = KeyValue()
|
|
|
|
|
+ _elem17.read(iprot)
|
|
|
|
|
+ self.key_value_metadata.append(_elem17)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 9:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.data_page_offset = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 10:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.index_page_offset = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 11:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.dictionary_page_offset = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('ColumnMetaData')
|
|
|
|
|
+ if self.type is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('type', TType.I32, 1)
|
|
|
|
|
+ oprot.writeI32(self.type)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.encodings is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('encodings', TType.LIST, 2)
|
|
|
|
|
+ oprot.writeListBegin(TType.I32, len(self.encodings))
|
|
|
|
|
+ for iter18 in self.encodings:
|
|
|
|
|
+ oprot.writeI32(iter18)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.path_in_schema is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('path_in_schema', TType.LIST, 3)
|
|
|
|
|
+ oprot.writeListBegin(TType.STRING, len(self.path_in_schema))
|
|
|
|
|
+ for iter19 in self.path_in_schema:
|
|
|
|
|
+ oprot.writeString(iter19)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.codec is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('codec', TType.I32, 4)
|
|
|
|
|
+ oprot.writeI32(self.codec)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.num_values is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('num_values', TType.I64, 5)
|
|
|
|
|
+ oprot.writeI64(self.num_values)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.total_uncompressed_size is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('total_uncompressed_size', TType.I64, 6)
|
|
|
|
|
+ oprot.writeI64(self.total_uncompressed_size)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.total_compressed_size is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('total_compressed_size', TType.I64, 7)
|
|
|
|
|
+ oprot.writeI64(self.total_compressed_size)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.key_value_metadata is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('key_value_metadata', TType.LIST, 8)
|
|
|
|
|
+ oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
|
|
|
|
|
+ for iter20 in self.key_value_metadata:
|
|
|
|
|
+ iter20.write(oprot)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.data_page_offset is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('data_page_offset', TType.I64, 9)
|
|
|
|
|
+ oprot.writeI64(self.data_page_offset)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.index_page_offset is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('index_page_offset', TType.I64, 10)
|
|
|
|
|
+ oprot.writeI64(self.index_page_offset)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.dictionary_page_offset is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('dictionary_page_offset', TType.I64, 11)
|
|
|
|
|
+ oprot.writeI64(self.dictionary_page_offset)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.type is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field type is unset!')
|
|
|
|
|
+ if self.encodings is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field encodings is unset!')
|
|
|
|
|
+ if self.path_in_schema is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field path_in_schema is unset!')
|
|
|
|
|
+ if self.codec is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field codec is unset!')
|
|
|
|
|
+ if self.num_values is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field num_values is unset!')
|
|
|
|
|
+ if self.total_uncompressed_size is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field total_uncompressed_size is unset!')
|
|
|
|
|
+ if self.total_compressed_size is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field total_compressed_size is unset!')
|
|
|
|
|
+ if self.data_page_offset is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field data_page_offset is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class ColumnChunk:
|
|
|
|
|
+ """
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - file_path: File where column data is stored. If not set, assumed to be same file as
|
|
|
|
|
+ metadata. This path is relative to the current file.
|
|
|
|
|
+
|
|
|
|
|
+ - file_offset: Byte offset in file_path to the ColumnMetaData *
|
|
|
|
|
+ - meta_data: Column metadata for this chunk. This is the same content as what is at
|
|
|
|
|
+ file_path/file_offset. Having it here has it replicated in the file
|
|
|
|
|
+ metadata.
|
|
|
|
|
+
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.STRING, 'file_path', None, None, ), # 1
|
|
|
|
|
+ (2, TType.I64, 'file_offset', None, None, ), # 2
|
|
|
|
|
+ (3, TType.STRUCT, 'meta_data', (ColumnMetaData, ColumnMetaData.thrift_spec), None, ), # 3
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, file_path=None, file_offset=None, meta_data=None,):
|
|
|
|
|
+ self.file_path = file_path
|
|
|
|
|
+ self.file_offset = file_offset
|
|
|
|
|
+ self.meta_data = meta_data
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.STRING:
|
|
|
|
|
+ self.file_path = iprot.readString();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.file_offset = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 3:
|
|
|
|
|
+ if ftype == TType.STRUCT:
|
|
|
|
|
+ self.meta_data = ColumnMetaData()
|
|
|
|
|
+ self.meta_data.read(iprot)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('ColumnChunk')
|
|
|
|
|
+ if self.file_path is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('file_path', TType.STRING, 1)
|
|
|
|
|
+ oprot.writeString(self.file_path)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.file_offset is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('file_offset', TType.I64, 2)
|
|
|
|
|
+ oprot.writeI64(self.file_offset)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.meta_data is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('meta_data', TType.STRUCT, 3)
|
|
|
|
|
+ self.meta_data.write(oprot)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.file_offset is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field file_offset is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class RowGroup:
|
|
|
|
|
+ """
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - columns
|
|
|
|
|
+ - total_byte_size: Total byte size of all the uncompressed column data in this row group *
|
|
|
|
|
+ - num_rows: Number of rows in this row group *
|
|
|
|
|
+ - sorting_columns: If set, specifies a sort ordering of the rows in this RowGroup.
|
|
|
|
|
+ The sorting columns can be a subset of all the columns.
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.LIST, 'columns', (TType.STRUCT,(ColumnChunk, ColumnChunk.thrift_spec)), None, ), # 1
|
|
|
|
|
+ (2, TType.I64, 'total_byte_size', None, None, ), # 2
|
|
|
|
|
+ (3, TType.I64, 'num_rows', None, None, ), # 3
|
|
|
|
|
+ (4, TType.LIST, 'sorting_columns', (TType.STRUCT,(SortingColumn, SortingColumn.thrift_spec)), None, ), # 4
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, columns=None, total_byte_size=None, num_rows=None, sorting_columns=None,):
|
|
|
|
|
+ self.columns = columns
|
|
|
|
|
+ self.total_byte_size = total_byte_size
|
|
|
|
|
+ self.num_rows = num_rows
|
|
|
|
|
+ self.sorting_columns = sorting_columns
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.columns = []
|
|
|
|
|
+ (_etype24, _size21) = iprot.readListBegin()
|
|
|
|
|
+ for _i25 in xrange(_size21):
|
|
|
|
|
+ _elem26 = ColumnChunk()
|
|
|
|
|
+ _elem26.read(iprot)
|
|
|
|
|
+ self.columns.append(_elem26)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.total_byte_size = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 3:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.num_rows = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 4:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.sorting_columns = []
|
|
|
|
|
+ (_etype30, _size27) = iprot.readListBegin()
|
|
|
|
|
+ for _i31 in xrange(_size27):
|
|
|
|
|
+ _elem32 = SortingColumn()
|
|
|
|
|
+ _elem32.read(iprot)
|
|
|
|
|
+ self.sorting_columns.append(_elem32)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('RowGroup')
|
|
|
|
|
+ if self.columns is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('columns', TType.LIST, 1)
|
|
|
|
|
+ oprot.writeListBegin(TType.STRUCT, len(self.columns))
|
|
|
|
|
+ for iter33 in self.columns:
|
|
|
|
|
+ iter33.write(oprot)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.total_byte_size is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('total_byte_size', TType.I64, 2)
|
|
|
|
|
+ oprot.writeI64(self.total_byte_size)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.num_rows is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('num_rows', TType.I64, 3)
|
|
|
|
|
+ oprot.writeI64(self.num_rows)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.sorting_columns is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('sorting_columns', TType.LIST, 4)
|
|
|
|
|
+ oprot.writeListBegin(TType.STRUCT, len(self.sorting_columns))
|
|
|
|
|
+ for iter34 in self.sorting_columns:
|
|
|
|
|
+ iter34.write(oprot)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.columns is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field columns is unset!')
|
|
|
|
|
+ if self.total_byte_size is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field total_byte_size is unset!')
|
|
|
|
|
+ if self.num_rows is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|
|
|
|
|
+
|
|
|
|
|
+class FileMetaData:
|
|
|
|
|
+ """
|
|
|
|
|
+ Description for file metadata
|
|
|
|
|
+
|
|
|
|
|
+ Attributes:
|
|
|
|
|
+ - version: Version of this file *
|
|
|
|
|
+ - schema: Parquet schema for this file. This schema contains metadata for all the columns.
|
|
|
|
|
+ The schema is represented as a tree with a single root. The nodes of the tree
|
|
|
|
|
+ are flattened to a list by doing a depth-first traversal.
|
|
|
|
|
+ The column metadata contains the path in the schema for that column which can be
|
|
|
|
|
+ used to map columns to nodes in the schema.
|
|
|
|
|
+ The first element is the root *
|
|
|
|
|
+ - num_rows: Number of rows in this file *
|
|
|
|
|
+ - row_groups: Row groups in this file *
|
|
|
|
|
+ - key_value_metadata: Optional key/value metadata *
|
|
|
|
|
+ - created_by: String for application that wrote this file. This should be in the format
|
|
|
|
|
+ <Application> version <App Version> (build <App Build Hash>).
|
|
|
|
|
+ e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
|
|
|
|
|
+
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ thrift_spec = (
|
|
|
|
|
+ None, # 0
|
|
|
|
|
+ (1, TType.I32, 'version', None, None, ), # 1
|
|
|
|
|
+ (2, TType.LIST, 'schema', (TType.STRUCT,(SchemaElement, SchemaElement.thrift_spec)), None, ), # 2
|
|
|
|
|
+ (3, TType.I64, 'num_rows', None, None, ), # 3
|
|
|
|
|
+ (4, TType.LIST, 'row_groups', (TType.STRUCT,(RowGroup, RowGroup.thrift_spec)), None, ), # 4
|
|
|
|
|
+ (5, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 5
|
|
|
|
|
+ (6, TType.STRING, 'created_by', None, None, ), # 6
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, version=None, schema=None, num_rows=None, row_groups=None, key_value_metadata=None, created_by=None,):
|
|
|
|
|
+ self.version = version
|
|
|
|
|
+ self.schema = schema
|
|
|
|
|
+ self.num_rows = num_rows
|
|
|
|
|
+ self.row_groups = row_groups
|
|
|
|
|
+ self.key_value_metadata = key_value_metadata
|
|
|
|
|
+ self.created_by = created_by
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, iprot):
|
|
|
|
|
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
|
|
+ return
|
|
|
|
|
+ iprot.readStructBegin()
|
|
|
|
|
+ while True:
|
|
|
|
|
+ (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
|
|
+ if ftype == TType.STOP:
|
|
|
|
|
+ break
|
|
|
|
|
+ if fid == 1:
|
|
|
|
|
+ if ftype == TType.I32:
|
|
|
|
|
+ self.version = iprot.readI32();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 2:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.schema = []
|
|
|
|
|
+ (_etype38, _size35) = iprot.readListBegin()
|
|
|
|
|
+ for _i39 in xrange(_size35):
|
|
|
|
|
+ _elem40 = SchemaElement()
|
|
|
|
|
+ _elem40.read(iprot)
|
|
|
|
|
+ self.schema.append(_elem40)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 3:
|
|
|
|
|
+ if ftype == TType.I64:
|
|
|
|
|
+ self.num_rows = iprot.readI64();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 4:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.row_groups = []
|
|
|
|
|
+ (_etype44, _size41) = iprot.readListBegin()
|
|
|
|
|
+ for _i45 in xrange(_size41):
|
|
|
|
|
+ _elem46 = RowGroup()
|
|
|
|
|
+ _elem46.read(iprot)
|
|
|
|
|
+ self.row_groups.append(_elem46)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 5:
|
|
|
|
|
+ if ftype == TType.LIST:
|
|
|
|
|
+ self.key_value_metadata = []
|
|
|
|
|
+ (_etype50, _size47) = iprot.readListBegin()
|
|
|
|
|
+ for _i51 in xrange(_size47):
|
|
|
|
|
+ _elem52 = KeyValue()
|
|
|
|
|
+ _elem52.read(iprot)
|
|
|
|
|
+ self.key_value_metadata.append(_elem52)
|
|
|
|
|
+ iprot.readListEnd()
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ elif fid == 6:
|
|
|
|
|
+ if ftype == TType.STRING:
|
|
|
|
|
+ self.created_by = iprot.readString();
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ else:
|
|
|
|
|
+ iprot.skip(ftype)
|
|
|
|
|
+ iprot.readFieldEnd()
|
|
|
|
|
+ iprot.readStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def write(self, oprot):
|
|
|
|
|
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
|
|
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
|
|
+ return
|
|
|
|
|
+ oprot.writeStructBegin('FileMetaData')
|
|
|
|
|
+ if self.version is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('version', TType.I32, 1)
|
|
|
|
|
+ oprot.writeI32(self.version)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.schema is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('schema', TType.LIST, 2)
|
|
|
|
|
+ oprot.writeListBegin(TType.STRUCT, len(self.schema))
|
|
|
|
|
+ for iter53 in self.schema:
|
|
|
|
|
+ iter53.write(oprot)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.num_rows is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('num_rows', TType.I64, 3)
|
|
|
|
|
+ oprot.writeI64(self.num_rows)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.row_groups is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('row_groups', TType.LIST, 4)
|
|
|
|
|
+ oprot.writeListBegin(TType.STRUCT, len(self.row_groups))
|
|
|
|
|
+ for iter54 in self.row_groups:
|
|
|
|
|
+ iter54.write(oprot)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.key_value_metadata is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('key_value_metadata', TType.LIST, 5)
|
|
|
|
|
+ oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
|
|
|
|
|
+ for iter55 in self.key_value_metadata:
|
|
|
|
|
+ iter55.write(oprot)
|
|
|
|
|
+ oprot.writeListEnd()
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ if self.created_by is not None:
|
|
|
|
|
+ oprot.writeFieldBegin('created_by', TType.STRING, 6)
|
|
|
|
|
+ oprot.writeString(self.created_by)
|
|
|
|
|
+ oprot.writeFieldEnd()
|
|
|
|
|
+ oprot.writeFieldStop()
|
|
|
|
|
+ oprot.writeStructEnd()
|
|
|
|
|
+
|
|
|
|
|
+ def validate(self):
|
|
|
|
|
+ if self.version is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field version is unset!')
|
|
|
|
|
+ if self.schema is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field schema is unset!')
|
|
|
|
|
+ if self.num_rows is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
|
|
|
|
|
+ if self.row_groups is None:
|
|
|
|
|
+ raise TProtocol.TProtocolException(message='Required field row_groups is unset!')
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ def __repr__(self):
|
|
|
|
|
+ L = ['%s=%r' % (key, value)
|
|
|
|
|
+ for key, value in self.__dict__.iteritems()]
|
|
|
|
|
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
|
|
+
|
|
|
|
|
+ def __eq__(self, other):
|
|
|
|
|
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
|
|
+
|
|
|
|
|
+ def __ne__(self, other):
|
|
|
|
|
+ return not (self == other)
|