|
|
@@ -1,1424 +0,0 @@
|
|
|
-#
|
|
|
-# Autogenerated by Thrift Compiler (0.9.0)
|
|
|
-#
|
|
|
-# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
|
|
-#
|
|
|
-# options string: py
|
|
|
-#
|
|
|
-
|
|
|
-from thrift.Thrift import TType, TMessageType, TException, TApplicationException
|
|
|
-
|
|
|
-from thrift.transport import TTransport
|
|
|
-from thrift.protocol import TBinaryProtocol, TProtocol
|
|
|
-try:
|
|
|
- from thrift.protocol import fastbinary
|
|
|
-except:
|
|
|
- fastbinary = None
|
|
|
-
|
|
|
-
|
|
|
-class Type:
|
|
|
- """
|
|
|
- Types supported by Parquet. These types are intended to be used in combination
|
|
|
- with the encodings to control the on disk storage format.
|
|
|
- For example INT16 is not included as a type since a good encoding of INT32
|
|
|
- would handle this.
|
|
|
- """
|
|
|
- BOOLEAN = 0
|
|
|
- INT32 = 1
|
|
|
- INT64 = 2
|
|
|
- INT96 = 3
|
|
|
- FLOAT = 4
|
|
|
- DOUBLE = 5
|
|
|
- BYTE_ARRAY = 6
|
|
|
- FIXED_LEN_BYTE_ARRAY = 7
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 0: "BOOLEAN",
|
|
|
- 1: "INT32",
|
|
|
- 2: "INT64",
|
|
|
- 3: "INT96",
|
|
|
- 4: "FLOAT",
|
|
|
- 5: "DOUBLE",
|
|
|
- 6: "BYTE_ARRAY",
|
|
|
- 7: "FIXED_LEN_BYTE_ARRAY",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "BOOLEAN": 0,
|
|
|
- "INT32": 1,
|
|
|
- "INT64": 2,
|
|
|
- "INT96": 3,
|
|
|
- "FLOAT": 4,
|
|
|
- "DOUBLE": 5,
|
|
|
- "BYTE_ARRAY": 6,
|
|
|
- "FIXED_LEN_BYTE_ARRAY": 7,
|
|
|
- }
|
|
|
-
|
|
|
-class ConvertedType:
|
|
|
- """
|
|
|
- Common types used by frameworks(e.g. hive, pig) using parquet. This helps map
|
|
|
- between types in those frameworks to the base types in parquet. This is only
|
|
|
- metadata and not needed to read or write the data.
|
|
|
- """
|
|
|
- UTF8 = 0
|
|
|
- MAP = 1
|
|
|
- MAP_KEY_VALUE = 2
|
|
|
- LIST = 3
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 0: "UTF8",
|
|
|
- 1: "MAP",
|
|
|
- 2: "MAP_KEY_VALUE",
|
|
|
- 3: "LIST",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "UTF8": 0,
|
|
|
- "MAP": 1,
|
|
|
- "MAP_KEY_VALUE": 2,
|
|
|
- "LIST": 3,
|
|
|
- }
|
|
|
-
|
|
|
-class FieldRepetitionType:
|
|
|
- """
|
|
|
- Representation of Schemas
|
|
|
- """
|
|
|
- REQUIRED = 0
|
|
|
- OPTIONAL = 1
|
|
|
- REPEATED = 2
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 0: "REQUIRED",
|
|
|
- 1: "OPTIONAL",
|
|
|
- 2: "REPEATED",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "REQUIRED": 0,
|
|
|
- "OPTIONAL": 1,
|
|
|
- "REPEATED": 2,
|
|
|
- }
|
|
|
-
|
|
|
-class Encoding:
|
|
|
- """
|
|
|
- Encodings supported by Parquet. Not all encodings are valid for all types. These
|
|
|
- enums are also used to specify the encoding of definition and repetition levels.
|
|
|
- See the accompanying doc for the details of the more complicated encodings.
|
|
|
- """
|
|
|
- PLAIN = 0
|
|
|
- GROUP_VAR_INT = 1
|
|
|
- PLAIN_DICTIONARY = 2
|
|
|
- RLE = 3
|
|
|
- BIT_PACKED = 4
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 0: "PLAIN",
|
|
|
- 1: "GROUP_VAR_INT",
|
|
|
- 2: "PLAIN_DICTIONARY",
|
|
|
- 3: "RLE",
|
|
|
- 4: "BIT_PACKED",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "PLAIN": 0,
|
|
|
- "GROUP_VAR_INT": 1,
|
|
|
- "PLAIN_DICTIONARY": 2,
|
|
|
- "RLE": 3,
|
|
|
- "BIT_PACKED": 4,
|
|
|
- }
|
|
|
-
|
|
|
-class CompressionCodec:
|
|
|
- """
|
|
|
- Supported compression algorithms.
|
|
|
- """
|
|
|
- UNCOMPRESSED = 0
|
|
|
- SNAPPY = 1
|
|
|
- GZIP = 2
|
|
|
- LZO = 3
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 0: "UNCOMPRESSED",
|
|
|
- 1: "SNAPPY",
|
|
|
- 2: "GZIP",
|
|
|
- 3: "LZO",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "UNCOMPRESSED": 0,
|
|
|
- "SNAPPY": 1,
|
|
|
- "GZIP": 2,
|
|
|
- "LZO": 3,
|
|
|
- }
|
|
|
-
|
|
|
-class PageType:
|
|
|
- DATA_PAGE = 0
|
|
|
- INDEX_PAGE = 1
|
|
|
- DICTIONARY_PAGE = 2
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 0: "DATA_PAGE",
|
|
|
- 1: "INDEX_PAGE",
|
|
|
- 2: "DICTIONARY_PAGE",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "DATA_PAGE": 0,
|
|
|
- "INDEX_PAGE": 1,
|
|
|
- "DICTIONARY_PAGE": 2,
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-class SchemaElement:
|
|
|
- """
|
|
|
- Represents a element inside a schema definition.
|
|
|
- - if it is a group (inner node) then type is undefined and num_children is defined
|
|
|
- - if it is a primitive type (leaf) then type is defined and num_children is undefined
|
|
|
- the nodes are listed in depth first traversal order.
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - type: Data type for this field. Not set if the current element is a non-leaf node
|
|
|
- - type_length: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
|
|
|
- Otherwise, if specified, this is the maximum bit length to store any of the values.
|
|
|
- (e.g. a low cardinality INT col could have this set to 3). Note that this is
|
|
|
- in the schema, and therefore fixed for the entire file.
|
|
|
- - repetition_type: repetition of the field. The root of the schema does not have a repetition_type.
|
|
|
- All other nodes must have one
|
|
|
- - name: Name of the field in the schema
|
|
|
- - num_children: Nested fields. Since thrift does not support nested fields,
|
|
|
- the nesting is flattened to a single list by a depth-first traversal.
|
|
|
- The children count is used to construct the nested relationship.
|
|
|
- This field is not set when the element is a primitive type
|
|
|
- - converted_type: When the schema is the result of a conversion from another model
|
|
|
- Used to record the original type to help with cross conversion.
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.I32, 'type', None, None, ), # 1
|
|
|
- (2, TType.I32, 'type_length', None, None, ), # 2
|
|
|
- (3, TType.I32, 'repetition_type', None, None, ), # 3
|
|
|
- (4, TType.STRING, 'name', None, None, ), # 4
|
|
|
- (5, TType.I32, 'num_children', None, None, ), # 5
|
|
|
- (6, TType.I32, 'converted_type', None, None, ), # 6
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, type=None, type_length=None, repetition_type=None, name=None, num_children=None, converted_type=None,):
|
|
|
- self.type = type
|
|
|
- self.type_length = type_length
|
|
|
- self.repetition_type = repetition_type
|
|
|
- self.name = name
|
|
|
- self.num_children = num_children
|
|
|
- self.converted_type = converted_type
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.type = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.type_length = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.repetition_type = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.name = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 5:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.num_children = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 6:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.converted_type = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('SchemaElement')
|
|
|
- if self.type is not None:
|
|
|
- oprot.writeFieldBegin('type', TType.I32, 1)
|
|
|
- oprot.writeI32(self.type)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.type_length is not None:
|
|
|
- oprot.writeFieldBegin('type_length', TType.I32, 2)
|
|
|
- oprot.writeI32(self.type_length)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.repetition_type is not None:
|
|
|
- oprot.writeFieldBegin('repetition_type', TType.I32, 3)
|
|
|
- oprot.writeI32(self.repetition_type)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.name is not None:
|
|
|
- oprot.writeFieldBegin('name', TType.STRING, 4)
|
|
|
- oprot.writeString(self.name)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.num_children is not None:
|
|
|
- oprot.writeFieldBegin('num_children', TType.I32, 5)
|
|
|
- oprot.writeI32(self.num_children)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.converted_type is not None:
|
|
|
- oprot.writeFieldBegin('converted_type', TType.I32, 6)
|
|
|
- oprot.writeI32(self.converted_type)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.name is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field name is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class DataPageHeader:
|
|
|
- """
|
|
|
- Data page header
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - num_values: Number of values, including NULLs, in this data page. *
|
|
|
- - encoding: Encoding used for this data page *
|
|
|
- - definition_level_encoding: Encoding used for definition levels *
|
|
|
- - repetition_level_encoding: Encoding used for repetition levels *
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.I32, 'num_values', None, None, ), # 1
|
|
|
- (2, TType.I32, 'encoding', None, None, ), # 2
|
|
|
- (3, TType.I32, 'definition_level_encoding', None, None, ), # 3
|
|
|
- (4, TType.I32, 'repetition_level_encoding', None, None, ), # 4
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, num_values=None, encoding=None, definition_level_encoding=None, repetition_level_encoding=None,):
|
|
|
- self.num_values = num_values
|
|
|
- self.encoding = encoding
|
|
|
- self.definition_level_encoding = definition_level_encoding
|
|
|
- self.repetition_level_encoding = repetition_level_encoding
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.num_values = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.encoding = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.definition_level_encoding = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.repetition_level_encoding = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('DataPageHeader')
|
|
|
- if self.num_values is not None:
|
|
|
- oprot.writeFieldBegin('num_values', TType.I32, 1)
|
|
|
- oprot.writeI32(self.num_values)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.encoding is not None:
|
|
|
- oprot.writeFieldBegin('encoding', TType.I32, 2)
|
|
|
- oprot.writeI32(self.encoding)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.definition_level_encoding is not None:
|
|
|
- oprot.writeFieldBegin('definition_level_encoding', TType.I32, 3)
|
|
|
- oprot.writeI32(self.definition_level_encoding)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.repetition_level_encoding is not None:
|
|
|
- oprot.writeFieldBegin('repetition_level_encoding', TType.I32, 4)
|
|
|
- oprot.writeI32(self.repetition_level_encoding)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.num_values is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field num_values is unset!')
|
|
|
- if self.encoding is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field encoding is unset!')
|
|
|
- if self.definition_level_encoding is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field definition_level_encoding is unset!')
|
|
|
- if self.repetition_level_encoding is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field repetition_level_encoding is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class IndexPageHeader:
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- )
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('IndexPageHeader')
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class DictionaryPageHeader:
|
|
|
- """
|
|
|
- TODO: *
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - num_values: Number of values in the dictionary *
|
|
|
- - encoding: Encoding using this dictionary page *
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.I32, 'num_values', None, None, ), # 1
|
|
|
- (2, TType.I32, 'encoding', None, None, ), # 2
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, num_values=None, encoding=None,):
|
|
|
- self.num_values = num_values
|
|
|
- self.encoding = encoding
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.num_values = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.encoding = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('DictionaryPageHeader')
|
|
|
- if self.num_values is not None:
|
|
|
- oprot.writeFieldBegin('num_values', TType.I32, 1)
|
|
|
- oprot.writeI32(self.num_values)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.encoding is not None:
|
|
|
- oprot.writeFieldBegin('encoding', TType.I32, 2)
|
|
|
- oprot.writeI32(self.encoding)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.num_values is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field num_values is unset!')
|
|
|
- if self.encoding is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field encoding is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class PageHeader:
|
|
|
- """
|
|
|
- Attributes:
|
|
|
- - type: the type of the page: indicates which of the *_header fields is set *
|
|
|
- - uncompressed_page_size: Uncompressed page size in bytes (not including this header) *
|
|
|
- - compressed_page_size: Compressed page size in bytes (not including this header) *
|
|
|
- - crc: 32bit crc for the data below. This allows for disabling checksumming in HDFS
|
|
|
- if only a few pages needs to be read
|
|
|
-
|
|
|
- - data_page_header
|
|
|
- - index_page_header
|
|
|
- - dictionary_page_header
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.I32, 'type', None, None, ), # 1
|
|
|
- (2, TType.I32, 'uncompressed_page_size', None, None, ), # 2
|
|
|
- (3, TType.I32, 'compressed_page_size', None, None, ), # 3
|
|
|
- (4, TType.I32, 'crc', None, None, ), # 4
|
|
|
- (5, TType.STRUCT, 'data_page_header', (DataPageHeader, DataPageHeader.thrift_spec), None, ), # 5
|
|
|
- (6, TType.STRUCT, 'index_page_header', (IndexPageHeader, IndexPageHeader.thrift_spec), None, ), # 6
|
|
|
- (7, TType.STRUCT, 'dictionary_page_header', (DictionaryPageHeader, DictionaryPageHeader.thrift_spec), None, ), # 7
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, type=None, uncompressed_page_size=None, compressed_page_size=None, crc=None, data_page_header=None, index_page_header=None, dictionary_page_header=None,):
|
|
|
- self.type = type
|
|
|
- self.uncompressed_page_size = uncompressed_page_size
|
|
|
- self.compressed_page_size = compressed_page_size
|
|
|
- self.crc = crc
|
|
|
- self.data_page_header = data_page_header
|
|
|
- self.index_page_header = index_page_header
|
|
|
- self.dictionary_page_header = dictionary_page_header
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.type = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.uncompressed_page_size = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.compressed_page_size = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.crc = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 5:
|
|
|
- if ftype == TType.STRUCT:
|
|
|
- self.data_page_header = DataPageHeader()
|
|
|
- self.data_page_header.read(iprot)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 6:
|
|
|
- if ftype == TType.STRUCT:
|
|
|
- self.index_page_header = IndexPageHeader()
|
|
|
- self.index_page_header.read(iprot)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 7:
|
|
|
- if ftype == TType.STRUCT:
|
|
|
- self.dictionary_page_header = DictionaryPageHeader()
|
|
|
- self.dictionary_page_header.read(iprot)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('PageHeader')
|
|
|
- if self.type is not None:
|
|
|
- oprot.writeFieldBegin('type', TType.I32, 1)
|
|
|
- oprot.writeI32(self.type)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.uncompressed_page_size is not None:
|
|
|
- oprot.writeFieldBegin('uncompressed_page_size', TType.I32, 2)
|
|
|
- oprot.writeI32(self.uncompressed_page_size)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.compressed_page_size is not None:
|
|
|
- oprot.writeFieldBegin('compressed_page_size', TType.I32, 3)
|
|
|
- oprot.writeI32(self.compressed_page_size)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.crc is not None:
|
|
|
- oprot.writeFieldBegin('crc', TType.I32, 4)
|
|
|
- oprot.writeI32(self.crc)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.data_page_header is not None:
|
|
|
- oprot.writeFieldBegin('data_page_header', TType.STRUCT, 5)
|
|
|
- self.data_page_header.write(oprot)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.index_page_header is not None:
|
|
|
- oprot.writeFieldBegin('index_page_header', TType.STRUCT, 6)
|
|
|
- self.index_page_header.write(oprot)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.dictionary_page_header is not None:
|
|
|
- oprot.writeFieldBegin('dictionary_page_header', TType.STRUCT, 7)
|
|
|
- self.dictionary_page_header.write(oprot)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.type is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field type is unset!')
|
|
|
- if self.uncompressed_page_size is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field uncompressed_page_size is unset!')
|
|
|
- if self.compressed_page_size is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field compressed_page_size is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class KeyValue:
|
|
|
- """
|
|
|
- Wrapper struct to store key values
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - key
|
|
|
- - value
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.STRING, 'key', None, None, ), # 1
|
|
|
- (2, TType.STRING, 'value', None, None, ), # 2
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, key=None, value=None,):
|
|
|
- self.key = key
|
|
|
- self.value = value
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.key = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.value = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('KeyValue')
|
|
|
- if self.key is not None:
|
|
|
- oprot.writeFieldBegin('key', TType.STRING, 1)
|
|
|
- oprot.writeString(self.key)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.value is not None:
|
|
|
- oprot.writeFieldBegin('value', TType.STRING, 2)
|
|
|
- oprot.writeString(self.value)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.key is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field key is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class SortingColumn:
|
|
|
- """
|
|
|
- Wrapper struct to specify sort order
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - nulls_first: The column index (in this row group)
|
|
|
- 1: required i32 column_idx
|
|
|
-
|
|
|
- /** If true, indicates this column is sorted in descending order.
|
|
|
- 2: required bool descending
|
|
|
-
|
|
|
- /** If true, nulls will come before non-null values, otherwise,
|
|
|
- * nulls go at the end.
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- None, # 1
|
|
|
- None, # 2
|
|
|
- (3, TType.BOOL, 'nulls_first', None, None, ), # 3
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, nulls_first=None,):
|
|
|
- self.nulls_first = nulls_first
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 3:
|
|
|
- if ftype == TType.BOOL:
|
|
|
- self.nulls_first = iprot.readBool();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('SortingColumn')
|
|
|
- if self.nulls_first is not None:
|
|
|
- oprot.writeFieldBegin('nulls_first', TType.BOOL, 3)
|
|
|
- oprot.writeBool(self.nulls_first)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.nulls_first is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field nulls_first is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class ColumnMetaData:
|
|
|
- """
|
|
|
- Description for column metadata
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - type: Type of this column *
|
|
|
- - encodings: Set of all encodings used for this column. The purpose is to validate
|
|
|
- whether we can decode those pages. *
|
|
|
- - path_in_schema: Path in schema *
|
|
|
- - codec: Compression codec *
|
|
|
- - num_values: Number of values in this column *
|
|
|
- - total_uncompressed_size: total byte size of all uncompressed pages in this column chunk (including the headers) *
|
|
|
- - total_compressed_size: total byte size of all compressed pages in this column chunk (including the headers) *
|
|
|
- - key_value_metadata: Optional key/value metadata *
|
|
|
- - data_page_offset: Byte offset from beginning of file to first data page *
|
|
|
- - index_page_offset: Byte offset from beginning of file to root index page *
|
|
|
- - dictionary_page_offset: Byte offset from the beginning of file to first (only) dictionary page *
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.I32, 'type', None, None, ), # 1
|
|
|
- (2, TType.LIST, 'encodings', (TType.I32,None), None, ), # 2
|
|
|
- (3, TType.LIST, 'path_in_schema', (TType.STRING,None), None, ), # 3
|
|
|
- (4, TType.I32, 'codec', None, None, ), # 4
|
|
|
- (5, TType.I64, 'num_values', None, None, ), # 5
|
|
|
- (6, TType.I64, 'total_uncompressed_size', None, None, ), # 6
|
|
|
- (7, TType.I64, 'total_compressed_size', None, None, ), # 7
|
|
|
- (8, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 8
|
|
|
- (9, TType.I64, 'data_page_offset', None, None, ), # 9
|
|
|
- (10, TType.I64, 'index_page_offset', None, None, ), # 10
|
|
|
- (11, TType.I64, 'dictionary_page_offset', None, None, ), # 11
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, type=None, encodings=None, path_in_schema=None, codec=None, num_values=None, total_uncompressed_size=None, total_compressed_size=None, key_value_metadata=None, data_page_offset=None, index_page_offset=None, dictionary_page_offset=None,):
|
|
|
- self.type = type
|
|
|
- self.encodings = encodings
|
|
|
- self.path_in_schema = path_in_schema
|
|
|
- self.codec = codec
|
|
|
- self.num_values = num_values
|
|
|
- self.total_uncompressed_size = total_uncompressed_size
|
|
|
- self.total_compressed_size = total_compressed_size
|
|
|
- self.key_value_metadata = key_value_metadata
|
|
|
- self.data_page_offset = data_page_offset
|
|
|
- self.index_page_offset = index_page_offset
|
|
|
- self.dictionary_page_offset = dictionary_page_offset
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.type = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.encodings = []
|
|
|
- (_etype3, _size0) = iprot.readListBegin()
|
|
|
- for _i4 in xrange(_size0):
|
|
|
- _elem5 = iprot.readI32();
|
|
|
- self.encodings.append(_elem5)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.path_in_schema = []
|
|
|
- (_etype9, _size6) = iprot.readListBegin()
|
|
|
- for _i10 in xrange(_size6):
|
|
|
- _elem11 = iprot.readString();
|
|
|
- self.path_in_schema.append(_elem11)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.codec = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 5:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.num_values = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 6:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.total_uncompressed_size = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 7:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.total_compressed_size = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 8:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.key_value_metadata = []
|
|
|
- (_etype15, _size12) = iprot.readListBegin()
|
|
|
- for _i16 in xrange(_size12):
|
|
|
- _elem17 = KeyValue()
|
|
|
- _elem17.read(iprot)
|
|
|
- self.key_value_metadata.append(_elem17)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 9:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.data_page_offset = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 10:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.index_page_offset = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 11:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.dictionary_page_offset = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('ColumnMetaData')
|
|
|
- if self.type is not None:
|
|
|
- oprot.writeFieldBegin('type', TType.I32, 1)
|
|
|
- oprot.writeI32(self.type)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.encodings is not None:
|
|
|
- oprot.writeFieldBegin('encodings', TType.LIST, 2)
|
|
|
- oprot.writeListBegin(TType.I32, len(self.encodings))
|
|
|
- for iter18 in self.encodings:
|
|
|
- oprot.writeI32(iter18)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.path_in_schema is not None:
|
|
|
- oprot.writeFieldBegin('path_in_schema', TType.LIST, 3)
|
|
|
- oprot.writeListBegin(TType.STRING, len(self.path_in_schema))
|
|
|
- for iter19 in self.path_in_schema:
|
|
|
- oprot.writeString(iter19)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.codec is not None:
|
|
|
- oprot.writeFieldBegin('codec', TType.I32, 4)
|
|
|
- oprot.writeI32(self.codec)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.num_values is not None:
|
|
|
- oprot.writeFieldBegin('num_values', TType.I64, 5)
|
|
|
- oprot.writeI64(self.num_values)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.total_uncompressed_size is not None:
|
|
|
- oprot.writeFieldBegin('total_uncompressed_size', TType.I64, 6)
|
|
|
- oprot.writeI64(self.total_uncompressed_size)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.total_compressed_size is not None:
|
|
|
- oprot.writeFieldBegin('total_compressed_size', TType.I64, 7)
|
|
|
- oprot.writeI64(self.total_compressed_size)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.key_value_metadata is not None:
|
|
|
- oprot.writeFieldBegin('key_value_metadata', TType.LIST, 8)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
|
|
|
- for iter20 in self.key_value_metadata:
|
|
|
- iter20.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.data_page_offset is not None:
|
|
|
- oprot.writeFieldBegin('data_page_offset', TType.I64, 9)
|
|
|
- oprot.writeI64(self.data_page_offset)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.index_page_offset is not None:
|
|
|
- oprot.writeFieldBegin('index_page_offset', TType.I64, 10)
|
|
|
- oprot.writeI64(self.index_page_offset)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.dictionary_page_offset is not None:
|
|
|
- oprot.writeFieldBegin('dictionary_page_offset', TType.I64, 11)
|
|
|
- oprot.writeI64(self.dictionary_page_offset)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.type is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field type is unset!')
|
|
|
- if self.encodings is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field encodings is unset!')
|
|
|
- if self.path_in_schema is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field path_in_schema is unset!')
|
|
|
- if self.codec is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field codec is unset!')
|
|
|
- if self.num_values is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field num_values is unset!')
|
|
|
- if self.total_uncompressed_size is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field total_uncompressed_size is unset!')
|
|
|
- if self.total_compressed_size is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field total_compressed_size is unset!')
|
|
|
- if self.data_page_offset is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field data_page_offset is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class ColumnChunk:
|
|
|
- """
|
|
|
- Attributes:
|
|
|
- - file_path: File where column data is stored. If not set, assumed to be same file as
|
|
|
- metadata. This path is relative to the current file.
|
|
|
-
|
|
|
- - file_offset: Byte offset in file_path to the ColumnMetaData *
|
|
|
- - meta_data: Column metadata for this chunk. This is the same content as what is at
|
|
|
- file_path/file_offset. Having it here has it replicated in the file
|
|
|
- metadata.
|
|
|
-
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.STRING, 'file_path', None, None, ), # 1
|
|
|
- (2, TType.I64, 'file_offset', None, None, ), # 2
|
|
|
- (3, TType.STRUCT, 'meta_data', (ColumnMetaData, ColumnMetaData.thrift_spec), None, ), # 3
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, file_path=None, file_offset=None, meta_data=None,):
|
|
|
- self.file_path = file_path
|
|
|
- self.file_offset = file_offset
|
|
|
- self.meta_data = meta_data
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.file_path = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.file_offset = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.STRUCT:
|
|
|
- self.meta_data = ColumnMetaData()
|
|
|
- self.meta_data.read(iprot)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('ColumnChunk')
|
|
|
- if self.file_path is not None:
|
|
|
- oprot.writeFieldBegin('file_path', TType.STRING, 1)
|
|
|
- oprot.writeString(self.file_path)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.file_offset is not None:
|
|
|
- oprot.writeFieldBegin('file_offset', TType.I64, 2)
|
|
|
- oprot.writeI64(self.file_offset)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.meta_data is not None:
|
|
|
- oprot.writeFieldBegin('meta_data', TType.STRUCT, 3)
|
|
|
- self.meta_data.write(oprot)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.file_offset is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field file_offset is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class RowGroup:
|
|
|
- """
|
|
|
- Attributes:
|
|
|
- - columns
|
|
|
- - total_byte_size: Total byte size of all the uncompressed column data in this row group *
|
|
|
- - num_rows: Number of rows in this row group *
|
|
|
- - sorting_columns: If set, specifies a sort ordering of the rows in this RowGroup.
|
|
|
- The sorting columns can be a subset of all the columns.
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.LIST, 'columns', (TType.STRUCT,(ColumnChunk, ColumnChunk.thrift_spec)), None, ), # 1
|
|
|
- (2, TType.I64, 'total_byte_size', None, None, ), # 2
|
|
|
- (3, TType.I64, 'num_rows', None, None, ), # 3
|
|
|
- (4, TType.LIST, 'sorting_columns', (TType.STRUCT,(SortingColumn, SortingColumn.thrift_spec)), None, ), # 4
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, columns=None, total_byte_size=None, num_rows=None, sorting_columns=None,):
|
|
|
- self.columns = columns
|
|
|
- self.total_byte_size = total_byte_size
|
|
|
- self.num_rows = num_rows
|
|
|
- self.sorting_columns = sorting_columns
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.columns = []
|
|
|
- (_etype24, _size21) = iprot.readListBegin()
|
|
|
- for _i25 in xrange(_size21):
|
|
|
- _elem26 = ColumnChunk()
|
|
|
- _elem26.read(iprot)
|
|
|
- self.columns.append(_elem26)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.total_byte_size = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.num_rows = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.sorting_columns = []
|
|
|
- (_etype30, _size27) = iprot.readListBegin()
|
|
|
- for _i31 in xrange(_size27):
|
|
|
- _elem32 = SortingColumn()
|
|
|
- _elem32.read(iprot)
|
|
|
- self.sorting_columns.append(_elem32)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('RowGroup')
|
|
|
- if self.columns is not None:
|
|
|
- oprot.writeFieldBegin('columns', TType.LIST, 1)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.columns))
|
|
|
- for iter33 in self.columns:
|
|
|
- iter33.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.total_byte_size is not None:
|
|
|
- oprot.writeFieldBegin('total_byte_size', TType.I64, 2)
|
|
|
- oprot.writeI64(self.total_byte_size)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.num_rows is not None:
|
|
|
- oprot.writeFieldBegin('num_rows', TType.I64, 3)
|
|
|
- oprot.writeI64(self.num_rows)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.sorting_columns is not None:
|
|
|
- oprot.writeFieldBegin('sorting_columns', TType.LIST, 4)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.sorting_columns))
|
|
|
- for iter34 in self.sorting_columns:
|
|
|
- iter34.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.columns is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field columns is unset!')
|
|
|
- if self.total_byte_size is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field total_byte_size is unset!')
|
|
|
- if self.num_rows is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class FileMetaData:
|
|
|
- """
|
|
|
- Description for file metadata
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - version: Version of this file *
|
|
|
- - schema: Parquet schema for this file. This schema contains metadata for all the columns.
|
|
|
- The schema is represented as a tree with a single root. The nodes of the tree
|
|
|
- are flattened to a list by doing a depth-first traversal.
|
|
|
- The column metadata contains the path in the schema for that column which can be
|
|
|
- used to map columns to nodes in the schema.
|
|
|
- The first element is the root *
|
|
|
- - num_rows: Number of rows in this file *
|
|
|
- - row_groups: Row groups in this file *
|
|
|
- - key_value_metadata: Optional key/value metadata *
|
|
|
- - created_by: String for application that wrote this file. This should be in the format
|
|
|
- <Application> version <App Version> (build <App Build Hash>).
|
|
|
- e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
|
|
|
-
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.I32, 'version', None, None, ), # 1
|
|
|
- (2, TType.LIST, 'schema', (TType.STRUCT,(SchemaElement, SchemaElement.thrift_spec)), None, ), # 2
|
|
|
- (3, TType.I64, 'num_rows', None, None, ), # 3
|
|
|
- (4, TType.LIST, 'row_groups', (TType.STRUCT,(RowGroup, RowGroup.thrift_spec)), None, ), # 4
|
|
|
- (5, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 5
|
|
|
- (6, TType.STRING, 'created_by', None, None, ), # 6
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, version=None, schema=None, num_rows=None, row_groups=None, key_value_metadata=None, created_by=None,):
|
|
|
- self.version = version
|
|
|
- self.schema = schema
|
|
|
- self.num_rows = num_rows
|
|
|
- self.row_groups = row_groups
|
|
|
- self.key_value_metadata = key_value_metadata
|
|
|
- self.created_by = created_by
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.version = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.schema = []
|
|
|
- (_etype38, _size35) = iprot.readListBegin()
|
|
|
- for _i39 in xrange(_size35):
|
|
|
- _elem40 = SchemaElement()
|
|
|
- _elem40.read(iprot)
|
|
|
- self.schema.append(_elem40)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.num_rows = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.row_groups = []
|
|
|
- (_etype44, _size41) = iprot.readListBegin()
|
|
|
- for _i45 in xrange(_size41):
|
|
|
- _elem46 = RowGroup()
|
|
|
- _elem46.read(iprot)
|
|
|
- self.row_groups.append(_elem46)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 5:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.key_value_metadata = []
|
|
|
- (_etype50, _size47) = iprot.readListBegin()
|
|
|
- for _i51 in xrange(_size47):
|
|
|
- _elem52 = KeyValue()
|
|
|
- _elem52.read(iprot)
|
|
|
- self.key_value_metadata.append(_elem52)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 6:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.created_by = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('FileMetaData')
|
|
|
- if self.version is not None:
|
|
|
- oprot.writeFieldBegin('version', TType.I32, 1)
|
|
|
- oprot.writeI32(self.version)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.schema is not None:
|
|
|
- oprot.writeFieldBegin('schema', TType.LIST, 2)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.schema))
|
|
|
- for iter53 in self.schema:
|
|
|
- iter53.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.num_rows is not None:
|
|
|
- oprot.writeFieldBegin('num_rows', TType.I64, 3)
|
|
|
- oprot.writeI64(self.num_rows)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.row_groups is not None:
|
|
|
- oprot.writeFieldBegin('row_groups', TType.LIST, 4)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.row_groups))
|
|
|
- for iter54 in self.row_groups:
|
|
|
- iter54.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.key_value_metadata is not None:
|
|
|
- oprot.writeFieldBegin('key_value_metadata', TType.LIST, 5)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
|
|
|
- for iter55 in self.key_value_metadata:
|
|
|
- iter55.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.created_by is not None:
|
|
|
- oprot.writeFieldBegin('created_by', TType.STRING, 6)
|
|
|
- oprot.writeString(self.created_by)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
-
|
|
|
- def validate(self):
|
|
|
- if self.version is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field version is unset!')
|
|
|
- if self.schema is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field schema is unset!')
|
|
|
- if self.num_rows is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
|
|
|
- if self.row_groups is None:
|
|
|
- raise TProtocol.TProtocolException(message='Required field row_groups is unset!')
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|