# # Autogenerated by Thrift Compiler (0.9.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # # options string: py # from thrift.Thrift import TType, TMessageType, TException, TApplicationException from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol, TProtocol try: from thrift.protocol import fastbinary except: fastbinary = None class Type: """ Types supported by Parquet. These types are intended to be used in combination with the encodings to control the on disk storage format. For example INT16 is not included as a type since a good encoding of INT32 would handle this. """ BOOLEAN = 0 INT32 = 1 INT64 = 2 INT96 = 3 FLOAT = 4 DOUBLE = 5 BYTE_ARRAY = 6 FIXED_LEN_BYTE_ARRAY = 7 _VALUES_TO_NAMES = { 0: "BOOLEAN", 1: "INT32", 2: "INT64", 3: "INT96", 4: "FLOAT", 5: "DOUBLE", 6: "BYTE_ARRAY", 7: "FIXED_LEN_BYTE_ARRAY", } _NAMES_TO_VALUES = { "BOOLEAN": 0, "INT32": 1, "INT64": 2, "INT96": 3, "FLOAT": 4, "DOUBLE": 5, "BYTE_ARRAY": 6, "FIXED_LEN_BYTE_ARRAY": 7, } class ConvertedType: """ Common types used by frameworks(e.g. hive, pig) using parquet. This helps map between types in those frameworks to the base types in parquet. This is only metadata and not needed to read or write the data. """ UTF8 = 0 MAP = 1 MAP_KEY_VALUE = 2 LIST = 3 _VALUES_TO_NAMES = { 0: "UTF8", 1: "MAP", 2: "MAP_KEY_VALUE", 3: "LIST", } _NAMES_TO_VALUES = { "UTF8": 0, "MAP": 1, "MAP_KEY_VALUE": 2, "LIST": 3, } class FieldRepetitionType: """ Representation of Schemas """ REQUIRED = 0 OPTIONAL = 1 REPEATED = 2 _VALUES_TO_NAMES = { 0: "REQUIRED", 1: "OPTIONAL", 2: "REPEATED", } _NAMES_TO_VALUES = { "REQUIRED": 0, "OPTIONAL": 1, "REPEATED": 2, } class Encoding: """ Encodings supported by Parquet. Not all encodings are valid for all types. These enums are also used to specify the encoding of definition and repetition levels. See the accompanying doc for the details of the more complicated encodings. """ PLAIN = 0 GROUP_VAR_INT = 1 PLAIN_DICTIONARY = 2 RLE = 3 BIT_PACKED = 4 _VALUES_TO_NAMES = { 0: "PLAIN", 1: "GROUP_VAR_INT", 2: "PLAIN_DICTIONARY", 3: "RLE", 4: "BIT_PACKED", } _NAMES_TO_VALUES = { "PLAIN": 0, "GROUP_VAR_INT": 1, "PLAIN_DICTIONARY": 2, "RLE": 3, "BIT_PACKED": 4, } class CompressionCodec: """ Supported compression algorithms. """ UNCOMPRESSED = 0 SNAPPY = 1 GZIP = 2 LZO = 3 _VALUES_TO_NAMES = { 0: "UNCOMPRESSED", 1: "SNAPPY", 2: "GZIP", 3: "LZO", } _NAMES_TO_VALUES = { "UNCOMPRESSED": 0, "SNAPPY": 1, "GZIP": 2, "LZO": 3, } class PageType: DATA_PAGE = 0 INDEX_PAGE = 1 DICTIONARY_PAGE = 2 _VALUES_TO_NAMES = { 0: "DATA_PAGE", 1: "INDEX_PAGE", 2: "DICTIONARY_PAGE", } _NAMES_TO_VALUES = { "DATA_PAGE": 0, "INDEX_PAGE": 1, "DICTIONARY_PAGE": 2, } class SchemaElement: """ Represents a element inside a schema definition. - if it is a group (inner node) then type is undefined and num_children is defined - if it is a primitive type (leaf) then type is defined and num_children is undefined the nodes are listed in depth first traversal order. Attributes: - type: Data type for this field. Not set if the current element is a non-leaf node - type_length: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales. Otherwise, if specified, this is the maximum bit length to store any of the values. (e.g. a low cardinality INT col could have this set to 3). Note that this is in the schema, and therefore fixed for the entire file. - repetition_type: repetition of the field. The root of the schema does not have a repetition_type. All other nodes must have one - name: Name of the field in the schema - num_children: Nested fields. Since thrift does not support nested fields, the nesting is flattened to a single list by a depth-first traversal. The children count is used to construct the nested relationship. This field is not set when the element is a primitive type - converted_type: When the schema is the result of a conversion from another model Used to record the original type to help with cross conversion. """ thrift_spec = ( None, # 0 (1, TType.I32, 'type', None, None, ), # 1 (2, TType.I32, 'type_length', None, None, ), # 2 (3, TType.I32, 'repetition_type', None, None, ), # 3 (4, TType.STRING, 'name', None, None, ), # 4 (5, TType.I32, 'num_children', None, None, ), # 5 (6, TType.I32, 'converted_type', None, None, ), # 6 ) def __init__(self, type=None, type_length=None, repetition_type=None, name=None, num_children=None, converted_type=None,): self.type = type self.type_length = type_length self.repetition_type = repetition_type self.name = name self.num_children = num_children self.converted_type = converted_type def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.I32: self.type = iprot.readI32(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.I32: self.type_length = iprot.readI32(); else: iprot.skip(ftype) elif fid == 3: if ftype == TType.I32: self.repetition_type = iprot.readI32(); else: iprot.skip(ftype) elif fid == 4: if ftype == TType.STRING: self.name = iprot.readString(); else: iprot.skip(ftype) elif fid == 5: if ftype == TType.I32: self.num_children = iprot.readI32(); else: iprot.skip(ftype) elif fid == 6: if ftype == TType.I32: self.converted_type = iprot.readI32(); else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('SchemaElement') if self.type is not None: oprot.writeFieldBegin('type', TType.I32, 1) oprot.writeI32(self.type) oprot.writeFieldEnd() if self.type_length is not None: oprot.writeFieldBegin('type_length', TType.I32, 2) oprot.writeI32(self.type_length) oprot.writeFieldEnd() if self.repetition_type is not None: oprot.writeFieldBegin('repetition_type', TType.I32, 3) oprot.writeI32(self.repetition_type) oprot.writeFieldEnd() if self.name is not None: oprot.writeFieldBegin('name', TType.STRING, 4) oprot.writeString(self.name) oprot.writeFieldEnd() if self.num_children is not None: oprot.writeFieldBegin('num_children', TType.I32, 5) oprot.writeI32(self.num_children) oprot.writeFieldEnd() if self.converted_type is not None: oprot.writeFieldBegin('converted_type', TType.I32, 6) oprot.writeI32(self.converted_type) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.name is None: raise TProtocol.TProtocolException(message='Required field name is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class DataPageHeader: """ Data page header Attributes: - num_values: Number of values, including NULLs, in this data page. * - encoding: Encoding used for this data page * - definition_level_encoding: Encoding used for definition levels * - repetition_level_encoding: Encoding used for repetition levels * """ thrift_spec = ( None, # 0 (1, TType.I32, 'num_values', None, None, ), # 1 (2, TType.I32, 'encoding', None, None, ), # 2 (3, TType.I32, 'definition_level_encoding', None, None, ), # 3 (4, TType.I32, 'repetition_level_encoding', None, None, ), # 4 ) def __init__(self, num_values=None, encoding=None, definition_level_encoding=None, repetition_level_encoding=None,): self.num_values = num_values self.encoding = encoding self.definition_level_encoding = definition_level_encoding self.repetition_level_encoding = repetition_level_encoding def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.I32: self.num_values = iprot.readI32(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.I32: self.encoding = iprot.readI32(); else: iprot.skip(ftype) elif fid == 3: if ftype == TType.I32: self.definition_level_encoding = iprot.readI32(); else: iprot.skip(ftype) elif fid == 4: if ftype == TType.I32: self.repetition_level_encoding = iprot.readI32(); else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('DataPageHeader') if self.num_values is not None: oprot.writeFieldBegin('num_values', TType.I32, 1) oprot.writeI32(self.num_values) oprot.writeFieldEnd() if self.encoding is not None: oprot.writeFieldBegin('encoding', TType.I32, 2) oprot.writeI32(self.encoding) oprot.writeFieldEnd() if self.definition_level_encoding is not None: oprot.writeFieldBegin('definition_level_encoding', TType.I32, 3) oprot.writeI32(self.definition_level_encoding) oprot.writeFieldEnd() if self.repetition_level_encoding is not None: oprot.writeFieldBegin('repetition_level_encoding', TType.I32, 4) oprot.writeI32(self.repetition_level_encoding) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.num_values is None: raise TProtocol.TProtocolException(message='Required field num_values is unset!') if self.encoding is None: raise TProtocol.TProtocolException(message='Required field encoding is unset!') if self.definition_level_encoding is None: raise TProtocol.TProtocolException(message='Required field definition_level_encoding is unset!') if self.repetition_level_encoding is None: raise TProtocol.TProtocolException(message='Required field repetition_level_encoding is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class IndexPageHeader: thrift_spec = ( ) def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('IndexPageHeader') oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class DictionaryPageHeader: """ TODO: * Attributes: - num_values: Number of values in the dictionary * - encoding: Encoding using this dictionary page * """ thrift_spec = ( None, # 0 (1, TType.I32, 'num_values', None, None, ), # 1 (2, TType.I32, 'encoding', None, None, ), # 2 ) def __init__(self, num_values=None, encoding=None,): self.num_values = num_values self.encoding = encoding def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.I32: self.num_values = iprot.readI32(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.I32: self.encoding = iprot.readI32(); else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('DictionaryPageHeader') if self.num_values is not None: oprot.writeFieldBegin('num_values', TType.I32, 1) oprot.writeI32(self.num_values) oprot.writeFieldEnd() if self.encoding is not None: oprot.writeFieldBegin('encoding', TType.I32, 2) oprot.writeI32(self.encoding) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.num_values is None: raise TProtocol.TProtocolException(message='Required field num_values is unset!') if self.encoding is None: raise TProtocol.TProtocolException(message='Required field encoding is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class PageHeader: """ Attributes: - type: the type of the page: indicates which of the *_header fields is set * - uncompressed_page_size: Uncompressed page size in bytes (not including this header) * - compressed_page_size: Compressed page size in bytes (not including this header) * - crc: 32bit crc for the data below. This allows for disabling checksumming in HDFS if only a few pages needs to be read - data_page_header - index_page_header - dictionary_page_header """ thrift_spec = ( None, # 0 (1, TType.I32, 'type', None, None, ), # 1 (2, TType.I32, 'uncompressed_page_size', None, None, ), # 2 (3, TType.I32, 'compressed_page_size', None, None, ), # 3 (4, TType.I32, 'crc', None, None, ), # 4 (5, TType.STRUCT, 'data_page_header', (DataPageHeader, DataPageHeader.thrift_spec), None, ), # 5 (6, TType.STRUCT, 'index_page_header', (IndexPageHeader, IndexPageHeader.thrift_spec), None, ), # 6 (7, TType.STRUCT, 'dictionary_page_header', (DictionaryPageHeader, DictionaryPageHeader.thrift_spec), None, ), # 7 ) def __init__(self, type=None, uncompressed_page_size=None, compressed_page_size=None, crc=None, data_page_header=None, index_page_header=None, dictionary_page_header=None,): self.type = type self.uncompressed_page_size = uncompressed_page_size self.compressed_page_size = compressed_page_size self.crc = crc self.data_page_header = data_page_header self.index_page_header = index_page_header self.dictionary_page_header = dictionary_page_header def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.I32: self.type = iprot.readI32(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.I32: self.uncompressed_page_size = iprot.readI32(); else: iprot.skip(ftype) elif fid == 3: if ftype == TType.I32: self.compressed_page_size = iprot.readI32(); else: iprot.skip(ftype) elif fid == 4: if ftype == TType.I32: self.crc = iprot.readI32(); else: iprot.skip(ftype) elif fid == 5: if ftype == TType.STRUCT: self.data_page_header = DataPageHeader() self.data_page_header.read(iprot) else: iprot.skip(ftype) elif fid == 6: if ftype == TType.STRUCT: self.index_page_header = IndexPageHeader() self.index_page_header.read(iprot) else: iprot.skip(ftype) elif fid == 7: if ftype == TType.STRUCT: self.dictionary_page_header = DictionaryPageHeader() self.dictionary_page_header.read(iprot) else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('PageHeader') if self.type is not None: oprot.writeFieldBegin('type', TType.I32, 1) oprot.writeI32(self.type) oprot.writeFieldEnd() if self.uncompressed_page_size is not None: oprot.writeFieldBegin('uncompressed_page_size', TType.I32, 2) oprot.writeI32(self.uncompressed_page_size) oprot.writeFieldEnd() if self.compressed_page_size is not None: oprot.writeFieldBegin('compressed_page_size', TType.I32, 3) oprot.writeI32(self.compressed_page_size) oprot.writeFieldEnd() if self.crc is not None: oprot.writeFieldBegin('crc', TType.I32, 4) oprot.writeI32(self.crc) oprot.writeFieldEnd() if self.data_page_header is not None: oprot.writeFieldBegin('data_page_header', TType.STRUCT, 5) self.data_page_header.write(oprot) oprot.writeFieldEnd() if self.index_page_header is not None: oprot.writeFieldBegin('index_page_header', TType.STRUCT, 6) self.index_page_header.write(oprot) oprot.writeFieldEnd() if self.dictionary_page_header is not None: oprot.writeFieldBegin('dictionary_page_header', TType.STRUCT, 7) self.dictionary_page_header.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.type is None: raise TProtocol.TProtocolException(message='Required field type is unset!') if self.uncompressed_page_size is None: raise TProtocol.TProtocolException(message='Required field uncompressed_page_size is unset!') if self.compressed_page_size is None: raise TProtocol.TProtocolException(message='Required field compressed_page_size is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class KeyValue: """ Wrapper struct to store key values Attributes: - key - value """ thrift_spec = ( None, # 0 (1, TType.STRING, 'key', None, None, ), # 1 (2, TType.STRING, 'value', None, None, ), # 2 ) def __init__(self, key=None, value=None,): self.key = key self.value = value def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.STRING: self.key = iprot.readString(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.STRING: self.value = iprot.readString(); else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('KeyValue') if self.key is not None: oprot.writeFieldBegin('key', TType.STRING, 1) oprot.writeString(self.key) oprot.writeFieldEnd() if self.value is not None: oprot.writeFieldBegin('value', TType.STRING, 2) oprot.writeString(self.value) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.key is None: raise TProtocol.TProtocolException(message='Required field key is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class SortingColumn: """ Wrapper struct to specify sort order Attributes: - nulls_first: The column index (in this row group) 1: required i32 column_idx /** If true, indicates this column is sorted in descending order. 2: required bool descending /** If true, nulls will come before non-null values, otherwise, * nulls go at the end. """ thrift_spec = ( None, # 0 None, # 1 None, # 2 (3, TType.BOOL, 'nulls_first', None, None, ), # 3 ) def __init__(self, nulls_first=None,): self.nulls_first = nulls_first def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 3: if ftype == TType.BOOL: self.nulls_first = iprot.readBool(); else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('SortingColumn') if self.nulls_first is not None: oprot.writeFieldBegin('nulls_first', TType.BOOL, 3) oprot.writeBool(self.nulls_first) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.nulls_first is None: raise TProtocol.TProtocolException(message='Required field nulls_first is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class ColumnMetaData: """ Description for column metadata Attributes: - type: Type of this column * - encodings: Set of all encodings used for this column. The purpose is to validate whether we can decode those pages. * - path_in_schema: Path in schema * - codec: Compression codec * - num_values: Number of values in this column * - total_uncompressed_size: total byte size of all uncompressed pages in this column chunk (including the headers) * - total_compressed_size: total byte size of all compressed pages in this column chunk (including the headers) * - key_value_metadata: Optional key/value metadata * - data_page_offset: Byte offset from beginning of file to first data page * - index_page_offset: Byte offset from beginning of file to root index page * - dictionary_page_offset: Byte offset from the beginning of file to first (only) dictionary page * """ thrift_spec = ( None, # 0 (1, TType.I32, 'type', None, None, ), # 1 (2, TType.LIST, 'encodings', (TType.I32,None), None, ), # 2 (3, TType.LIST, 'path_in_schema', (TType.STRING,None), None, ), # 3 (4, TType.I32, 'codec', None, None, ), # 4 (5, TType.I64, 'num_values', None, None, ), # 5 (6, TType.I64, 'total_uncompressed_size', None, None, ), # 6 (7, TType.I64, 'total_compressed_size', None, None, ), # 7 (8, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 8 (9, TType.I64, 'data_page_offset', None, None, ), # 9 (10, TType.I64, 'index_page_offset', None, None, ), # 10 (11, TType.I64, 'dictionary_page_offset', None, None, ), # 11 ) def __init__(self, type=None, encodings=None, path_in_schema=None, codec=None, num_values=None, total_uncompressed_size=None, total_compressed_size=None, key_value_metadata=None, data_page_offset=None, index_page_offset=None, dictionary_page_offset=None,): self.type = type self.encodings = encodings self.path_in_schema = path_in_schema self.codec = codec self.num_values = num_values self.total_uncompressed_size = total_uncompressed_size self.total_compressed_size = total_compressed_size self.key_value_metadata = key_value_metadata self.data_page_offset = data_page_offset self.index_page_offset = index_page_offset self.dictionary_page_offset = dictionary_page_offset def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.I32: self.type = iprot.readI32(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.encodings = [] (_etype3, _size0) = iprot.readListBegin() for _i4 in xrange(_size0): _elem5 = iprot.readI32(); self.encodings.append(_elem5) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.path_in_schema = [] (_etype9, _size6) = iprot.readListBegin() for _i10 in xrange(_size6): _elem11 = iprot.readString(); self.path_in_schema.append(_elem11) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: if ftype == TType.I32: self.codec = iprot.readI32(); else: iprot.skip(ftype) elif fid == 5: if ftype == TType.I64: self.num_values = iprot.readI64(); else: iprot.skip(ftype) elif fid == 6: if ftype == TType.I64: self.total_uncompressed_size = iprot.readI64(); else: iprot.skip(ftype) elif fid == 7: if ftype == TType.I64: self.total_compressed_size = iprot.readI64(); else: iprot.skip(ftype) elif fid == 8: if ftype == TType.LIST: self.key_value_metadata = [] (_etype15, _size12) = iprot.readListBegin() for _i16 in xrange(_size12): _elem17 = KeyValue() _elem17.read(iprot) self.key_value_metadata.append(_elem17) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 9: if ftype == TType.I64: self.data_page_offset = iprot.readI64(); else: iprot.skip(ftype) elif fid == 10: if ftype == TType.I64: self.index_page_offset = iprot.readI64(); else: iprot.skip(ftype) elif fid == 11: if ftype == TType.I64: self.dictionary_page_offset = iprot.readI64(); else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('ColumnMetaData') if self.type is not None: oprot.writeFieldBegin('type', TType.I32, 1) oprot.writeI32(self.type) oprot.writeFieldEnd() if self.encodings is not None: oprot.writeFieldBegin('encodings', TType.LIST, 2) oprot.writeListBegin(TType.I32, len(self.encodings)) for iter18 in self.encodings: oprot.writeI32(iter18) oprot.writeListEnd() oprot.writeFieldEnd() if self.path_in_schema is not None: oprot.writeFieldBegin('path_in_schema', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.path_in_schema)) for iter19 in self.path_in_schema: oprot.writeString(iter19) oprot.writeListEnd() oprot.writeFieldEnd() if self.codec is not None: oprot.writeFieldBegin('codec', TType.I32, 4) oprot.writeI32(self.codec) oprot.writeFieldEnd() if self.num_values is not None: oprot.writeFieldBegin('num_values', TType.I64, 5) oprot.writeI64(self.num_values) oprot.writeFieldEnd() if self.total_uncompressed_size is not None: oprot.writeFieldBegin('total_uncompressed_size', TType.I64, 6) oprot.writeI64(self.total_uncompressed_size) oprot.writeFieldEnd() if self.total_compressed_size is not None: oprot.writeFieldBegin('total_compressed_size', TType.I64, 7) oprot.writeI64(self.total_compressed_size) oprot.writeFieldEnd() if self.key_value_metadata is not None: oprot.writeFieldBegin('key_value_metadata', TType.LIST, 8) oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata)) for iter20 in self.key_value_metadata: iter20.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.data_page_offset is not None: oprot.writeFieldBegin('data_page_offset', TType.I64, 9) oprot.writeI64(self.data_page_offset) oprot.writeFieldEnd() if self.index_page_offset is not None: oprot.writeFieldBegin('index_page_offset', TType.I64, 10) oprot.writeI64(self.index_page_offset) oprot.writeFieldEnd() if self.dictionary_page_offset is not None: oprot.writeFieldBegin('dictionary_page_offset', TType.I64, 11) oprot.writeI64(self.dictionary_page_offset) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.type is None: raise TProtocol.TProtocolException(message='Required field type is unset!') if self.encodings is None: raise TProtocol.TProtocolException(message='Required field encodings is unset!') if self.path_in_schema is None: raise TProtocol.TProtocolException(message='Required field path_in_schema is unset!') if self.codec is None: raise TProtocol.TProtocolException(message='Required field codec is unset!') if self.num_values is None: raise TProtocol.TProtocolException(message='Required field num_values is unset!') if self.total_uncompressed_size is None: raise TProtocol.TProtocolException(message='Required field total_uncompressed_size is unset!') if self.total_compressed_size is None: raise TProtocol.TProtocolException(message='Required field total_compressed_size is unset!') if self.data_page_offset is None: raise TProtocol.TProtocolException(message='Required field data_page_offset is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class ColumnChunk: """ Attributes: - file_path: File where column data is stored. If not set, assumed to be same file as metadata. This path is relative to the current file. - file_offset: Byte offset in file_path to the ColumnMetaData * - meta_data: Column metadata for this chunk. This is the same content as what is at file_path/file_offset. Having it here has it replicated in the file metadata. """ thrift_spec = ( None, # 0 (1, TType.STRING, 'file_path', None, None, ), # 1 (2, TType.I64, 'file_offset', None, None, ), # 2 (3, TType.STRUCT, 'meta_data', (ColumnMetaData, ColumnMetaData.thrift_spec), None, ), # 3 ) def __init__(self, file_path=None, file_offset=None, meta_data=None,): self.file_path = file_path self.file_offset = file_offset self.meta_data = meta_data def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.STRING: self.file_path = iprot.readString(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.I64: self.file_offset = iprot.readI64(); else: iprot.skip(ftype) elif fid == 3: if ftype == TType.STRUCT: self.meta_data = ColumnMetaData() self.meta_data.read(iprot) else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('ColumnChunk') if self.file_path is not None: oprot.writeFieldBegin('file_path', TType.STRING, 1) oprot.writeString(self.file_path) oprot.writeFieldEnd() if self.file_offset is not None: oprot.writeFieldBegin('file_offset', TType.I64, 2) oprot.writeI64(self.file_offset) oprot.writeFieldEnd() if self.meta_data is not None: oprot.writeFieldBegin('meta_data', TType.STRUCT, 3) self.meta_data.write(oprot) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.file_offset is None: raise TProtocol.TProtocolException(message='Required field file_offset is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class RowGroup: """ Attributes: - columns - total_byte_size: Total byte size of all the uncompressed column data in this row group * - num_rows: Number of rows in this row group * - sorting_columns: If set, specifies a sort ordering of the rows in this RowGroup. The sorting columns can be a subset of all the columns. """ thrift_spec = ( None, # 0 (1, TType.LIST, 'columns', (TType.STRUCT,(ColumnChunk, ColumnChunk.thrift_spec)), None, ), # 1 (2, TType.I64, 'total_byte_size', None, None, ), # 2 (3, TType.I64, 'num_rows', None, None, ), # 3 (4, TType.LIST, 'sorting_columns', (TType.STRUCT,(SortingColumn, SortingColumn.thrift_spec)), None, ), # 4 ) def __init__(self, columns=None, total_byte_size=None, num_rows=None, sorting_columns=None,): self.columns = columns self.total_byte_size = total_byte_size self.num_rows = num_rows self.sorting_columns = sorting_columns def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.LIST: self.columns = [] (_etype24, _size21) = iprot.readListBegin() for _i25 in xrange(_size21): _elem26 = ColumnChunk() _elem26.read(iprot) self.columns.append(_elem26) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.I64: self.total_byte_size = iprot.readI64(); else: iprot.skip(ftype) elif fid == 3: if ftype == TType.I64: self.num_rows = iprot.readI64(); else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.sorting_columns = [] (_etype30, _size27) = iprot.readListBegin() for _i31 in xrange(_size27): _elem32 = SortingColumn() _elem32.read(iprot) self.sorting_columns.append(_elem32) iprot.readListEnd() else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('RowGroup') if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.columns)) for iter33 in self.columns: iter33.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.total_byte_size is not None: oprot.writeFieldBegin('total_byte_size', TType.I64, 2) oprot.writeI64(self.total_byte_size) oprot.writeFieldEnd() if self.num_rows is not None: oprot.writeFieldBegin('num_rows', TType.I64, 3) oprot.writeI64(self.num_rows) oprot.writeFieldEnd() if self.sorting_columns is not None: oprot.writeFieldBegin('sorting_columns', TType.LIST, 4) oprot.writeListBegin(TType.STRUCT, len(self.sorting_columns)) for iter34 in self.sorting_columns: iter34.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.columns is None: raise TProtocol.TProtocolException(message='Required field columns is unset!') if self.total_byte_size is None: raise TProtocol.TProtocolException(message='Required field total_byte_size is unset!') if self.num_rows is None: raise TProtocol.TProtocolException(message='Required field num_rows is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class FileMetaData: """ Description for file metadata Attributes: - version: Version of this file * - schema: Parquet schema for this file. This schema contains metadata for all the columns. The schema is represented as a tree with a single root. The nodes of the tree are flattened to a list by doing a depth-first traversal. The column metadata contains the path in the schema for that column which can be used to map columns to nodes in the schema. The first element is the root * - num_rows: Number of rows in this file * - row_groups: Row groups in this file * - key_value_metadata: Optional key/value metadata * - created_by: String for application that wrote this file. This should be in the format version (build ). e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55) """ thrift_spec = ( None, # 0 (1, TType.I32, 'version', None, None, ), # 1 (2, TType.LIST, 'schema', (TType.STRUCT,(SchemaElement, SchemaElement.thrift_spec)), None, ), # 2 (3, TType.I64, 'num_rows', None, None, ), # 3 (4, TType.LIST, 'row_groups', (TType.STRUCT,(RowGroup, RowGroup.thrift_spec)), None, ), # 4 (5, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 5 (6, TType.STRING, 'created_by', None, None, ), # 6 ) def __init__(self, version=None, schema=None, num_rows=None, row_groups=None, key_value_metadata=None, created_by=None,): self.version = version self.schema = schema self.num_rows = num_rows self.row_groups = row_groups self.key_value_metadata = key_value_metadata self.created_by = created_by def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) return iprot.readStructBegin() while True: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break if fid == 1: if ftype == TType.I32: self.version = iprot.readI32(); else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.schema = [] (_etype38, _size35) = iprot.readListBegin() for _i39 in xrange(_size35): _elem40 = SchemaElement() _elem40.read(iprot) self.schema.append(_elem40) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.I64: self.num_rows = iprot.readI64(); else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.row_groups = [] (_etype44, _size41) = iprot.readListBegin() for _i45 in xrange(_size41): _elem46 = RowGroup() _elem46.read(iprot) self.row_groups.append(_elem46) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 5: if ftype == TType.LIST: self.key_value_metadata = [] (_etype50, _size47) = iprot.readListBegin() for _i51 in xrange(_size47): _elem52 = KeyValue() _elem52.read(iprot) self.key_value_metadata.append(_elem52) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 6: if ftype == TType.STRING: self.created_by = iprot.readString(); else: iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() iprot.readStructEnd() def write(self, oprot): if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('FileMetaData') if self.version is not None: oprot.writeFieldBegin('version', TType.I32, 1) oprot.writeI32(self.version) oprot.writeFieldEnd() if self.schema is not None: oprot.writeFieldBegin('schema', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.schema)) for iter53 in self.schema: iter53.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.num_rows is not None: oprot.writeFieldBegin('num_rows', TType.I64, 3) oprot.writeI64(self.num_rows) oprot.writeFieldEnd() if self.row_groups is not None: oprot.writeFieldBegin('row_groups', TType.LIST, 4) oprot.writeListBegin(TType.STRUCT, len(self.row_groups)) for iter54 in self.row_groups: iter54.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.key_value_metadata is not None: oprot.writeFieldBegin('key_value_metadata', TType.LIST, 5) oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata)) for iter55 in self.key_value_metadata: iter55.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.created_by is not None: oprot.writeFieldBegin('created_by', TType.STRING, 6) oprot.writeString(self.created_by) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.version is None: raise TProtocol.TProtocolException(message='Required field version is unset!') if self.schema is None: raise TProtocol.TProtocolException(message='Required field schema is unset!') if self.num_rows is None: raise TProtocol.TProtocolException(message='Required field num_rows is unset!') if self.row_groups is None: raise TProtocol.TProtocolException(message='Required field row_groups is unset!') return def __repr__(self): L = ['%s=%r' % (key, value) for key, value in self.__dict__.iteritems()] return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other)