| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424 |
- #
- # Autogenerated by Thrift Compiler (0.9.0)
- #
- # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- #
- # options string: py
- #
- from thrift.Thrift import TType, TMessageType, TException, TApplicationException
- from thrift.transport import TTransport
- from thrift.protocol import TBinaryProtocol, TProtocol
- try:
- from thrift.protocol import fastbinary
- except:
- fastbinary = None
- class Type:
- """
- Types supported by Parquet. These types are intended to be used in combination
- with the encodings to control the on disk storage format.
- For example INT16 is not included as a type since a good encoding of INT32
- would handle this.
- """
- BOOLEAN = 0
- INT32 = 1
- INT64 = 2
- INT96 = 3
- FLOAT = 4
- DOUBLE = 5
- BYTE_ARRAY = 6
- FIXED_LEN_BYTE_ARRAY = 7
- _VALUES_TO_NAMES = {
- 0: "BOOLEAN",
- 1: "INT32",
- 2: "INT64",
- 3: "INT96",
- 4: "FLOAT",
- 5: "DOUBLE",
- 6: "BYTE_ARRAY",
- 7: "FIXED_LEN_BYTE_ARRAY",
- }
- _NAMES_TO_VALUES = {
- "BOOLEAN": 0,
- "INT32": 1,
- "INT64": 2,
- "INT96": 3,
- "FLOAT": 4,
- "DOUBLE": 5,
- "BYTE_ARRAY": 6,
- "FIXED_LEN_BYTE_ARRAY": 7,
- }
- class ConvertedType:
- """
- Common types used by frameworks(e.g. hive, pig) using parquet. This helps map
- between types in those frameworks to the base types in parquet. This is only
- metadata and not needed to read or write the data.
- """
- UTF8 = 0
- MAP = 1
- MAP_KEY_VALUE = 2
- LIST = 3
- _VALUES_TO_NAMES = {
- 0: "UTF8",
- 1: "MAP",
- 2: "MAP_KEY_VALUE",
- 3: "LIST",
- }
- _NAMES_TO_VALUES = {
- "UTF8": 0,
- "MAP": 1,
- "MAP_KEY_VALUE": 2,
- "LIST": 3,
- }
- class FieldRepetitionType:
- """
- Representation of Schemas
- """
- REQUIRED = 0
- OPTIONAL = 1
- REPEATED = 2
- _VALUES_TO_NAMES = {
- 0: "REQUIRED",
- 1: "OPTIONAL",
- 2: "REPEATED",
- }
- _NAMES_TO_VALUES = {
- "REQUIRED": 0,
- "OPTIONAL": 1,
- "REPEATED": 2,
- }
- class Encoding:
- """
- Encodings supported by Parquet. Not all encodings are valid for all types. These
- enums are also used to specify the encoding of definition and repetition levels.
- See the accompanying doc for the details of the more complicated encodings.
- """
- PLAIN = 0
- GROUP_VAR_INT = 1
- PLAIN_DICTIONARY = 2
- RLE = 3
- BIT_PACKED = 4
- _VALUES_TO_NAMES = {
- 0: "PLAIN",
- 1: "GROUP_VAR_INT",
- 2: "PLAIN_DICTIONARY",
- 3: "RLE",
- 4: "BIT_PACKED",
- }
- _NAMES_TO_VALUES = {
- "PLAIN": 0,
- "GROUP_VAR_INT": 1,
- "PLAIN_DICTIONARY": 2,
- "RLE": 3,
- "BIT_PACKED": 4,
- }
- class CompressionCodec:
- """
- Supported compression algorithms.
- """
- UNCOMPRESSED = 0
- SNAPPY = 1
- GZIP = 2
- LZO = 3
- _VALUES_TO_NAMES = {
- 0: "UNCOMPRESSED",
- 1: "SNAPPY",
- 2: "GZIP",
- 3: "LZO",
- }
- _NAMES_TO_VALUES = {
- "UNCOMPRESSED": 0,
- "SNAPPY": 1,
- "GZIP": 2,
- "LZO": 3,
- }
- class PageType:
- DATA_PAGE = 0
- INDEX_PAGE = 1
- DICTIONARY_PAGE = 2
- _VALUES_TO_NAMES = {
- 0: "DATA_PAGE",
- 1: "INDEX_PAGE",
- 2: "DICTIONARY_PAGE",
- }
- _NAMES_TO_VALUES = {
- "DATA_PAGE": 0,
- "INDEX_PAGE": 1,
- "DICTIONARY_PAGE": 2,
- }
- class SchemaElement:
- """
- Represents a element inside a schema definition.
- - if it is a group (inner node) then type is undefined and num_children is defined
- - if it is a primitive type (leaf) then type is defined and num_children is undefined
- the nodes are listed in depth first traversal order.
- Attributes:
- - type: Data type for this field. Not set if the current element is a non-leaf node
- - type_length: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
- Otherwise, if specified, this is the maximum bit length to store any of the values.
- (e.g. a low cardinality INT col could have this set to 3). Note that this is
- in the schema, and therefore fixed for the entire file.
- - repetition_type: repetition of the field. The root of the schema does not have a repetition_type.
- All other nodes must have one
- - name: Name of the field in the schema
- - num_children: Nested fields. Since thrift does not support nested fields,
- the nesting is flattened to a single list by a depth-first traversal.
- The children count is used to construct the nested relationship.
- This field is not set when the element is a primitive type
- - converted_type: When the schema is the result of a conversion from another model
- Used to record the original type to help with cross conversion.
- """
- thrift_spec = (
- None, # 0
- (1, TType.I32, 'type', None, None, ), # 1
- (2, TType.I32, 'type_length', None, None, ), # 2
- (3, TType.I32, 'repetition_type', None, None, ), # 3
- (4, TType.STRING, 'name', None, None, ), # 4
- (5, TType.I32, 'num_children', None, None, ), # 5
- (6, TType.I32, 'converted_type', None, None, ), # 6
- )
- def __init__(self, type=None, type_length=None, repetition_type=None, name=None, num_children=None, converted_type=None,):
- self.type = type
- self.type_length = type_length
- self.repetition_type = repetition_type
- self.name = name
- self.num_children = num_children
- self.converted_type = converted_type
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.I32:
- self.type = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.I32:
- self.type_length = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 3:
- if ftype == TType.I32:
- self.repetition_type = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 4:
- if ftype == TType.STRING:
- self.name = iprot.readString();
- else:
- iprot.skip(ftype)
- elif fid == 5:
- if ftype == TType.I32:
- self.num_children = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 6:
- if ftype == TType.I32:
- self.converted_type = iprot.readI32();
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('SchemaElement')
- if self.type is not None:
- oprot.writeFieldBegin('type', TType.I32, 1)
- oprot.writeI32(self.type)
- oprot.writeFieldEnd()
- if self.type_length is not None:
- oprot.writeFieldBegin('type_length', TType.I32, 2)
- oprot.writeI32(self.type_length)
- oprot.writeFieldEnd()
- if self.repetition_type is not None:
- oprot.writeFieldBegin('repetition_type', TType.I32, 3)
- oprot.writeI32(self.repetition_type)
- oprot.writeFieldEnd()
- if self.name is not None:
- oprot.writeFieldBegin('name', TType.STRING, 4)
- oprot.writeString(self.name)
- oprot.writeFieldEnd()
- if self.num_children is not None:
- oprot.writeFieldBegin('num_children', TType.I32, 5)
- oprot.writeI32(self.num_children)
- oprot.writeFieldEnd()
- if self.converted_type is not None:
- oprot.writeFieldBegin('converted_type', TType.I32, 6)
- oprot.writeI32(self.converted_type)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.name is None:
- raise TProtocol.TProtocolException(message='Required field name is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class DataPageHeader:
- """
- Data page header
- Attributes:
- - num_values: Number of values, including NULLs, in this data page. *
- - encoding: Encoding used for this data page *
- - definition_level_encoding: Encoding used for definition levels *
- - repetition_level_encoding: Encoding used for repetition levels *
- """
- thrift_spec = (
- None, # 0
- (1, TType.I32, 'num_values', None, None, ), # 1
- (2, TType.I32, 'encoding', None, None, ), # 2
- (3, TType.I32, 'definition_level_encoding', None, None, ), # 3
- (4, TType.I32, 'repetition_level_encoding', None, None, ), # 4
- )
- def __init__(self, num_values=None, encoding=None, definition_level_encoding=None, repetition_level_encoding=None,):
- self.num_values = num_values
- self.encoding = encoding
- self.definition_level_encoding = definition_level_encoding
- self.repetition_level_encoding = repetition_level_encoding
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.I32:
- self.num_values = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.I32:
- self.encoding = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 3:
- if ftype == TType.I32:
- self.definition_level_encoding = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 4:
- if ftype == TType.I32:
- self.repetition_level_encoding = iprot.readI32();
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('DataPageHeader')
- if self.num_values is not None:
- oprot.writeFieldBegin('num_values', TType.I32, 1)
- oprot.writeI32(self.num_values)
- oprot.writeFieldEnd()
- if self.encoding is not None:
- oprot.writeFieldBegin('encoding', TType.I32, 2)
- oprot.writeI32(self.encoding)
- oprot.writeFieldEnd()
- if self.definition_level_encoding is not None:
- oprot.writeFieldBegin('definition_level_encoding', TType.I32, 3)
- oprot.writeI32(self.definition_level_encoding)
- oprot.writeFieldEnd()
- if self.repetition_level_encoding is not None:
- oprot.writeFieldBegin('repetition_level_encoding', TType.I32, 4)
- oprot.writeI32(self.repetition_level_encoding)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.num_values is None:
- raise TProtocol.TProtocolException(message='Required field num_values is unset!')
- if self.encoding is None:
- raise TProtocol.TProtocolException(message='Required field encoding is unset!')
- if self.definition_level_encoding is None:
- raise TProtocol.TProtocolException(message='Required field definition_level_encoding is unset!')
- if self.repetition_level_encoding is None:
- raise TProtocol.TProtocolException(message='Required field repetition_level_encoding is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class IndexPageHeader:
- thrift_spec = (
- )
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('IndexPageHeader')
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class DictionaryPageHeader:
- """
- TODO: *
- Attributes:
- - num_values: Number of values in the dictionary *
- - encoding: Encoding using this dictionary page *
- """
- thrift_spec = (
- None, # 0
- (1, TType.I32, 'num_values', None, None, ), # 1
- (2, TType.I32, 'encoding', None, None, ), # 2
- )
- def __init__(self, num_values=None, encoding=None,):
- self.num_values = num_values
- self.encoding = encoding
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.I32:
- self.num_values = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.I32:
- self.encoding = iprot.readI32();
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('DictionaryPageHeader')
- if self.num_values is not None:
- oprot.writeFieldBegin('num_values', TType.I32, 1)
- oprot.writeI32(self.num_values)
- oprot.writeFieldEnd()
- if self.encoding is not None:
- oprot.writeFieldBegin('encoding', TType.I32, 2)
- oprot.writeI32(self.encoding)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.num_values is None:
- raise TProtocol.TProtocolException(message='Required field num_values is unset!')
- if self.encoding is None:
- raise TProtocol.TProtocolException(message='Required field encoding is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class PageHeader:
- """
- Attributes:
- - type: the type of the page: indicates which of the *_header fields is set *
- - uncompressed_page_size: Uncompressed page size in bytes (not including this header) *
- - compressed_page_size: Compressed page size in bytes (not including this header) *
- - crc: 32bit crc for the data below. This allows for disabling checksumming in HDFS
- if only a few pages needs to be read
- - data_page_header
- - index_page_header
- - dictionary_page_header
- """
- thrift_spec = (
- None, # 0
- (1, TType.I32, 'type', None, None, ), # 1
- (2, TType.I32, 'uncompressed_page_size', None, None, ), # 2
- (3, TType.I32, 'compressed_page_size', None, None, ), # 3
- (4, TType.I32, 'crc', None, None, ), # 4
- (5, TType.STRUCT, 'data_page_header', (DataPageHeader, DataPageHeader.thrift_spec), None, ), # 5
- (6, TType.STRUCT, 'index_page_header', (IndexPageHeader, IndexPageHeader.thrift_spec), None, ), # 6
- (7, TType.STRUCT, 'dictionary_page_header', (DictionaryPageHeader, DictionaryPageHeader.thrift_spec), None, ), # 7
- )
- def __init__(self, type=None, uncompressed_page_size=None, compressed_page_size=None, crc=None, data_page_header=None, index_page_header=None, dictionary_page_header=None,):
- self.type = type
- self.uncompressed_page_size = uncompressed_page_size
- self.compressed_page_size = compressed_page_size
- self.crc = crc
- self.data_page_header = data_page_header
- self.index_page_header = index_page_header
- self.dictionary_page_header = dictionary_page_header
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.I32:
- self.type = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.I32:
- self.uncompressed_page_size = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 3:
- if ftype == TType.I32:
- self.compressed_page_size = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 4:
- if ftype == TType.I32:
- self.crc = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 5:
- if ftype == TType.STRUCT:
- self.data_page_header = DataPageHeader()
- self.data_page_header.read(iprot)
- else:
- iprot.skip(ftype)
- elif fid == 6:
- if ftype == TType.STRUCT:
- self.index_page_header = IndexPageHeader()
- self.index_page_header.read(iprot)
- else:
- iprot.skip(ftype)
- elif fid == 7:
- if ftype == TType.STRUCT:
- self.dictionary_page_header = DictionaryPageHeader()
- self.dictionary_page_header.read(iprot)
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('PageHeader')
- if self.type is not None:
- oprot.writeFieldBegin('type', TType.I32, 1)
- oprot.writeI32(self.type)
- oprot.writeFieldEnd()
- if self.uncompressed_page_size is not None:
- oprot.writeFieldBegin('uncompressed_page_size', TType.I32, 2)
- oprot.writeI32(self.uncompressed_page_size)
- oprot.writeFieldEnd()
- if self.compressed_page_size is not None:
- oprot.writeFieldBegin('compressed_page_size', TType.I32, 3)
- oprot.writeI32(self.compressed_page_size)
- oprot.writeFieldEnd()
- if self.crc is not None:
- oprot.writeFieldBegin('crc', TType.I32, 4)
- oprot.writeI32(self.crc)
- oprot.writeFieldEnd()
- if self.data_page_header is not None:
- oprot.writeFieldBegin('data_page_header', TType.STRUCT, 5)
- self.data_page_header.write(oprot)
- oprot.writeFieldEnd()
- if self.index_page_header is not None:
- oprot.writeFieldBegin('index_page_header', TType.STRUCT, 6)
- self.index_page_header.write(oprot)
- oprot.writeFieldEnd()
- if self.dictionary_page_header is not None:
- oprot.writeFieldBegin('dictionary_page_header', TType.STRUCT, 7)
- self.dictionary_page_header.write(oprot)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.type is None:
- raise TProtocol.TProtocolException(message='Required field type is unset!')
- if self.uncompressed_page_size is None:
- raise TProtocol.TProtocolException(message='Required field uncompressed_page_size is unset!')
- if self.compressed_page_size is None:
- raise TProtocol.TProtocolException(message='Required field compressed_page_size is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class KeyValue:
- """
- Wrapper struct to store key values
- Attributes:
- - key
- - value
- """
- thrift_spec = (
- None, # 0
- (1, TType.STRING, 'key', None, None, ), # 1
- (2, TType.STRING, 'value', None, None, ), # 2
- )
- def __init__(self, key=None, value=None,):
- self.key = key
- self.value = value
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.STRING:
- self.key = iprot.readString();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.STRING:
- self.value = iprot.readString();
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('KeyValue')
- if self.key is not None:
- oprot.writeFieldBegin('key', TType.STRING, 1)
- oprot.writeString(self.key)
- oprot.writeFieldEnd()
- if self.value is not None:
- oprot.writeFieldBegin('value', TType.STRING, 2)
- oprot.writeString(self.value)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.key is None:
- raise TProtocol.TProtocolException(message='Required field key is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class SortingColumn:
- """
- Wrapper struct to specify sort order
- Attributes:
- - nulls_first: The column index (in this row group)
- 1: required i32 column_idx
- /** If true, indicates this column is sorted in descending order.
- 2: required bool descending
- /** If true, nulls will come before non-null values, otherwise,
- * nulls go at the end.
- """
- thrift_spec = (
- None, # 0
- None, # 1
- None, # 2
- (3, TType.BOOL, 'nulls_first', None, None, ), # 3
- )
- def __init__(self, nulls_first=None,):
- self.nulls_first = nulls_first
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 3:
- if ftype == TType.BOOL:
- self.nulls_first = iprot.readBool();
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('SortingColumn')
- if self.nulls_first is not None:
- oprot.writeFieldBegin('nulls_first', TType.BOOL, 3)
- oprot.writeBool(self.nulls_first)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.nulls_first is None:
- raise TProtocol.TProtocolException(message='Required field nulls_first is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class ColumnMetaData:
- """
- Description for column metadata
- Attributes:
- - type: Type of this column *
- - encodings: Set of all encodings used for this column. The purpose is to validate
- whether we can decode those pages. *
- - path_in_schema: Path in schema *
- - codec: Compression codec *
- - num_values: Number of values in this column *
- - total_uncompressed_size: total byte size of all uncompressed pages in this column chunk (including the headers) *
- - total_compressed_size: total byte size of all compressed pages in this column chunk (including the headers) *
- - key_value_metadata: Optional key/value metadata *
- - data_page_offset: Byte offset from beginning of file to first data page *
- - index_page_offset: Byte offset from beginning of file to root index page *
- - dictionary_page_offset: Byte offset from the beginning of file to first (only) dictionary page *
- """
- thrift_spec = (
- None, # 0
- (1, TType.I32, 'type', None, None, ), # 1
- (2, TType.LIST, 'encodings', (TType.I32,None), None, ), # 2
- (3, TType.LIST, 'path_in_schema', (TType.STRING,None), None, ), # 3
- (4, TType.I32, 'codec', None, None, ), # 4
- (5, TType.I64, 'num_values', None, None, ), # 5
- (6, TType.I64, 'total_uncompressed_size', None, None, ), # 6
- (7, TType.I64, 'total_compressed_size', None, None, ), # 7
- (8, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 8
- (9, TType.I64, 'data_page_offset', None, None, ), # 9
- (10, TType.I64, 'index_page_offset', None, None, ), # 10
- (11, TType.I64, 'dictionary_page_offset', None, None, ), # 11
- )
- def __init__(self, type=None, encodings=None, path_in_schema=None, codec=None, num_values=None, total_uncompressed_size=None, total_compressed_size=None, key_value_metadata=None, data_page_offset=None, index_page_offset=None, dictionary_page_offset=None,):
- self.type = type
- self.encodings = encodings
- self.path_in_schema = path_in_schema
- self.codec = codec
- self.num_values = num_values
- self.total_uncompressed_size = total_uncompressed_size
- self.total_compressed_size = total_compressed_size
- self.key_value_metadata = key_value_metadata
- self.data_page_offset = data_page_offset
- self.index_page_offset = index_page_offset
- self.dictionary_page_offset = dictionary_page_offset
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.I32:
- self.type = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.LIST:
- self.encodings = []
- (_etype3, _size0) = iprot.readListBegin()
- for _i4 in xrange(_size0):
- _elem5 = iprot.readI32();
- self.encodings.append(_elem5)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- elif fid == 3:
- if ftype == TType.LIST:
- self.path_in_schema = []
- (_etype9, _size6) = iprot.readListBegin()
- for _i10 in xrange(_size6):
- _elem11 = iprot.readString();
- self.path_in_schema.append(_elem11)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- elif fid == 4:
- if ftype == TType.I32:
- self.codec = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 5:
- if ftype == TType.I64:
- self.num_values = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 6:
- if ftype == TType.I64:
- self.total_uncompressed_size = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 7:
- if ftype == TType.I64:
- self.total_compressed_size = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 8:
- if ftype == TType.LIST:
- self.key_value_metadata = []
- (_etype15, _size12) = iprot.readListBegin()
- for _i16 in xrange(_size12):
- _elem17 = KeyValue()
- _elem17.read(iprot)
- self.key_value_metadata.append(_elem17)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- elif fid == 9:
- if ftype == TType.I64:
- self.data_page_offset = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 10:
- if ftype == TType.I64:
- self.index_page_offset = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 11:
- if ftype == TType.I64:
- self.dictionary_page_offset = iprot.readI64();
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('ColumnMetaData')
- if self.type is not None:
- oprot.writeFieldBegin('type', TType.I32, 1)
- oprot.writeI32(self.type)
- oprot.writeFieldEnd()
- if self.encodings is not None:
- oprot.writeFieldBegin('encodings', TType.LIST, 2)
- oprot.writeListBegin(TType.I32, len(self.encodings))
- for iter18 in self.encodings:
- oprot.writeI32(iter18)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- if self.path_in_schema is not None:
- oprot.writeFieldBegin('path_in_schema', TType.LIST, 3)
- oprot.writeListBegin(TType.STRING, len(self.path_in_schema))
- for iter19 in self.path_in_schema:
- oprot.writeString(iter19)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- if self.codec is not None:
- oprot.writeFieldBegin('codec', TType.I32, 4)
- oprot.writeI32(self.codec)
- oprot.writeFieldEnd()
- if self.num_values is not None:
- oprot.writeFieldBegin('num_values', TType.I64, 5)
- oprot.writeI64(self.num_values)
- oprot.writeFieldEnd()
- if self.total_uncompressed_size is not None:
- oprot.writeFieldBegin('total_uncompressed_size', TType.I64, 6)
- oprot.writeI64(self.total_uncompressed_size)
- oprot.writeFieldEnd()
- if self.total_compressed_size is not None:
- oprot.writeFieldBegin('total_compressed_size', TType.I64, 7)
- oprot.writeI64(self.total_compressed_size)
- oprot.writeFieldEnd()
- if self.key_value_metadata is not None:
- oprot.writeFieldBegin('key_value_metadata', TType.LIST, 8)
- oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
- for iter20 in self.key_value_metadata:
- iter20.write(oprot)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- if self.data_page_offset is not None:
- oprot.writeFieldBegin('data_page_offset', TType.I64, 9)
- oprot.writeI64(self.data_page_offset)
- oprot.writeFieldEnd()
- if self.index_page_offset is not None:
- oprot.writeFieldBegin('index_page_offset', TType.I64, 10)
- oprot.writeI64(self.index_page_offset)
- oprot.writeFieldEnd()
- if self.dictionary_page_offset is not None:
- oprot.writeFieldBegin('dictionary_page_offset', TType.I64, 11)
- oprot.writeI64(self.dictionary_page_offset)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.type is None:
- raise TProtocol.TProtocolException(message='Required field type is unset!')
- if self.encodings is None:
- raise TProtocol.TProtocolException(message='Required field encodings is unset!')
- if self.path_in_schema is None:
- raise TProtocol.TProtocolException(message='Required field path_in_schema is unset!')
- if self.codec is None:
- raise TProtocol.TProtocolException(message='Required field codec is unset!')
- if self.num_values is None:
- raise TProtocol.TProtocolException(message='Required field num_values is unset!')
- if self.total_uncompressed_size is None:
- raise TProtocol.TProtocolException(message='Required field total_uncompressed_size is unset!')
- if self.total_compressed_size is None:
- raise TProtocol.TProtocolException(message='Required field total_compressed_size is unset!')
- if self.data_page_offset is None:
- raise TProtocol.TProtocolException(message='Required field data_page_offset is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class ColumnChunk:
- """
- Attributes:
- - file_path: File where column data is stored. If not set, assumed to be same file as
- metadata. This path is relative to the current file.
- - file_offset: Byte offset in file_path to the ColumnMetaData *
- - meta_data: Column metadata for this chunk. This is the same content as what is at
- file_path/file_offset. Having it here has it replicated in the file
- metadata.
- """
- thrift_spec = (
- None, # 0
- (1, TType.STRING, 'file_path', None, None, ), # 1
- (2, TType.I64, 'file_offset', None, None, ), # 2
- (3, TType.STRUCT, 'meta_data', (ColumnMetaData, ColumnMetaData.thrift_spec), None, ), # 3
- )
- def __init__(self, file_path=None, file_offset=None, meta_data=None,):
- self.file_path = file_path
- self.file_offset = file_offset
- self.meta_data = meta_data
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.STRING:
- self.file_path = iprot.readString();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.I64:
- self.file_offset = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 3:
- if ftype == TType.STRUCT:
- self.meta_data = ColumnMetaData()
- self.meta_data.read(iprot)
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('ColumnChunk')
- if self.file_path is not None:
- oprot.writeFieldBegin('file_path', TType.STRING, 1)
- oprot.writeString(self.file_path)
- oprot.writeFieldEnd()
- if self.file_offset is not None:
- oprot.writeFieldBegin('file_offset', TType.I64, 2)
- oprot.writeI64(self.file_offset)
- oprot.writeFieldEnd()
- if self.meta_data is not None:
- oprot.writeFieldBegin('meta_data', TType.STRUCT, 3)
- self.meta_data.write(oprot)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.file_offset is None:
- raise TProtocol.TProtocolException(message='Required field file_offset is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class RowGroup:
- """
- Attributes:
- - columns
- - total_byte_size: Total byte size of all the uncompressed column data in this row group *
- - num_rows: Number of rows in this row group *
- - sorting_columns: If set, specifies a sort ordering of the rows in this RowGroup.
- The sorting columns can be a subset of all the columns.
- """
- thrift_spec = (
- None, # 0
- (1, TType.LIST, 'columns', (TType.STRUCT,(ColumnChunk, ColumnChunk.thrift_spec)), None, ), # 1
- (2, TType.I64, 'total_byte_size', None, None, ), # 2
- (3, TType.I64, 'num_rows', None, None, ), # 3
- (4, TType.LIST, 'sorting_columns', (TType.STRUCT,(SortingColumn, SortingColumn.thrift_spec)), None, ), # 4
- )
- def __init__(self, columns=None, total_byte_size=None, num_rows=None, sorting_columns=None,):
- self.columns = columns
- self.total_byte_size = total_byte_size
- self.num_rows = num_rows
- self.sorting_columns = sorting_columns
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.LIST:
- self.columns = []
- (_etype24, _size21) = iprot.readListBegin()
- for _i25 in xrange(_size21):
- _elem26 = ColumnChunk()
- _elem26.read(iprot)
- self.columns.append(_elem26)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.I64:
- self.total_byte_size = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 3:
- if ftype == TType.I64:
- self.num_rows = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 4:
- if ftype == TType.LIST:
- self.sorting_columns = []
- (_etype30, _size27) = iprot.readListBegin()
- for _i31 in xrange(_size27):
- _elem32 = SortingColumn()
- _elem32.read(iprot)
- self.sorting_columns.append(_elem32)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('RowGroup')
- if self.columns is not None:
- oprot.writeFieldBegin('columns', TType.LIST, 1)
- oprot.writeListBegin(TType.STRUCT, len(self.columns))
- for iter33 in self.columns:
- iter33.write(oprot)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- if self.total_byte_size is not None:
- oprot.writeFieldBegin('total_byte_size', TType.I64, 2)
- oprot.writeI64(self.total_byte_size)
- oprot.writeFieldEnd()
- if self.num_rows is not None:
- oprot.writeFieldBegin('num_rows', TType.I64, 3)
- oprot.writeI64(self.num_rows)
- oprot.writeFieldEnd()
- if self.sorting_columns is not None:
- oprot.writeFieldBegin('sorting_columns', TType.LIST, 4)
- oprot.writeListBegin(TType.STRUCT, len(self.sorting_columns))
- for iter34 in self.sorting_columns:
- iter34.write(oprot)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.columns is None:
- raise TProtocol.TProtocolException(message='Required field columns is unset!')
- if self.total_byte_size is None:
- raise TProtocol.TProtocolException(message='Required field total_byte_size is unset!')
- if self.num_rows is None:
- raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
- class FileMetaData:
- """
- Description for file metadata
- Attributes:
- - version: Version of this file *
- - schema: Parquet schema for this file. This schema contains metadata for all the columns.
- The schema is represented as a tree with a single root. The nodes of the tree
- are flattened to a list by doing a depth-first traversal.
- The column metadata contains the path in the schema for that column which can be
- used to map columns to nodes in the schema.
- The first element is the root *
- - num_rows: Number of rows in this file *
- - row_groups: Row groups in this file *
- - key_value_metadata: Optional key/value metadata *
- - created_by: String for application that wrote this file. This should be in the format
- <Application> version <App Version> (build <App Build Hash>).
- e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
- """
- thrift_spec = (
- None, # 0
- (1, TType.I32, 'version', None, None, ), # 1
- (2, TType.LIST, 'schema', (TType.STRUCT,(SchemaElement, SchemaElement.thrift_spec)), None, ), # 2
- (3, TType.I64, 'num_rows', None, None, ), # 3
- (4, TType.LIST, 'row_groups', (TType.STRUCT,(RowGroup, RowGroup.thrift_spec)), None, ), # 4
- (5, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 5
- (6, TType.STRING, 'created_by', None, None, ), # 6
- )
- def __init__(self, version=None, schema=None, num_rows=None, row_groups=None, key_value_metadata=None, created_by=None,):
- self.version = version
- self.schema = schema
- self.num_rows = num_rows
- self.row_groups = row_groups
- self.key_value_metadata = key_value_metadata
- self.created_by = created_by
- def read(self, iprot):
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
- return
- iprot.readStructBegin()
- while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
- break
- if fid == 1:
- if ftype == TType.I32:
- self.version = iprot.readI32();
- else:
- iprot.skip(ftype)
- elif fid == 2:
- if ftype == TType.LIST:
- self.schema = []
- (_etype38, _size35) = iprot.readListBegin()
- for _i39 in xrange(_size35):
- _elem40 = SchemaElement()
- _elem40.read(iprot)
- self.schema.append(_elem40)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- elif fid == 3:
- if ftype == TType.I64:
- self.num_rows = iprot.readI64();
- else:
- iprot.skip(ftype)
- elif fid == 4:
- if ftype == TType.LIST:
- self.row_groups = []
- (_etype44, _size41) = iprot.readListBegin()
- for _i45 in xrange(_size41):
- _elem46 = RowGroup()
- _elem46.read(iprot)
- self.row_groups.append(_elem46)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- elif fid == 5:
- if ftype == TType.LIST:
- self.key_value_metadata = []
- (_etype50, _size47) = iprot.readListBegin()
- for _i51 in xrange(_size47):
- _elem52 = KeyValue()
- _elem52.read(iprot)
- self.key_value_metadata.append(_elem52)
- iprot.readListEnd()
- else:
- iprot.skip(ftype)
- elif fid == 6:
- if ftype == TType.STRING:
- self.created_by = iprot.readString();
- else:
- iprot.skip(ftype)
- else:
- iprot.skip(ftype)
- iprot.readFieldEnd()
- iprot.readStructEnd()
- def write(self, oprot):
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
- return
- oprot.writeStructBegin('FileMetaData')
- if self.version is not None:
- oprot.writeFieldBegin('version', TType.I32, 1)
- oprot.writeI32(self.version)
- oprot.writeFieldEnd()
- if self.schema is not None:
- oprot.writeFieldBegin('schema', TType.LIST, 2)
- oprot.writeListBegin(TType.STRUCT, len(self.schema))
- for iter53 in self.schema:
- iter53.write(oprot)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- if self.num_rows is not None:
- oprot.writeFieldBegin('num_rows', TType.I64, 3)
- oprot.writeI64(self.num_rows)
- oprot.writeFieldEnd()
- if self.row_groups is not None:
- oprot.writeFieldBegin('row_groups', TType.LIST, 4)
- oprot.writeListBegin(TType.STRUCT, len(self.row_groups))
- for iter54 in self.row_groups:
- iter54.write(oprot)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- if self.key_value_metadata is not None:
- oprot.writeFieldBegin('key_value_metadata', TType.LIST, 5)
- oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
- for iter55 in self.key_value_metadata:
- iter55.write(oprot)
- oprot.writeListEnd()
- oprot.writeFieldEnd()
- if self.created_by is not None:
- oprot.writeFieldBegin('created_by', TType.STRING, 6)
- oprot.writeString(self.created_by)
- oprot.writeFieldEnd()
- oprot.writeFieldStop()
- oprot.writeStructEnd()
- def validate(self):
- if self.version is None:
- raise TProtocol.TProtocolException(message='Required field version is unset!')
- if self.schema is None:
- raise TProtocol.TProtocolException(message='Required field schema is unset!')
- if self.num_rows is None:
- raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
- if self.row_groups is None:
- raise TProtocol.TProtocolException(message='Required field row_groups is unset!')
- return
- def __repr__(self):
- L = ['%s=%r' % (key, value)
- for key, value in self.__dict__.iteritems()]
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
- def __eq__(self, other):
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
- def __ne__(self, other):
- return not (self == other)
|