ttypes.py 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424
  1. #
  2. # Autogenerated by Thrift Compiler (0.9.0)
  3. #
  4. # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
  5. #
  6. # options string: py
  7. #
  8. from thrift.Thrift import TType, TMessageType, TException, TApplicationException
  9. from thrift.transport import TTransport
  10. from thrift.protocol import TBinaryProtocol, TProtocol
  11. try:
  12. from thrift.protocol import fastbinary
  13. except:
  14. fastbinary = None
  15. class Type:
  16. """
  17. Types supported by Parquet. These types are intended to be used in combination
  18. with the encodings to control the on disk storage format.
  19. For example INT16 is not included as a type since a good encoding of INT32
  20. would handle this.
  21. """
  22. BOOLEAN = 0
  23. INT32 = 1
  24. INT64 = 2
  25. INT96 = 3
  26. FLOAT = 4
  27. DOUBLE = 5
  28. BYTE_ARRAY = 6
  29. FIXED_LEN_BYTE_ARRAY = 7
  30. _VALUES_TO_NAMES = {
  31. 0: "BOOLEAN",
  32. 1: "INT32",
  33. 2: "INT64",
  34. 3: "INT96",
  35. 4: "FLOAT",
  36. 5: "DOUBLE",
  37. 6: "BYTE_ARRAY",
  38. 7: "FIXED_LEN_BYTE_ARRAY",
  39. }
  40. _NAMES_TO_VALUES = {
  41. "BOOLEAN": 0,
  42. "INT32": 1,
  43. "INT64": 2,
  44. "INT96": 3,
  45. "FLOAT": 4,
  46. "DOUBLE": 5,
  47. "BYTE_ARRAY": 6,
  48. "FIXED_LEN_BYTE_ARRAY": 7,
  49. }
  50. class ConvertedType:
  51. """
  52. Common types used by frameworks(e.g. hive, pig) using parquet. This helps map
  53. between types in those frameworks to the base types in parquet. This is only
  54. metadata and not needed to read or write the data.
  55. """
  56. UTF8 = 0
  57. MAP = 1
  58. MAP_KEY_VALUE = 2
  59. LIST = 3
  60. _VALUES_TO_NAMES = {
  61. 0: "UTF8",
  62. 1: "MAP",
  63. 2: "MAP_KEY_VALUE",
  64. 3: "LIST",
  65. }
  66. _NAMES_TO_VALUES = {
  67. "UTF8": 0,
  68. "MAP": 1,
  69. "MAP_KEY_VALUE": 2,
  70. "LIST": 3,
  71. }
  72. class FieldRepetitionType:
  73. """
  74. Representation of Schemas
  75. """
  76. REQUIRED = 0
  77. OPTIONAL = 1
  78. REPEATED = 2
  79. _VALUES_TO_NAMES = {
  80. 0: "REQUIRED",
  81. 1: "OPTIONAL",
  82. 2: "REPEATED",
  83. }
  84. _NAMES_TO_VALUES = {
  85. "REQUIRED": 0,
  86. "OPTIONAL": 1,
  87. "REPEATED": 2,
  88. }
  89. class Encoding:
  90. """
  91. Encodings supported by Parquet. Not all encodings are valid for all types. These
  92. enums are also used to specify the encoding of definition and repetition levels.
  93. See the accompanying doc for the details of the more complicated encodings.
  94. """
  95. PLAIN = 0
  96. GROUP_VAR_INT = 1
  97. PLAIN_DICTIONARY = 2
  98. RLE = 3
  99. BIT_PACKED = 4
  100. _VALUES_TO_NAMES = {
  101. 0: "PLAIN",
  102. 1: "GROUP_VAR_INT",
  103. 2: "PLAIN_DICTIONARY",
  104. 3: "RLE",
  105. 4: "BIT_PACKED",
  106. }
  107. _NAMES_TO_VALUES = {
  108. "PLAIN": 0,
  109. "GROUP_VAR_INT": 1,
  110. "PLAIN_DICTIONARY": 2,
  111. "RLE": 3,
  112. "BIT_PACKED": 4,
  113. }
  114. class CompressionCodec:
  115. """
  116. Supported compression algorithms.
  117. """
  118. UNCOMPRESSED = 0
  119. SNAPPY = 1
  120. GZIP = 2
  121. LZO = 3
  122. _VALUES_TO_NAMES = {
  123. 0: "UNCOMPRESSED",
  124. 1: "SNAPPY",
  125. 2: "GZIP",
  126. 3: "LZO",
  127. }
  128. _NAMES_TO_VALUES = {
  129. "UNCOMPRESSED": 0,
  130. "SNAPPY": 1,
  131. "GZIP": 2,
  132. "LZO": 3,
  133. }
  134. class PageType:
  135. DATA_PAGE = 0
  136. INDEX_PAGE = 1
  137. DICTIONARY_PAGE = 2
  138. _VALUES_TO_NAMES = {
  139. 0: "DATA_PAGE",
  140. 1: "INDEX_PAGE",
  141. 2: "DICTIONARY_PAGE",
  142. }
  143. _NAMES_TO_VALUES = {
  144. "DATA_PAGE": 0,
  145. "INDEX_PAGE": 1,
  146. "DICTIONARY_PAGE": 2,
  147. }
  148. class SchemaElement:
  149. """
  150. Represents a element inside a schema definition.
  151. - if it is a group (inner node) then type is undefined and num_children is defined
  152. - if it is a primitive type (leaf) then type is defined and num_children is undefined
  153. the nodes are listed in depth first traversal order.
  154. Attributes:
  155. - type: Data type for this field. Not set if the current element is a non-leaf node
  156. - type_length: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
  157. Otherwise, if specified, this is the maximum bit length to store any of the values.
  158. (e.g. a low cardinality INT col could have this set to 3). Note that this is
  159. in the schema, and therefore fixed for the entire file.
  160. - repetition_type: repetition of the field. The root of the schema does not have a repetition_type.
  161. All other nodes must have one
  162. - name: Name of the field in the schema
  163. - num_children: Nested fields. Since thrift does not support nested fields,
  164. the nesting is flattened to a single list by a depth-first traversal.
  165. The children count is used to construct the nested relationship.
  166. This field is not set when the element is a primitive type
  167. - converted_type: When the schema is the result of a conversion from another model
  168. Used to record the original type to help with cross conversion.
  169. """
  170. thrift_spec = (
  171. None, # 0
  172. (1, TType.I32, 'type', None, None, ), # 1
  173. (2, TType.I32, 'type_length', None, None, ), # 2
  174. (3, TType.I32, 'repetition_type', None, None, ), # 3
  175. (4, TType.STRING, 'name', None, None, ), # 4
  176. (5, TType.I32, 'num_children', None, None, ), # 5
  177. (6, TType.I32, 'converted_type', None, None, ), # 6
  178. )
  179. def __init__(self, type=None, type_length=None, repetition_type=None, name=None, num_children=None, converted_type=None,):
  180. self.type = type
  181. self.type_length = type_length
  182. self.repetition_type = repetition_type
  183. self.name = name
  184. self.num_children = num_children
  185. self.converted_type = converted_type
  186. def read(self, iprot):
  187. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  188. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  189. return
  190. iprot.readStructBegin()
  191. while True:
  192. (fname, ftype, fid) = iprot.readFieldBegin()
  193. if ftype == TType.STOP:
  194. break
  195. if fid == 1:
  196. if ftype == TType.I32:
  197. self.type = iprot.readI32();
  198. else:
  199. iprot.skip(ftype)
  200. elif fid == 2:
  201. if ftype == TType.I32:
  202. self.type_length = iprot.readI32();
  203. else:
  204. iprot.skip(ftype)
  205. elif fid == 3:
  206. if ftype == TType.I32:
  207. self.repetition_type = iprot.readI32();
  208. else:
  209. iprot.skip(ftype)
  210. elif fid == 4:
  211. if ftype == TType.STRING:
  212. self.name = iprot.readString();
  213. else:
  214. iprot.skip(ftype)
  215. elif fid == 5:
  216. if ftype == TType.I32:
  217. self.num_children = iprot.readI32();
  218. else:
  219. iprot.skip(ftype)
  220. elif fid == 6:
  221. if ftype == TType.I32:
  222. self.converted_type = iprot.readI32();
  223. else:
  224. iprot.skip(ftype)
  225. else:
  226. iprot.skip(ftype)
  227. iprot.readFieldEnd()
  228. iprot.readStructEnd()
  229. def write(self, oprot):
  230. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  231. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  232. return
  233. oprot.writeStructBegin('SchemaElement')
  234. if self.type is not None:
  235. oprot.writeFieldBegin('type', TType.I32, 1)
  236. oprot.writeI32(self.type)
  237. oprot.writeFieldEnd()
  238. if self.type_length is not None:
  239. oprot.writeFieldBegin('type_length', TType.I32, 2)
  240. oprot.writeI32(self.type_length)
  241. oprot.writeFieldEnd()
  242. if self.repetition_type is not None:
  243. oprot.writeFieldBegin('repetition_type', TType.I32, 3)
  244. oprot.writeI32(self.repetition_type)
  245. oprot.writeFieldEnd()
  246. if self.name is not None:
  247. oprot.writeFieldBegin('name', TType.STRING, 4)
  248. oprot.writeString(self.name)
  249. oprot.writeFieldEnd()
  250. if self.num_children is not None:
  251. oprot.writeFieldBegin('num_children', TType.I32, 5)
  252. oprot.writeI32(self.num_children)
  253. oprot.writeFieldEnd()
  254. if self.converted_type is not None:
  255. oprot.writeFieldBegin('converted_type', TType.I32, 6)
  256. oprot.writeI32(self.converted_type)
  257. oprot.writeFieldEnd()
  258. oprot.writeFieldStop()
  259. oprot.writeStructEnd()
  260. def validate(self):
  261. if self.name is None:
  262. raise TProtocol.TProtocolException(message='Required field name is unset!')
  263. return
  264. def __repr__(self):
  265. L = ['%s=%r' % (key, value)
  266. for key, value in self.__dict__.iteritems()]
  267. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  268. def __eq__(self, other):
  269. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  270. def __ne__(self, other):
  271. return not (self == other)
  272. class DataPageHeader:
  273. """
  274. Data page header
  275. Attributes:
  276. - num_values: Number of values, including NULLs, in this data page. *
  277. - encoding: Encoding used for this data page *
  278. - definition_level_encoding: Encoding used for definition levels *
  279. - repetition_level_encoding: Encoding used for repetition levels *
  280. """
  281. thrift_spec = (
  282. None, # 0
  283. (1, TType.I32, 'num_values', None, None, ), # 1
  284. (2, TType.I32, 'encoding', None, None, ), # 2
  285. (3, TType.I32, 'definition_level_encoding', None, None, ), # 3
  286. (4, TType.I32, 'repetition_level_encoding', None, None, ), # 4
  287. )
  288. def __init__(self, num_values=None, encoding=None, definition_level_encoding=None, repetition_level_encoding=None,):
  289. self.num_values = num_values
  290. self.encoding = encoding
  291. self.definition_level_encoding = definition_level_encoding
  292. self.repetition_level_encoding = repetition_level_encoding
  293. def read(self, iprot):
  294. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  295. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  296. return
  297. iprot.readStructBegin()
  298. while True:
  299. (fname, ftype, fid) = iprot.readFieldBegin()
  300. if ftype == TType.STOP:
  301. break
  302. if fid == 1:
  303. if ftype == TType.I32:
  304. self.num_values = iprot.readI32();
  305. else:
  306. iprot.skip(ftype)
  307. elif fid == 2:
  308. if ftype == TType.I32:
  309. self.encoding = iprot.readI32();
  310. else:
  311. iprot.skip(ftype)
  312. elif fid == 3:
  313. if ftype == TType.I32:
  314. self.definition_level_encoding = iprot.readI32();
  315. else:
  316. iprot.skip(ftype)
  317. elif fid == 4:
  318. if ftype == TType.I32:
  319. self.repetition_level_encoding = iprot.readI32();
  320. else:
  321. iprot.skip(ftype)
  322. else:
  323. iprot.skip(ftype)
  324. iprot.readFieldEnd()
  325. iprot.readStructEnd()
  326. def write(self, oprot):
  327. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  328. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  329. return
  330. oprot.writeStructBegin('DataPageHeader')
  331. if self.num_values is not None:
  332. oprot.writeFieldBegin('num_values', TType.I32, 1)
  333. oprot.writeI32(self.num_values)
  334. oprot.writeFieldEnd()
  335. if self.encoding is not None:
  336. oprot.writeFieldBegin('encoding', TType.I32, 2)
  337. oprot.writeI32(self.encoding)
  338. oprot.writeFieldEnd()
  339. if self.definition_level_encoding is not None:
  340. oprot.writeFieldBegin('definition_level_encoding', TType.I32, 3)
  341. oprot.writeI32(self.definition_level_encoding)
  342. oprot.writeFieldEnd()
  343. if self.repetition_level_encoding is not None:
  344. oprot.writeFieldBegin('repetition_level_encoding', TType.I32, 4)
  345. oprot.writeI32(self.repetition_level_encoding)
  346. oprot.writeFieldEnd()
  347. oprot.writeFieldStop()
  348. oprot.writeStructEnd()
  349. def validate(self):
  350. if self.num_values is None:
  351. raise TProtocol.TProtocolException(message='Required field num_values is unset!')
  352. if self.encoding is None:
  353. raise TProtocol.TProtocolException(message='Required field encoding is unset!')
  354. if self.definition_level_encoding is None:
  355. raise TProtocol.TProtocolException(message='Required field definition_level_encoding is unset!')
  356. if self.repetition_level_encoding is None:
  357. raise TProtocol.TProtocolException(message='Required field repetition_level_encoding is unset!')
  358. return
  359. def __repr__(self):
  360. L = ['%s=%r' % (key, value)
  361. for key, value in self.__dict__.iteritems()]
  362. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  363. def __eq__(self, other):
  364. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  365. def __ne__(self, other):
  366. return not (self == other)
  367. class IndexPageHeader:
  368. thrift_spec = (
  369. )
  370. def read(self, iprot):
  371. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  372. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  373. return
  374. iprot.readStructBegin()
  375. while True:
  376. (fname, ftype, fid) = iprot.readFieldBegin()
  377. if ftype == TType.STOP:
  378. break
  379. else:
  380. iprot.skip(ftype)
  381. iprot.readFieldEnd()
  382. iprot.readStructEnd()
  383. def write(self, oprot):
  384. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  385. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  386. return
  387. oprot.writeStructBegin('IndexPageHeader')
  388. oprot.writeFieldStop()
  389. oprot.writeStructEnd()
  390. def validate(self):
  391. return
  392. def __repr__(self):
  393. L = ['%s=%r' % (key, value)
  394. for key, value in self.__dict__.iteritems()]
  395. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  396. def __eq__(self, other):
  397. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  398. def __ne__(self, other):
  399. return not (self == other)
  400. class DictionaryPageHeader:
  401. """
  402. TODO: *
  403. Attributes:
  404. - num_values: Number of values in the dictionary *
  405. - encoding: Encoding using this dictionary page *
  406. """
  407. thrift_spec = (
  408. None, # 0
  409. (1, TType.I32, 'num_values', None, None, ), # 1
  410. (2, TType.I32, 'encoding', None, None, ), # 2
  411. )
  412. def __init__(self, num_values=None, encoding=None,):
  413. self.num_values = num_values
  414. self.encoding = encoding
  415. def read(self, iprot):
  416. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  417. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  418. return
  419. iprot.readStructBegin()
  420. while True:
  421. (fname, ftype, fid) = iprot.readFieldBegin()
  422. if ftype == TType.STOP:
  423. break
  424. if fid == 1:
  425. if ftype == TType.I32:
  426. self.num_values = iprot.readI32();
  427. else:
  428. iprot.skip(ftype)
  429. elif fid == 2:
  430. if ftype == TType.I32:
  431. self.encoding = iprot.readI32();
  432. else:
  433. iprot.skip(ftype)
  434. else:
  435. iprot.skip(ftype)
  436. iprot.readFieldEnd()
  437. iprot.readStructEnd()
  438. def write(self, oprot):
  439. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  440. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  441. return
  442. oprot.writeStructBegin('DictionaryPageHeader')
  443. if self.num_values is not None:
  444. oprot.writeFieldBegin('num_values', TType.I32, 1)
  445. oprot.writeI32(self.num_values)
  446. oprot.writeFieldEnd()
  447. if self.encoding is not None:
  448. oprot.writeFieldBegin('encoding', TType.I32, 2)
  449. oprot.writeI32(self.encoding)
  450. oprot.writeFieldEnd()
  451. oprot.writeFieldStop()
  452. oprot.writeStructEnd()
  453. def validate(self):
  454. if self.num_values is None:
  455. raise TProtocol.TProtocolException(message='Required field num_values is unset!')
  456. if self.encoding is None:
  457. raise TProtocol.TProtocolException(message='Required field encoding is unset!')
  458. return
  459. def __repr__(self):
  460. L = ['%s=%r' % (key, value)
  461. for key, value in self.__dict__.iteritems()]
  462. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  463. def __eq__(self, other):
  464. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  465. def __ne__(self, other):
  466. return not (self == other)
  467. class PageHeader:
  468. """
  469. Attributes:
  470. - type: the type of the page: indicates which of the *_header fields is set *
  471. - uncompressed_page_size: Uncompressed page size in bytes (not including this header) *
  472. - compressed_page_size: Compressed page size in bytes (not including this header) *
  473. - crc: 32bit crc for the data below. This allows for disabling checksumming in HDFS
  474. if only a few pages needs to be read
  475. - data_page_header
  476. - index_page_header
  477. - dictionary_page_header
  478. """
  479. thrift_spec = (
  480. None, # 0
  481. (1, TType.I32, 'type', None, None, ), # 1
  482. (2, TType.I32, 'uncompressed_page_size', None, None, ), # 2
  483. (3, TType.I32, 'compressed_page_size', None, None, ), # 3
  484. (4, TType.I32, 'crc', None, None, ), # 4
  485. (5, TType.STRUCT, 'data_page_header', (DataPageHeader, DataPageHeader.thrift_spec), None, ), # 5
  486. (6, TType.STRUCT, 'index_page_header', (IndexPageHeader, IndexPageHeader.thrift_spec), None, ), # 6
  487. (7, TType.STRUCT, 'dictionary_page_header', (DictionaryPageHeader, DictionaryPageHeader.thrift_spec), None, ), # 7
  488. )
  489. def __init__(self, type=None, uncompressed_page_size=None, compressed_page_size=None, crc=None, data_page_header=None, index_page_header=None, dictionary_page_header=None,):
  490. self.type = type
  491. self.uncompressed_page_size = uncompressed_page_size
  492. self.compressed_page_size = compressed_page_size
  493. self.crc = crc
  494. self.data_page_header = data_page_header
  495. self.index_page_header = index_page_header
  496. self.dictionary_page_header = dictionary_page_header
  497. def read(self, iprot):
  498. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  499. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  500. return
  501. iprot.readStructBegin()
  502. while True:
  503. (fname, ftype, fid) = iprot.readFieldBegin()
  504. if ftype == TType.STOP:
  505. break
  506. if fid == 1:
  507. if ftype == TType.I32:
  508. self.type = iprot.readI32();
  509. else:
  510. iprot.skip(ftype)
  511. elif fid == 2:
  512. if ftype == TType.I32:
  513. self.uncompressed_page_size = iprot.readI32();
  514. else:
  515. iprot.skip(ftype)
  516. elif fid == 3:
  517. if ftype == TType.I32:
  518. self.compressed_page_size = iprot.readI32();
  519. else:
  520. iprot.skip(ftype)
  521. elif fid == 4:
  522. if ftype == TType.I32:
  523. self.crc = iprot.readI32();
  524. else:
  525. iprot.skip(ftype)
  526. elif fid == 5:
  527. if ftype == TType.STRUCT:
  528. self.data_page_header = DataPageHeader()
  529. self.data_page_header.read(iprot)
  530. else:
  531. iprot.skip(ftype)
  532. elif fid == 6:
  533. if ftype == TType.STRUCT:
  534. self.index_page_header = IndexPageHeader()
  535. self.index_page_header.read(iprot)
  536. else:
  537. iprot.skip(ftype)
  538. elif fid == 7:
  539. if ftype == TType.STRUCT:
  540. self.dictionary_page_header = DictionaryPageHeader()
  541. self.dictionary_page_header.read(iprot)
  542. else:
  543. iprot.skip(ftype)
  544. else:
  545. iprot.skip(ftype)
  546. iprot.readFieldEnd()
  547. iprot.readStructEnd()
  548. def write(self, oprot):
  549. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  550. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  551. return
  552. oprot.writeStructBegin('PageHeader')
  553. if self.type is not None:
  554. oprot.writeFieldBegin('type', TType.I32, 1)
  555. oprot.writeI32(self.type)
  556. oprot.writeFieldEnd()
  557. if self.uncompressed_page_size is not None:
  558. oprot.writeFieldBegin('uncompressed_page_size', TType.I32, 2)
  559. oprot.writeI32(self.uncompressed_page_size)
  560. oprot.writeFieldEnd()
  561. if self.compressed_page_size is not None:
  562. oprot.writeFieldBegin('compressed_page_size', TType.I32, 3)
  563. oprot.writeI32(self.compressed_page_size)
  564. oprot.writeFieldEnd()
  565. if self.crc is not None:
  566. oprot.writeFieldBegin('crc', TType.I32, 4)
  567. oprot.writeI32(self.crc)
  568. oprot.writeFieldEnd()
  569. if self.data_page_header is not None:
  570. oprot.writeFieldBegin('data_page_header', TType.STRUCT, 5)
  571. self.data_page_header.write(oprot)
  572. oprot.writeFieldEnd()
  573. if self.index_page_header is not None:
  574. oprot.writeFieldBegin('index_page_header', TType.STRUCT, 6)
  575. self.index_page_header.write(oprot)
  576. oprot.writeFieldEnd()
  577. if self.dictionary_page_header is not None:
  578. oprot.writeFieldBegin('dictionary_page_header', TType.STRUCT, 7)
  579. self.dictionary_page_header.write(oprot)
  580. oprot.writeFieldEnd()
  581. oprot.writeFieldStop()
  582. oprot.writeStructEnd()
  583. def validate(self):
  584. if self.type is None:
  585. raise TProtocol.TProtocolException(message='Required field type is unset!')
  586. if self.uncompressed_page_size is None:
  587. raise TProtocol.TProtocolException(message='Required field uncompressed_page_size is unset!')
  588. if self.compressed_page_size is None:
  589. raise TProtocol.TProtocolException(message='Required field compressed_page_size is unset!')
  590. return
  591. def __repr__(self):
  592. L = ['%s=%r' % (key, value)
  593. for key, value in self.__dict__.iteritems()]
  594. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  595. def __eq__(self, other):
  596. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  597. def __ne__(self, other):
  598. return not (self == other)
  599. class KeyValue:
  600. """
  601. Wrapper struct to store key values
  602. Attributes:
  603. - key
  604. - value
  605. """
  606. thrift_spec = (
  607. None, # 0
  608. (1, TType.STRING, 'key', None, None, ), # 1
  609. (2, TType.STRING, 'value', None, None, ), # 2
  610. )
  611. def __init__(self, key=None, value=None,):
  612. self.key = key
  613. self.value = value
  614. def read(self, iprot):
  615. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  616. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  617. return
  618. iprot.readStructBegin()
  619. while True:
  620. (fname, ftype, fid) = iprot.readFieldBegin()
  621. if ftype == TType.STOP:
  622. break
  623. if fid == 1:
  624. if ftype == TType.STRING:
  625. self.key = iprot.readString();
  626. else:
  627. iprot.skip(ftype)
  628. elif fid == 2:
  629. if ftype == TType.STRING:
  630. self.value = iprot.readString();
  631. else:
  632. iprot.skip(ftype)
  633. else:
  634. iprot.skip(ftype)
  635. iprot.readFieldEnd()
  636. iprot.readStructEnd()
  637. def write(self, oprot):
  638. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  639. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  640. return
  641. oprot.writeStructBegin('KeyValue')
  642. if self.key is not None:
  643. oprot.writeFieldBegin('key', TType.STRING, 1)
  644. oprot.writeString(self.key)
  645. oprot.writeFieldEnd()
  646. if self.value is not None:
  647. oprot.writeFieldBegin('value', TType.STRING, 2)
  648. oprot.writeString(self.value)
  649. oprot.writeFieldEnd()
  650. oprot.writeFieldStop()
  651. oprot.writeStructEnd()
  652. def validate(self):
  653. if self.key is None:
  654. raise TProtocol.TProtocolException(message='Required field key is unset!')
  655. return
  656. def __repr__(self):
  657. L = ['%s=%r' % (key, value)
  658. for key, value in self.__dict__.iteritems()]
  659. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  660. def __eq__(self, other):
  661. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  662. def __ne__(self, other):
  663. return not (self == other)
  664. class SortingColumn:
  665. """
  666. Wrapper struct to specify sort order
  667. Attributes:
  668. - nulls_first: The column index (in this row group)
  669. 1: required i32 column_idx
  670. /** If true, indicates this column is sorted in descending order.
  671. 2: required bool descending
  672. /** If true, nulls will come before non-null values, otherwise,
  673. * nulls go at the end.
  674. """
  675. thrift_spec = (
  676. None, # 0
  677. None, # 1
  678. None, # 2
  679. (3, TType.BOOL, 'nulls_first', None, None, ), # 3
  680. )
  681. def __init__(self, nulls_first=None,):
  682. self.nulls_first = nulls_first
  683. def read(self, iprot):
  684. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  685. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  686. return
  687. iprot.readStructBegin()
  688. while True:
  689. (fname, ftype, fid) = iprot.readFieldBegin()
  690. if ftype == TType.STOP:
  691. break
  692. if fid == 3:
  693. if ftype == TType.BOOL:
  694. self.nulls_first = iprot.readBool();
  695. else:
  696. iprot.skip(ftype)
  697. else:
  698. iprot.skip(ftype)
  699. iprot.readFieldEnd()
  700. iprot.readStructEnd()
  701. def write(self, oprot):
  702. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  703. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  704. return
  705. oprot.writeStructBegin('SortingColumn')
  706. if self.nulls_first is not None:
  707. oprot.writeFieldBegin('nulls_first', TType.BOOL, 3)
  708. oprot.writeBool(self.nulls_first)
  709. oprot.writeFieldEnd()
  710. oprot.writeFieldStop()
  711. oprot.writeStructEnd()
  712. def validate(self):
  713. if self.nulls_first is None:
  714. raise TProtocol.TProtocolException(message='Required field nulls_first is unset!')
  715. return
  716. def __repr__(self):
  717. L = ['%s=%r' % (key, value)
  718. for key, value in self.__dict__.iteritems()]
  719. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  720. def __eq__(self, other):
  721. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  722. def __ne__(self, other):
  723. return not (self == other)
  724. class ColumnMetaData:
  725. """
  726. Description for column metadata
  727. Attributes:
  728. - type: Type of this column *
  729. - encodings: Set of all encodings used for this column. The purpose is to validate
  730. whether we can decode those pages. *
  731. - path_in_schema: Path in schema *
  732. - codec: Compression codec *
  733. - num_values: Number of values in this column *
  734. - total_uncompressed_size: total byte size of all uncompressed pages in this column chunk (including the headers) *
  735. - total_compressed_size: total byte size of all compressed pages in this column chunk (including the headers) *
  736. - key_value_metadata: Optional key/value metadata *
  737. - data_page_offset: Byte offset from beginning of file to first data page *
  738. - index_page_offset: Byte offset from beginning of file to root index page *
  739. - dictionary_page_offset: Byte offset from the beginning of file to first (only) dictionary page *
  740. """
  741. thrift_spec = (
  742. None, # 0
  743. (1, TType.I32, 'type', None, None, ), # 1
  744. (2, TType.LIST, 'encodings', (TType.I32,None), None, ), # 2
  745. (3, TType.LIST, 'path_in_schema', (TType.STRING,None), None, ), # 3
  746. (4, TType.I32, 'codec', None, None, ), # 4
  747. (5, TType.I64, 'num_values', None, None, ), # 5
  748. (6, TType.I64, 'total_uncompressed_size', None, None, ), # 6
  749. (7, TType.I64, 'total_compressed_size', None, None, ), # 7
  750. (8, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 8
  751. (9, TType.I64, 'data_page_offset', None, None, ), # 9
  752. (10, TType.I64, 'index_page_offset', None, None, ), # 10
  753. (11, TType.I64, 'dictionary_page_offset', None, None, ), # 11
  754. )
  755. def __init__(self, type=None, encodings=None, path_in_schema=None, codec=None, num_values=None, total_uncompressed_size=None, total_compressed_size=None, key_value_metadata=None, data_page_offset=None, index_page_offset=None, dictionary_page_offset=None,):
  756. self.type = type
  757. self.encodings = encodings
  758. self.path_in_schema = path_in_schema
  759. self.codec = codec
  760. self.num_values = num_values
  761. self.total_uncompressed_size = total_uncompressed_size
  762. self.total_compressed_size = total_compressed_size
  763. self.key_value_metadata = key_value_metadata
  764. self.data_page_offset = data_page_offset
  765. self.index_page_offset = index_page_offset
  766. self.dictionary_page_offset = dictionary_page_offset
  767. def read(self, iprot):
  768. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  769. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  770. return
  771. iprot.readStructBegin()
  772. while True:
  773. (fname, ftype, fid) = iprot.readFieldBegin()
  774. if ftype == TType.STOP:
  775. break
  776. if fid == 1:
  777. if ftype == TType.I32:
  778. self.type = iprot.readI32();
  779. else:
  780. iprot.skip(ftype)
  781. elif fid == 2:
  782. if ftype == TType.LIST:
  783. self.encodings = []
  784. (_etype3, _size0) = iprot.readListBegin()
  785. for _i4 in xrange(_size0):
  786. _elem5 = iprot.readI32();
  787. self.encodings.append(_elem5)
  788. iprot.readListEnd()
  789. else:
  790. iprot.skip(ftype)
  791. elif fid == 3:
  792. if ftype == TType.LIST:
  793. self.path_in_schema = []
  794. (_etype9, _size6) = iprot.readListBegin()
  795. for _i10 in xrange(_size6):
  796. _elem11 = iprot.readString();
  797. self.path_in_schema.append(_elem11)
  798. iprot.readListEnd()
  799. else:
  800. iprot.skip(ftype)
  801. elif fid == 4:
  802. if ftype == TType.I32:
  803. self.codec = iprot.readI32();
  804. else:
  805. iprot.skip(ftype)
  806. elif fid == 5:
  807. if ftype == TType.I64:
  808. self.num_values = iprot.readI64();
  809. else:
  810. iprot.skip(ftype)
  811. elif fid == 6:
  812. if ftype == TType.I64:
  813. self.total_uncompressed_size = iprot.readI64();
  814. else:
  815. iprot.skip(ftype)
  816. elif fid == 7:
  817. if ftype == TType.I64:
  818. self.total_compressed_size = iprot.readI64();
  819. else:
  820. iprot.skip(ftype)
  821. elif fid == 8:
  822. if ftype == TType.LIST:
  823. self.key_value_metadata = []
  824. (_etype15, _size12) = iprot.readListBegin()
  825. for _i16 in xrange(_size12):
  826. _elem17 = KeyValue()
  827. _elem17.read(iprot)
  828. self.key_value_metadata.append(_elem17)
  829. iprot.readListEnd()
  830. else:
  831. iprot.skip(ftype)
  832. elif fid == 9:
  833. if ftype == TType.I64:
  834. self.data_page_offset = iprot.readI64();
  835. else:
  836. iprot.skip(ftype)
  837. elif fid == 10:
  838. if ftype == TType.I64:
  839. self.index_page_offset = iprot.readI64();
  840. else:
  841. iprot.skip(ftype)
  842. elif fid == 11:
  843. if ftype == TType.I64:
  844. self.dictionary_page_offset = iprot.readI64();
  845. else:
  846. iprot.skip(ftype)
  847. else:
  848. iprot.skip(ftype)
  849. iprot.readFieldEnd()
  850. iprot.readStructEnd()
  851. def write(self, oprot):
  852. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  853. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  854. return
  855. oprot.writeStructBegin('ColumnMetaData')
  856. if self.type is not None:
  857. oprot.writeFieldBegin('type', TType.I32, 1)
  858. oprot.writeI32(self.type)
  859. oprot.writeFieldEnd()
  860. if self.encodings is not None:
  861. oprot.writeFieldBegin('encodings', TType.LIST, 2)
  862. oprot.writeListBegin(TType.I32, len(self.encodings))
  863. for iter18 in self.encodings:
  864. oprot.writeI32(iter18)
  865. oprot.writeListEnd()
  866. oprot.writeFieldEnd()
  867. if self.path_in_schema is not None:
  868. oprot.writeFieldBegin('path_in_schema', TType.LIST, 3)
  869. oprot.writeListBegin(TType.STRING, len(self.path_in_schema))
  870. for iter19 in self.path_in_schema:
  871. oprot.writeString(iter19)
  872. oprot.writeListEnd()
  873. oprot.writeFieldEnd()
  874. if self.codec is not None:
  875. oprot.writeFieldBegin('codec', TType.I32, 4)
  876. oprot.writeI32(self.codec)
  877. oprot.writeFieldEnd()
  878. if self.num_values is not None:
  879. oprot.writeFieldBegin('num_values', TType.I64, 5)
  880. oprot.writeI64(self.num_values)
  881. oprot.writeFieldEnd()
  882. if self.total_uncompressed_size is not None:
  883. oprot.writeFieldBegin('total_uncompressed_size', TType.I64, 6)
  884. oprot.writeI64(self.total_uncompressed_size)
  885. oprot.writeFieldEnd()
  886. if self.total_compressed_size is not None:
  887. oprot.writeFieldBegin('total_compressed_size', TType.I64, 7)
  888. oprot.writeI64(self.total_compressed_size)
  889. oprot.writeFieldEnd()
  890. if self.key_value_metadata is not None:
  891. oprot.writeFieldBegin('key_value_metadata', TType.LIST, 8)
  892. oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
  893. for iter20 in self.key_value_metadata:
  894. iter20.write(oprot)
  895. oprot.writeListEnd()
  896. oprot.writeFieldEnd()
  897. if self.data_page_offset is not None:
  898. oprot.writeFieldBegin('data_page_offset', TType.I64, 9)
  899. oprot.writeI64(self.data_page_offset)
  900. oprot.writeFieldEnd()
  901. if self.index_page_offset is not None:
  902. oprot.writeFieldBegin('index_page_offset', TType.I64, 10)
  903. oprot.writeI64(self.index_page_offset)
  904. oprot.writeFieldEnd()
  905. if self.dictionary_page_offset is not None:
  906. oprot.writeFieldBegin('dictionary_page_offset', TType.I64, 11)
  907. oprot.writeI64(self.dictionary_page_offset)
  908. oprot.writeFieldEnd()
  909. oprot.writeFieldStop()
  910. oprot.writeStructEnd()
  911. def validate(self):
  912. if self.type is None:
  913. raise TProtocol.TProtocolException(message='Required field type is unset!')
  914. if self.encodings is None:
  915. raise TProtocol.TProtocolException(message='Required field encodings is unset!')
  916. if self.path_in_schema is None:
  917. raise TProtocol.TProtocolException(message='Required field path_in_schema is unset!')
  918. if self.codec is None:
  919. raise TProtocol.TProtocolException(message='Required field codec is unset!')
  920. if self.num_values is None:
  921. raise TProtocol.TProtocolException(message='Required field num_values is unset!')
  922. if self.total_uncompressed_size is None:
  923. raise TProtocol.TProtocolException(message='Required field total_uncompressed_size is unset!')
  924. if self.total_compressed_size is None:
  925. raise TProtocol.TProtocolException(message='Required field total_compressed_size is unset!')
  926. if self.data_page_offset is None:
  927. raise TProtocol.TProtocolException(message='Required field data_page_offset is unset!')
  928. return
  929. def __repr__(self):
  930. L = ['%s=%r' % (key, value)
  931. for key, value in self.__dict__.iteritems()]
  932. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  933. def __eq__(self, other):
  934. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  935. def __ne__(self, other):
  936. return not (self == other)
  937. class ColumnChunk:
  938. """
  939. Attributes:
  940. - file_path: File where column data is stored. If not set, assumed to be same file as
  941. metadata. This path is relative to the current file.
  942. - file_offset: Byte offset in file_path to the ColumnMetaData *
  943. - meta_data: Column metadata for this chunk. This is the same content as what is at
  944. file_path/file_offset. Having it here has it replicated in the file
  945. metadata.
  946. """
  947. thrift_spec = (
  948. None, # 0
  949. (1, TType.STRING, 'file_path', None, None, ), # 1
  950. (2, TType.I64, 'file_offset', None, None, ), # 2
  951. (3, TType.STRUCT, 'meta_data', (ColumnMetaData, ColumnMetaData.thrift_spec), None, ), # 3
  952. )
  953. def __init__(self, file_path=None, file_offset=None, meta_data=None,):
  954. self.file_path = file_path
  955. self.file_offset = file_offset
  956. self.meta_data = meta_data
  957. def read(self, iprot):
  958. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  959. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  960. return
  961. iprot.readStructBegin()
  962. while True:
  963. (fname, ftype, fid) = iprot.readFieldBegin()
  964. if ftype == TType.STOP:
  965. break
  966. if fid == 1:
  967. if ftype == TType.STRING:
  968. self.file_path = iprot.readString();
  969. else:
  970. iprot.skip(ftype)
  971. elif fid == 2:
  972. if ftype == TType.I64:
  973. self.file_offset = iprot.readI64();
  974. else:
  975. iprot.skip(ftype)
  976. elif fid == 3:
  977. if ftype == TType.STRUCT:
  978. self.meta_data = ColumnMetaData()
  979. self.meta_data.read(iprot)
  980. else:
  981. iprot.skip(ftype)
  982. else:
  983. iprot.skip(ftype)
  984. iprot.readFieldEnd()
  985. iprot.readStructEnd()
  986. def write(self, oprot):
  987. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  988. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  989. return
  990. oprot.writeStructBegin('ColumnChunk')
  991. if self.file_path is not None:
  992. oprot.writeFieldBegin('file_path', TType.STRING, 1)
  993. oprot.writeString(self.file_path)
  994. oprot.writeFieldEnd()
  995. if self.file_offset is not None:
  996. oprot.writeFieldBegin('file_offset', TType.I64, 2)
  997. oprot.writeI64(self.file_offset)
  998. oprot.writeFieldEnd()
  999. if self.meta_data is not None:
  1000. oprot.writeFieldBegin('meta_data', TType.STRUCT, 3)
  1001. self.meta_data.write(oprot)
  1002. oprot.writeFieldEnd()
  1003. oprot.writeFieldStop()
  1004. oprot.writeStructEnd()
  1005. def validate(self):
  1006. if self.file_offset is None:
  1007. raise TProtocol.TProtocolException(message='Required field file_offset is unset!')
  1008. return
  1009. def __repr__(self):
  1010. L = ['%s=%r' % (key, value)
  1011. for key, value in self.__dict__.iteritems()]
  1012. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  1013. def __eq__(self, other):
  1014. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  1015. def __ne__(self, other):
  1016. return not (self == other)
  1017. class RowGroup:
  1018. """
  1019. Attributes:
  1020. - columns
  1021. - total_byte_size: Total byte size of all the uncompressed column data in this row group *
  1022. - num_rows: Number of rows in this row group *
  1023. - sorting_columns: If set, specifies a sort ordering of the rows in this RowGroup.
  1024. The sorting columns can be a subset of all the columns.
  1025. """
  1026. thrift_spec = (
  1027. None, # 0
  1028. (1, TType.LIST, 'columns', (TType.STRUCT,(ColumnChunk, ColumnChunk.thrift_spec)), None, ), # 1
  1029. (2, TType.I64, 'total_byte_size', None, None, ), # 2
  1030. (3, TType.I64, 'num_rows', None, None, ), # 3
  1031. (4, TType.LIST, 'sorting_columns', (TType.STRUCT,(SortingColumn, SortingColumn.thrift_spec)), None, ), # 4
  1032. )
  1033. def __init__(self, columns=None, total_byte_size=None, num_rows=None, sorting_columns=None,):
  1034. self.columns = columns
  1035. self.total_byte_size = total_byte_size
  1036. self.num_rows = num_rows
  1037. self.sorting_columns = sorting_columns
  1038. def read(self, iprot):
  1039. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  1040. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  1041. return
  1042. iprot.readStructBegin()
  1043. while True:
  1044. (fname, ftype, fid) = iprot.readFieldBegin()
  1045. if ftype == TType.STOP:
  1046. break
  1047. if fid == 1:
  1048. if ftype == TType.LIST:
  1049. self.columns = []
  1050. (_etype24, _size21) = iprot.readListBegin()
  1051. for _i25 in xrange(_size21):
  1052. _elem26 = ColumnChunk()
  1053. _elem26.read(iprot)
  1054. self.columns.append(_elem26)
  1055. iprot.readListEnd()
  1056. else:
  1057. iprot.skip(ftype)
  1058. elif fid == 2:
  1059. if ftype == TType.I64:
  1060. self.total_byte_size = iprot.readI64();
  1061. else:
  1062. iprot.skip(ftype)
  1063. elif fid == 3:
  1064. if ftype == TType.I64:
  1065. self.num_rows = iprot.readI64();
  1066. else:
  1067. iprot.skip(ftype)
  1068. elif fid == 4:
  1069. if ftype == TType.LIST:
  1070. self.sorting_columns = []
  1071. (_etype30, _size27) = iprot.readListBegin()
  1072. for _i31 in xrange(_size27):
  1073. _elem32 = SortingColumn()
  1074. _elem32.read(iprot)
  1075. self.sorting_columns.append(_elem32)
  1076. iprot.readListEnd()
  1077. else:
  1078. iprot.skip(ftype)
  1079. else:
  1080. iprot.skip(ftype)
  1081. iprot.readFieldEnd()
  1082. iprot.readStructEnd()
  1083. def write(self, oprot):
  1084. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  1085. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  1086. return
  1087. oprot.writeStructBegin('RowGroup')
  1088. if self.columns is not None:
  1089. oprot.writeFieldBegin('columns', TType.LIST, 1)
  1090. oprot.writeListBegin(TType.STRUCT, len(self.columns))
  1091. for iter33 in self.columns:
  1092. iter33.write(oprot)
  1093. oprot.writeListEnd()
  1094. oprot.writeFieldEnd()
  1095. if self.total_byte_size is not None:
  1096. oprot.writeFieldBegin('total_byte_size', TType.I64, 2)
  1097. oprot.writeI64(self.total_byte_size)
  1098. oprot.writeFieldEnd()
  1099. if self.num_rows is not None:
  1100. oprot.writeFieldBegin('num_rows', TType.I64, 3)
  1101. oprot.writeI64(self.num_rows)
  1102. oprot.writeFieldEnd()
  1103. if self.sorting_columns is not None:
  1104. oprot.writeFieldBegin('sorting_columns', TType.LIST, 4)
  1105. oprot.writeListBegin(TType.STRUCT, len(self.sorting_columns))
  1106. for iter34 in self.sorting_columns:
  1107. iter34.write(oprot)
  1108. oprot.writeListEnd()
  1109. oprot.writeFieldEnd()
  1110. oprot.writeFieldStop()
  1111. oprot.writeStructEnd()
  1112. def validate(self):
  1113. if self.columns is None:
  1114. raise TProtocol.TProtocolException(message='Required field columns is unset!')
  1115. if self.total_byte_size is None:
  1116. raise TProtocol.TProtocolException(message='Required field total_byte_size is unset!')
  1117. if self.num_rows is None:
  1118. raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
  1119. return
  1120. def __repr__(self):
  1121. L = ['%s=%r' % (key, value)
  1122. for key, value in self.__dict__.iteritems()]
  1123. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  1124. def __eq__(self, other):
  1125. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  1126. def __ne__(self, other):
  1127. return not (self == other)
  1128. class FileMetaData:
  1129. """
  1130. Description for file metadata
  1131. Attributes:
  1132. - version: Version of this file *
  1133. - schema: Parquet schema for this file. This schema contains metadata for all the columns.
  1134. The schema is represented as a tree with a single root. The nodes of the tree
  1135. are flattened to a list by doing a depth-first traversal.
  1136. The column metadata contains the path in the schema for that column which can be
  1137. used to map columns to nodes in the schema.
  1138. The first element is the root *
  1139. - num_rows: Number of rows in this file *
  1140. - row_groups: Row groups in this file *
  1141. - key_value_metadata: Optional key/value metadata *
  1142. - created_by: String for application that wrote this file. This should be in the format
  1143. <Application> version <App Version> (build <App Build Hash>).
  1144. e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
  1145. """
  1146. thrift_spec = (
  1147. None, # 0
  1148. (1, TType.I32, 'version', None, None, ), # 1
  1149. (2, TType.LIST, 'schema', (TType.STRUCT,(SchemaElement, SchemaElement.thrift_spec)), None, ), # 2
  1150. (3, TType.I64, 'num_rows', None, None, ), # 3
  1151. (4, TType.LIST, 'row_groups', (TType.STRUCT,(RowGroup, RowGroup.thrift_spec)), None, ), # 4
  1152. (5, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 5
  1153. (6, TType.STRING, 'created_by', None, None, ), # 6
  1154. )
  1155. def __init__(self, version=None, schema=None, num_rows=None, row_groups=None, key_value_metadata=None, created_by=None,):
  1156. self.version = version
  1157. self.schema = schema
  1158. self.num_rows = num_rows
  1159. self.row_groups = row_groups
  1160. self.key_value_metadata = key_value_metadata
  1161. self.created_by = created_by
  1162. def read(self, iprot):
  1163. if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
  1164. fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
  1165. return
  1166. iprot.readStructBegin()
  1167. while True:
  1168. (fname, ftype, fid) = iprot.readFieldBegin()
  1169. if ftype == TType.STOP:
  1170. break
  1171. if fid == 1:
  1172. if ftype == TType.I32:
  1173. self.version = iprot.readI32();
  1174. else:
  1175. iprot.skip(ftype)
  1176. elif fid == 2:
  1177. if ftype == TType.LIST:
  1178. self.schema = []
  1179. (_etype38, _size35) = iprot.readListBegin()
  1180. for _i39 in xrange(_size35):
  1181. _elem40 = SchemaElement()
  1182. _elem40.read(iprot)
  1183. self.schema.append(_elem40)
  1184. iprot.readListEnd()
  1185. else:
  1186. iprot.skip(ftype)
  1187. elif fid == 3:
  1188. if ftype == TType.I64:
  1189. self.num_rows = iprot.readI64();
  1190. else:
  1191. iprot.skip(ftype)
  1192. elif fid == 4:
  1193. if ftype == TType.LIST:
  1194. self.row_groups = []
  1195. (_etype44, _size41) = iprot.readListBegin()
  1196. for _i45 in xrange(_size41):
  1197. _elem46 = RowGroup()
  1198. _elem46.read(iprot)
  1199. self.row_groups.append(_elem46)
  1200. iprot.readListEnd()
  1201. else:
  1202. iprot.skip(ftype)
  1203. elif fid == 5:
  1204. if ftype == TType.LIST:
  1205. self.key_value_metadata = []
  1206. (_etype50, _size47) = iprot.readListBegin()
  1207. for _i51 in xrange(_size47):
  1208. _elem52 = KeyValue()
  1209. _elem52.read(iprot)
  1210. self.key_value_metadata.append(_elem52)
  1211. iprot.readListEnd()
  1212. else:
  1213. iprot.skip(ftype)
  1214. elif fid == 6:
  1215. if ftype == TType.STRING:
  1216. self.created_by = iprot.readString();
  1217. else:
  1218. iprot.skip(ftype)
  1219. else:
  1220. iprot.skip(ftype)
  1221. iprot.readFieldEnd()
  1222. iprot.readStructEnd()
  1223. def write(self, oprot):
  1224. if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
  1225. oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
  1226. return
  1227. oprot.writeStructBegin('FileMetaData')
  1228. if self.version is not None:
  1229. oprot.writeFieldBegin('version', TType.I32, 1)
  1230. oprot.writeI32(self.version)
  1231. oprot.writeFieldEnd()
  1232. if self.schema is not None:
  1233. oprot.writeFieldBegin('schema', TType.LIST, 2)
  1234. oprot.writeListBegin(TType.STRUCT, len(self.schema))
  1235. for iter53 in self.schema:
  1236. iter53.write(oprot)
  1237. oprot.writeListEnd()
  1238. oprot.writeFieldEnd()
  1239. if self.num_rows is not None:
  1240. oprot.writeFieldBegin('num_rows', TType.I64, 3)
  1241. oprot.writeI64(self.num_rows)
  1242. oprot.writeFieldEnd()
  1243. if self.row_groups is not None:
  1244. oprot.writeFieldBegin('row_groups', TType.LIST, 4)
  1245. oprot.writeListBegin(TType.STRUCT, len(self.row_groups))
  1246. for iter54 in self.row_groups:
  1247. iter54.write(oprot)
  1248. oprot.writeListEnd()
  1249. oprot.writeFieldEnd()
  1250. if self.key_value_metadata is not None:
  1251. oprot.writeFieldBegin('key_value_metadata', TType.LIST, 5)
  1252. oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
  1253. for iter55 in self.key_value_metadata:
  1254. iter55.write(oprot)
  1255. oprot.writeListEnd()
  1256. oprot.writeFieldEnd()
  1257. if self.created_by is not None:
  1258. oprot.writeFieldBegin('created_by', TType.STRING, 6)
  1259. oprot.writeString(self.created_by)
  1260. oprot.writeFieldEnd()
  1261. oprot.writeFieldStop()
  1262. oprot.writeStructEnd()
  1263. def validate(self):
  1264. if self.version is None:
  1265. raise TProtocol.TProtocolException(message='Required field version is unset!')
  1266. if self.schema is None:
  1267. raise TProtocol.TProtocolException(message='Required field schema is unset!')
  1268. if self.num_rows is None:
  1269. raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
  1270. if self.row_groups is None:
  1271. raise TProtocol.TProtocolException(message='Required field row_groups is unset!')
  1272. return
  1273. def __repr__(self):
  1274. L = ['%s=%r' % (key, value)
  1275. for key, value in self.__dict__.iteritems()]
  1276. return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
  1277. def __eq__(self, other):
  1278. return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
  1279. def __ne__(self, other):
  1280. return not (self == other)