parquet.thrift 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing,
  13. * software distributed under the License is distributed on an
  14. * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15. * KIND, either express or implied. See the License for the
  16. * specific language governing permissions and limitations
  17. * under the License.
  18. */
  19. /**
  20. * File format description for the parquet file format
  21. */
  22. namespace cpp parquet
  23. namespace java org.apache.parquet.format
  24. /**
  25. * Types supported by Parquet. These types are intended to be used in combination
  26. * with the encodings to control the on disk storage format.
  27. * For example INT16 is not included as a type since a good encoding of INT32
  28. * would handle this.
  29. */
  30. enum Type {
  31. BOOLEAN = 0;
  32. INT32 = 1;
  33. INT64 = 2;
  34. INT96 = 3;
  35. FLOAT = 4;
  36. DOUBLE = 5;
  37. BYTE_ARRAY = 6;
  38. FIXED_LEN_BYTE_ARRAY = 7;
  39. }
  40. /**
  41. * Common types used by frameworks(e.g. hive, pig) using parquet. This helps map
  42. * between types in those frameworks to the base types in parquet. This is only
  43. * metadata and not needed to read or write the data.
  44. */
  45. enum ConvertedType {
  46. /** a BYTE_ARRAY actually contains UTF8 encoded chars */
  47. UTF8 = 0;
  48. /** a map is converted as an optional field containing a repeated key/value pair */
  49. MAP = 1;
  50. /** a key/value pair is converted into a group of two fields */
  51. MAP_KEY_VALUE = 2;
  52. /** a list is converted into an optional field containing a repeated field for its
  53. * values */
  54. LIST = 3;
  55. /** an enum is converted into a binary field */
  56. ENUM = 4;
  57. /**
  58. * A decimal value.
  59. *
  60. * This may be used to annotate binary or fixed primitive types. The
  61. * underlying byte array stores the unscaled value encoded as two's
  62. * complement using big-endian byte order (the most significant byte is the
  63. * zeroth element). The value of the decimal is the value * 10^{-scale}.
  64. *
  65. * This must be accompanied by a (maximum) precision and a scale in the
  66. * SchemaElement. The precision specifies the number of digits in the decimal
  67. * and the scale stores the location of the decimal point. For example 1.23
  68. * would have precision 3 (3 total digits) and scale 2 (the decimal point is
  69. * 2 digits over).
  70. */
  71. DECIMAL = 5;
  72. /**
  73. * A Date
  74. *
  75. * Stored as days since Unix epoch, encoded as the INT32 physical type.
  76. *
  77. */
  78. DATE = 6;
  79. /**
  80. * A time
  81. *
  82. * The total number of milliseconds since midnight. The value is stored
  83. * as an INT32 physical type.
  84. */
  85. TIME_MILLIS = 7;
  86. /**
  87. * A time.
  88. *
  89. * The total number of microseconds since midnight. The value is stored as
  90. * an INT64 physical type.
  91. */
  92. TIME_MICROS = 8;
  93. /**
  94. * A date/time combination
  95. *
  96. * Date and time recorded as milliseconds since the Unix epoch. Recorded as
  97. * a physical type of INT64.
  98. */
  99. TIMESTAMP_MILLIS = 9;
  100. /**
  101. * A date/time combination
  102. *
  103. * Date and time recorded as microseconds since the Unix epoch. The value is
  104. * stored as an INT64 physical type.
  105. */
  106. TIMESTAMP_MICROS = 10;
  107. /**
  108. * An unsigned integer value.
  109. *
  110. * The number describes the maximum number of meainful data bits in
  111. * the stored value. 8, 16 and 32 bit values are stored using the
  112. * INT32 physical type. 64 bit values are stored using the INT64
  113. * physical type.
  114. *
  115. */
  116. UINT_8 = 11;
  117. UINT_16 = 12;
  118. UINT_32 = 13;
  119. UINT_64 = 14;
  120. /**
  121. * A signed integer value.
  122. *
  123. * The number describes the maximum number of meainful data bits in
  124. * the stored value. 8, 16 and 32 bit values are stored using the
  125. * INT32 physical type. 64 bit values are stored using the INT64
  126. * physical type.
  127. *
  128. */
  129. INT_8 = 15;
  130. INT_16 = 16;
  131. INT_32 = 17;
  132. INT_64 = 18;
  133. /**
  134. * An embedded JSON document
  135. *
  136. * A JSON document embedded within a single UTF8 column.
  137. */
  138. JSON = 19;
  139. /**
  140. * An embedded BSON document
  141. *
  142. * A BSON document embedded within a single BINARY column.
  143. */
  144. BSON = 20;
  145. /**
  146. * An interval of time
  147. *
  148. * This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12
  149. * This data is composed of three separate little endian unsigned
  150. * integers. Each stores a component of a duration of time. The first
  151. * integer identifies the number of months associated with the duration,
  152. * the second identifies the number of days associated with the duration
  153. * and the third identifies the number of milliseconds associated with
  154. * the provided duration. This duration of time is independent of any
  155. * particular timezone or date.
  156. */
  157. INTERVAL = 21;
  158. }
  159. /**
  160. * Representation of Schemas
  161. */
  162. enum FieldRepetitionType {
  163. /** This field is required (can not be null) and each record has exactly 1 value. */
  164. REQUIRED = 0;
  165. /** The field is optional (can be null) and each record has 0 or 1 values. */
  166. OPTIONAL = 1;
  167. /** The field is repeated and can contain 0 or more values */
  168. REPEATED = 2;
  169. }
  170. /**
  171. * Statistics per row group and per page
  172. * All fields are optional.
  173. */
  174. struct Statistics {
  175. /** min and max value of the column, encoded in PLAIN encoding */
  176. 1: optional binary max;
  177. 2: optional binary min;
  178. /** count of null value in the column */
  179. 3: optional i64 null_count;
  180. /** count of distinct values occurring */
  181. 4: optional i64 distinct_count;
  182. }
  183. /**
  184. * Represents a element inside a schema definition.
  185. * - if it is a group (inner node) then type is undefined and num_children is defined
  186. * - if it is a primitive type (leaf) then type is defined and num_children is undefined
  187. * the nodes are listed in depth first traversal order.
  188. */
  189. struct SchemaElement {
  190. /** Data type for this field. Not set if the current element is a non-leaf node */
  191. 1: optional Type type;
  192. /** If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
  193. * Otherwise, if specified, this is the maximum bit length to store any of the values.
  194. * (e.g. a low cardinality INT col could have this set to 3). Note that this is
  195. * in the schema, and therefore fixed for the entire file.
  196. */
  197. 2: optional i32 type_length;
  198. /** repetition of the field. The root of the schema does not have a repetition_type.
  199. * All other nodes must have one */
  200. 3: optional FieldRepetitionType repetition_type;
  201. /** Name of the field in the schema */
  202. 4: required string name;
  203. /** Nested fields. Since thrift does not support nested fields,
  204. * the nesting is flattened to a single list by a depth-first traversal.
  205. * The children count is used to construct the nested relationship.
  206. * This field is not set when the element is a primitive type
  207. */
  208. 5: optional i32 num_children;
  209. /** When the schema is the result of a conversion from another model
  210. * Used to record the original type to help with cross conversion.
  211. */
  212. 6: optional ConvertedType converted_type;
  213. /** Used when this column contains decimal data.
  214. * See the DECIMAL converted type for more details.
  215. */
  216. 7: optional i32 scale
  217. 8: optional i32 precision
  218. /** When the original schema supports field ids, this will save the
  219. * original field id in the parquet schema
  220. */
  221. 9: optional i32 field_id;
  222. }
  223. /**
  224. * Encodings supported by Parquet. Not all encodings are valid for all types. These
  225. * enums are also used to specify the encoding of definition and repetition levels.
  226. * See the accompanying doc for the details of the more complicated encodings.
  227. */
  228. enum Encoding {
  229. /** Default encoding.
  230. * BOOLEAN - 1 bit per value. 0 is false; 1 is true.
  231. * INT32 - 4 bytes per value. Stored as little-endian.
  232. * INT64 - 8 bytes per value. Stored as little-endian.
  233. * FLOAT - 4 bytes per value. IEEE. Stored as little-endian.
  234. * DOUBLE - 8 bytes per value. IEEE. Stored as little-endian.
  235. * BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
  236. * FIXED_LEN_BYTE_ARRAY - Just the bytes.
  237. */
  238. PLAIN = 0;
  239. /** Group VarInt encoding for INT32/INT64.
  240. * This encoding is deprecated. It was never used
  241. */
  242. // GROUP_VAR_INT = 1;
  243. /**
  244. * Deprecated: Dictionary encoding. The values in the dictionary are encoded in the
  245. * plain type.
  246. * in a data page use RLE_DICTIONARY instead.
  247. * in a Dictionary page use PLAIN instead
  248. */
  249. PLAIN_DICTIONARY = 2;
  250. /** Group packed run length encoding. Usable for definition/reptition levels
  251. * encoding and Booleans (on one bit: 0 is false; 1 is true.)
  252. */
  253. RLE = 3;
  254. /** Bit packed encoding. This can only be used if the data has a known max
  255. * width. Usable for definition/repetition levels encoding.
  256. */
  257. BIT_PACKED = 4;
  258. /** Delta encoding for integers. This can be used for int columns and works best
  259. * on sorted data
  260. */
  261. DELTA_BINARY_PACKED = 5;
  262. /** Encoding for byte arrays to separate the length values and the data. The lengths
  263. * are encoded using DELTA_BINARY_PACKED
  264. */
  265. DELTA_LENGTH_BYTE_ARRAY = 6;
  266. /** Incremental-encoded byte array. Prefix lengths are encoded using DELTA_BINARY_PACKED.
  267. * Suffixes are stored as delta length byte arrays.
  268. */
  269. DELTA_BYTE_ARRAY = 7;
  270. /** Dictionary encoding: the ids are encoded using the RLE encoding
  271. */
  272. RLE_DICTIONARY = 8;
  273. }
  274. /**
  275. * Supported compression algorithms.
  276. */
  277. enum CompressionCodec {
  278. UNCOMPRESSED = 0;
  279. SNAPPY = 1;
  280. GZIP = 2;
  281. LZO = 3;
  282. }
  283. enum PageType {
  284. DATA_PAGE = 0;
  285. INDEX_PAGE = 1;
  286. DICTIONARY_PAGE = 2;
  287. DATA_PAGE_V2 = 3;
  288. }
  289. /** Data page header */
  290. struct DataPageHeader {
  291. /** Number of values, including NULLs, in this data page. **/
  292. 1: required i32 num_values
  293. /** Encoding used for this data page **/
  294. 2: required Encoding encoding
  295. /** Encoding used for definition levels **/
  296. 3: required Encoding definition_level_encoding;
  297. /** Encoding used for repetition levels **/
  298. 4: required Encoding repetition_level_encoding;
  299. /** Optional statistics for the data in this page**/
  300. 5: optional Statistics statistics;
  301. }
  302. struct IndexPageHeader {
  303. /** TODO: **/
  304. }
  305. struct DictionaryPageHeader {
  306. /** Number of values in the dictionary **/
  307. 1: required i32 num_values;
  308. /** Encoding using this dictionary page **/
  309. 2: required Encoding encoding
  310. /** If true, the entries in the dictionary are sorted in ascending order **/
  311. 3: optional bool is_sorted;
  312. }
  313. /**
  314. * New page format alowing reading levels without decompressing the data
  315. * Repetition and definition levels are uncompressed
  316. * The remaining section containing the data is compressed if is_compressed is true
  317. **/
  318. struct DataPageHeaderV2 {
  319. /** Number of values, including NULLs, in this data page. **/
  320. 1: required i32 num_values
  321. /** Number of NULL values, in this data page.
  322. Number of non-null = num_values - num_nulls which is also the number of values in the data section **/
  323. 2: required i32 num_nulls
  324. /** Number of rows in this data page. which means pages change on record boundaries (r = 0) **/
  325. 3: required i32 num_rows
  326. /** Encoding used for data in this page **/
  327. 4: required Encoding encoding
  328. // repetition levels and definition levels are always using RLE (without size in it)
  329. /** length of the repetition levels */
  330. 5: required i32 definition_levels_byte_length;
  331. /** length of the definition levels */
  332. 6: required i32 repetition_levels_byte_length;
  333. /** whether the values are compressed.
  334. Which means the section of the page between
  335. definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included)
  336. is compressed with the compression_codec.
  337. If missing it is considered compressed */
  338. 7: optional bool is_compressed = 1;
  339. /** optional statistics for this column chunk */
  340. 8: optional Statistics statistics;
  341. }
  342. struct PageHeader {
  343. /** the type of the page: indicates which of the *_header fields is set **/
  344. 1: required PageType type
  345. /** Uncompressed page size in bytes (not including this header) **/
  346. 2: required i32 uncompressed_page_size
  347. /** Compressed page size in bytes (not including this header) **/
  348. 3: required i32 compressed_page_size
  349. /** 32bit crc for the data below. This allows for disabling checksumming in HDFS
  350. * if only a few pages needs to be read
  351. **/
  352. 4: optional i32 crc
  353. // Headers for page specific data. One only will be set.
  354. 5: optional DataPageHeader data_page_header;
  355. 6: optional IndexPageHeader index_page_header;
  356. 7: optional DictionaryPageHeader dictionary_page_header;
  357. 8: optional DataPageHeaderV2 data_page_header_v2;
  358. }
  359. /**
  360. * Wrapper struct to store key values
  361. */
  362. struct KeyValue {
  363. 1: required string key
  364. 2: optional string value
  365. }
  366. /**
  367. * Wrapper struct to specify sort order
  368. */
  369. struct SortingColumn {
  370. /** The column index (in this row group) **/
  371. 1: required i32 column_idx
  372. /** If true, indicates this column is sorted in descending order. **/
  373. 2: required bool descending
  374. /** If true, nulls will come before non-null values, otherwise,
  375. * nulls go at the end. */
  376. 3: required bool nulls_first
  377. }
  378. /**
  379. * statistics of a given page type and encoding
  380. */
  381. struct PageEncodingStats {
  382. /** the page type (data/dic/...) **/
  383. 1: required PageType page_type;
  384. /** encoding of the page **/
  385. 2: required Encoding encoding;
  386. /** number of pages of this type with this encoding **/
  387. 3: required i32 count;
  388. }
  389. /**
  390. * Description for column metadata
  391. */
  392. struct ColumnMetaData {
  393. /** Type of this column **/
  394. 1: required Type type
  395. /** Set of all encodings used for this column. The purpose is to validate
  396. * whether we can decode those pages. **/
  397. 2: required list<Encoding> encodings
  398. /** Path in schema **/
  399. 3: required list<string> path_in_schema
  400. /** Compression codec **/
  401. 4: required CompressionCodec codec
  402. /** Number of values in this column **/
  403. 5: required i64 num_values
  404. /** total byte size of all uncompressed pages in this column chunk (including the headers) **/
  405. 6: required i64 total_uncompressed_size
  406. /** total byte size of all compressed pages in this column chunk (including the headers) **/
  407. 7: required i64 total_compressed_size
  408. /** Optional key/value metadata **/
  409. 8: optional list<KeyValue> key_value_metadata
  410. /** Byte offset from beginning of file to first data page **/
  411. 9: required i64 data_page_offset
  412. /** Byte offset from beginning of file to root index page **/
  413. 10: optional i64 index_page_offset
  414. /** Byte offset from the beginning of file to first (only) dictionary page **/
  415. 11: optional i64 dictionary_page_offset
  416. /** optional statistics for this column chunk */
  417. 12: optional Statistics statistics;
  418. /** Set of all encodings used for pages in this column chunk.
  419. * This information can be used to determine if all data pages are
  420. * dictionary encoded for example **/
  421. 13: optional list<PageEncodingStats> encoding_stats;
  422. }
  423. struct ColumnChunk {
  424. /** File where column data is stored. If not set, assumed to be same file as
  425. * metadata. This path is relative to the current file.
  426. **/
  427. 1: optional string file_path
  428. /** Byte offset in file_path to the ColumnMetaData **/
  429. 2: required i64 file_offset
  430. /** Column metadata for this chunk. This is the same content as what is at
  431. * file_path/file_offset. Having it here has it replicated in the file
  432. * metadata.
  433. **/
  434. 3: optional ColumnMetaData meta_data
  435. }
  436. struct RowGroup {
  437. /** Metadata for each column chunk in this row group.
  438. * This list must have the same order as the SchemaElement list in FileMetaData.
  439. **/
  440. 1: required list<ColumnChunk> columns
  441. /** Total byte size of all the uncompressed column data in this row group **/
  442. 2: required i64 total_byte_size
  443. /** Number of rows in this row group **/
  444. 3: required i64 num_rows
  445. /** If set, specifies a sort ordering of the rows in this RowGroup.
  446. * The sorting columns can be a subset of all the columns.
  447. */
  448. 4: optional list<SortingColumn> sorting_columns
  449. }
  450. /**
  451. * Description for file metadata
  452. */
  453. struct FileMetaData {
  454. /** Version of this file **/
  455. 1: required i32 version
  456. /** Parquet schema for this file. This schema contains metadata for all the columns.
  457. * The schema is represented as a tree with a single root. The nodes of the tree
  458. * are flattened to a list by doing a depth-first traversal.
  459. * The column metadata contains the path in the schema for that column which can be
  460. * used to map columns to nodes in the schema.
  461. * The first element is the root **/
  462. 2: required list<SchemaElement> schema;
  463. /** Number of rows in this file **/
  464. 3: required i64 num_rows
  465. /** Row groups in this file **/
  466. 4: required list<RowGroup> row_groups
  467. /** Optional key/value metadata **/
  468. 5: optional list<KeyValue> key_value_metadata
  469. /** String for application that wrote this file. This should be in the format
  470. * <Application> version <App Version> (build <App Build Hash>).
  471. * e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
  472. **/
  473. 6: optional string created_by
  474. }