Browse Source

Bootstrapping of parquet support.

Initial support to read file footers.
Joe Crobak 12 years ago
commit
d021238fbb
6 changed files with 1528 additions and 0 deletions
  1. 1 0
      .gitignore
  2. 45 0
      parquet/__init__.py
  3. 11 0
      parquet/constants.py
  4. 1424 0
      parquet/ttypes.py
  5. 12 0
      setup.py
  6. 35 0
      test/test_read_support.py

+ 1 - 0
.gitignore

@@ -0,0 +1 @@
+*.pyc

+ 45 - 0
parquet/__init__.py

@@ -0,0 +1,45 @@
+import struct
+import thrift
+import logging
+from ttypes import FileMetaData
+from thrift.protocol import TCompactProtocol
+from thrift.transport import TTransport
+
+logger = logging.getLogger("parquet")
+
+def _check_header_magic_bytes(fo):
+    """Returns true if the file-like obj has the PAR1 magic bytes at the header"""
+    fo.seek(0, 0)
+    magic = fo.read(4)
+    return magic == 'PAR1'
+
+def _check_footer_magic_bytes(fo):
+    """Returns true if the file-like obj has the PAR1 magic bytes at the footer"""
+    fo.seek(-4, 2)  # seek to four bytes from the end of the file
+    magic = fo.read(4)
+    return magic == 'PAR1'
+
+
+def _get_footer_size(fo):
+    """Readers the footer size in bytes, which is serialized as little endian"""
+    fo.seek(-8, 2)
+    tup = struct.unpack("<i", fo.read(4))
+    return tup[0]
+
+def _read_footer(fo):
+    """Reads the footer from the given file object, returning a FileMetaData object. This method
+    assumes that the fo references a valid parquet file"""
+    footer_size = _get_footer_size(fo)
+    logger.debug("Footer size in bytes: %s", footer_size)
+    fo.seek(-(8+footer_size), 2)  # seek to beginning of footer
+    tin = TTransport.TFileObjectTransport(fo)
+    pin = TCompactProtocol.TCompactProtocol(tin)
+    fmd = FileMetaData()
+    fmd.read(pin)
+    return fmd
+
+def read_footer(filename):
+    with open(filename, 'rb') as fo:
+        if not _check_header_magic_bytes(fo) or not _check_footer_magic_bytes(fo):
+            raise ParquetFormatException("%s is not a valid parquet file (missing magic bytes".format(filename))
+        return _read_footer(fo)

+ 11 - 0
parquet/constants.py

@@ -0,0 +1,11 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
+from ttypes import *
+

+ 1424 - 0
parquet/ttypes.py

@@ -0,0 +1,1424 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
+
+from thrift.transport import TTransport
+from thrift.protocol import TBinaryProtocol, TProtocol
+try:
+  from thrift.protocol import fastbinary
+except:
+  fastbinary = None
+
+
+class Type:
+  """
+  Types supported by Parquet.  These types are intended to be used in combination
+  with the encodings to control the on disk storage format.
+  For example INT16 is not included as a type since a good encoding of INT32
+  would handle this.
+  """
+  BOOLEAN = 0
+  INT32 = 1
+  INT64 = 2
+  INT96 = 3
+  FLOAT = 4
+  DOUBLE = 5
+  BYTE_ARRAY = 6
+  FIXED_LEN_BYTE_ARRAY = 7
+
+  _VALUES_TO_NAMES = {
+    0: "BOOLEAN",
+    1: "INT32",
+    2: "INT64",
+    3: "INT96",
+    4: "FLOAT",
+    5: "DOUBLE",
+    6: "BYTE_ARRAY",
+    7: "FIXED_LEN_BYTE_ARRAY",
+  }
+
+  _NAMES_TO_VALUES = {
+    "BOOLEAN": 0,
+    "INT32": 1,
+    "INT64": 2,
+    "INT96": 3,
+    "FLOAT": 4,
+    "DOUBLE": 5,
+    "BYTE_ARRAY": 6,
+    "FIXED_LEN_BYTE_ARRAY": 7,
+  }
+
+class ConvertedType:
+  """
+  Common types used by frameworks(e.g. hive, pig) using parquet.  This helps map
+  between types in those frameworks to the base types in parquet.  This is only
+  metadata and not needed to read or write the data.
+  """
+  UTF8 = 0
+  MAP = 1
+  MAP_KEY_VALUE = 2
+  LIST = 3
+
+  _VALUES_TO_NAMES = {
+    0: "UTF8",
+    1: "MAP",
+    2: "MAP_KEY_VALUE",
+    3: "LIST",
+  }
+
+  _NAMES_TO_VALUES = {
+    "UTF8": 0,
+    "MAP": 1,
+    "MAP_KEY_VALUE": 2,
+    "LIST": 3,
+  }
+
+class FieldRepetitionType:
+  """
+  Representation of Schemas
+  """
+  REQUIRED = 0
+  OPTIONAL = 1
+  REPEATED = 2
+
+  _VALUES_TO_NAMES = {
+    0: "REQUIRED",
+    1: "OPTIONAL",
+    2: "REPEATED",
+  }
+
+  _NAMES_TO_VALUES = {
+    "REQUIRED": 0,
+    "OPTIONAL": 1,
+    "REPEATED": 2,
+  }
+
+class Encoding:
+  """
+  Encodings supported by Parquet.  Not all encodings are valid for all types.  These
+  enums are also used to specify the encoding of definition and repetition levels.
+  See the accompanying doc for the details of the more complicated encodings.
+  """
+  PLAIN = 0
+  GROUP_VAR_INT = 1
+  PLAIN_DICTIONARY = 2
+  RLE = 3
+  BIT_PACKED = 4
+
+  _VALUES_TO_NAMES = {
+    0: "PLAIN",
+    1: "GROUP_VAR_INT",
+    2: "PLAIN_DICTIONARY",
+    3: "RLE",
+    4: "BIT_PACKED",
+  }
+
+  _NAMES_TO_VALUES = {
+    "PLAIN": 0,
+    "GROUP_VAR_INT": 1,
+    "PLAIN_DICTIONARY": 2,
+    "RLE": 3,
+    "BIT_PACKED": 4,
+  }
+
+class CompressionCodec:
+  """
+  Supported compression algorithms.
+  """
+  UNCOMPRESSED = 0
+  SNAPPY = 1
+  GZIP = 2
+  LZO = 3
+
+  _VALUES_TO_NAMES = {
+    0: "UNCOMPRESSED",
+    1: "SNAPPY",
+    2: "GZIP",
+    3: "LZO",
+  }
+
+  _NAMES_TO_VALUES = {
+    "UNCOMPRESSED": 0,
+    "SNAPPY": 1,
+    "GZIP": 2,
+    "LZO": 3,
+  }
+
+class PageType:
+  DATA_PAGE = 0
+  INDEX_PAGE = 1
+  DICTIONARY_PAGE = 2
+
+  _VALUES_TO_NAMES = {
+    0: "DATA_PAGE",
+    1: "INDEX_PAGE",
+    2: "DICTIONARY_PAGE",
+  }
+
+  _NAMES_TO_VALUES = {
+    "DATA_PAGE": 0,
+    "INDEX_PAGE": 1,
+    "DICTIONARY_PAGE": 2,
+  }
+
+
+class SchemaElement:
+  """
+  Represents a element inside a schema definition.
+   - if it is a group (inner node) then type is undefined and num_children is defined
+   - if it is a primitive type (leaf) then type is defined and num_children is undefined
+  the nodes are listed in depth first traversal order.
+
+  Attributes:
+   - type: Data type for this field. Not set if the current element is a non-leaf node
+   - type_length: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
+  Otherwise, if specified, this is the maximum bit length to store any of the values.
+  (e.g. a low cardinality INT col could have this set to 3).  Note that this is
+  in the schema, and therefore fixed for the entire file.
+   - repetition_type: repetition of the field. The root of the schema does not have a repetition_type.
+  All other nodes must have one
+   - name: Name of the field in the schema
+   - num_children: Nested fields.  Since thrift does not support nested fields,
+  the nesting is flattened to a single list by a depth-first traversal.
+  The children count is used to construct the nested relationship.
+  This field is not set when the element is a primitive type
+   - converted_type: When the schema is the result of a conversion from another model
+  Used to record the original type to help with cross conversion.
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'type', None, None, ), # 1
+    (2, TType.I32, 'type_length', None, None, ), # 2
+    (3, TType.I32, 'repetition_type', None, None, ), # 3
+    (4, TType.STRING, 'name', None, None, ), # 4
+    (5, TType.I32, 'num_children', None, None, ), # 5
+    (6, TType.I32, 'converted_type', None, None, ), # 6
+  )
+
+  def __init__(self, type=None, type_length=None, repetition_type=None, name=None, num_children=None, converted_type=None,):
+    self.type = type
+    self.type_length = type_length
+    self.repetition_type = repetition_type
+    self.name = name
+    self.num_children = num_children
+    self.converted_type = converted_type
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.type = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.I32:
+          self.type_length = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.I32:
+          self.repetition_type = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.STRING:
+          self.name = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 5:
+        if ftype == TType.I32:
+          self.num_children = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 6:
+        if ftype == TType.I32:
+          self.converted_type = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('SchemaElement')
+    if self.type is not None:
+      oprot.writeFieldBegin('type', TType.I32, 1)
+      oprot.writeI32(self.type)
+      oprot.writeFieldEnd()
+    if self.type_length is not None:
+      oprot.writeFieldBegin('type_length', TType.I32, 2)
+      oprot.writeI32(self.type_length)
+      oprot.writeFieldEnd()
+    if self.repetition_type is not None:
+      oprot.writeFieldBegin('repetition_type', TType.I32, 3)
+      oprot.writeI32(self.repetition_type)
+      oprot.writeFieldEnd()
+    if self.name is not None:
+      oprot.writeFieldBegin('name', TType.STRING, 4)
+      oprot.writeString(self.name)
+      oprot.writeFieldEnd()
+    if self.num_children is not None:
+      oprot.writeFieldBegin('num_children', TType.I32, 5)
+      oprot.writeI32(self.num_children)
+      oprot.writeFieldEnd()
+    if self.converted_type is not None:
+      oprot.writeFieldBegin('converted_type', TType.I32, 6)
+      oprot.writeI32(self.converted_type)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.name is None:
+      raise TProtocol.TProtocolException(message='Required field name is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class DataPageHeader:
+  """
+  Data page header
+
+  Attributes:
+   - num_values: Number of values, including NULLs, in this data page. *
+   - encoding: Encoding used for this data page *
+   - definition_level_encoding: Encoding used for definition levels *
+   - repetition_level_encoding: Encoding used for repetition levels *
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'num_values', None, None, ), # 1
+    (2, TType.I32, 'encoding', None, None, ), # 2
+    (3, TType.I32, 'definition_level_encoding', None, None, ), # 3
+    (4, TType.I32, 'repetition_level_encoding', None, None, ), # 4
+  )
+
+  def __init__(self, num_values=None, encoding=None, definition_level_encoding=None, repetition_level_encoding=None,):
+    self.num_values = num_values
+    self.encoding = encoding
+    self.definition_level_encoding = definition_level_encoding
+    self.repetition_level_encoding = repetition_level_encoding
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.num_values = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.I32:
+          self.encoding = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.I32:
+          self.definition_level_encoding = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.I32:
+          self.repetition_level_encoding = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('DataPageHeader')
+    if self.num_values is not None:
+      oprot.writeFieldBegin('num_values', TType.I32, 1)
+      oprot.writeI32(self.num_values)
+      oprot.writeFieldEnd()
+    if self.encoding is not None:
+      oprot.writeFieldBegin('encoding', TType.I32, 2)
+      oprot.writeI32(self.encoding)
+      oprot.writeFieldEnd()
+    if self.definition_level_encoding is not None:
+      oprot.writeFieldBegin('definition_level_encoding', TType.I32, 3)
+      oprot.writeI32(self.definition_level_encoding)
+      oprot.writeFieldEnd()
+    if self.repetition_level_encoding is not None:
+      oprot.writeFieldBegin('repetition_level_encoding', TType.I32, 4)
+      oprot.writeI32(self.repetition_level_encoding)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.num_values is None:
+      raise TProtocol.TProtocolException(message='Required field num_values is unset!')
+    if self.encoding is None:
+      raise TProtocol.TProtocolException(message='Required field encoding is unset!')
+    if self.definition_level_encoding is None:
+      raise TProtocol.TProtocolException(message='Required field definition_level_encoding is unset!')
+    if self.repetition_level_encoding is None:
+      raise TProtocol.TProtocolException(message='Required field repetition_level_encoding is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class IndexPageHeader:
+
+  thrift_spec = (
+  )
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('IndexPageHeader')
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class DictionaryPageHeader:
+  """
+  TODO: *
+
+  Attributes:
+   - num_values: Number of values in the dictionary *
+   - encoding: Encoding using this dictionary page *
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'num_values', None, None, ), # 1
+    (2, TType.I32, 'encoding', None, None, ), # 2
+  )
+
+  def __init__(self, num_values=None, encoding=None,):
+    self.num_values = num_values
+    self.encoding = encoding
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.num_values = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.I32:
+          self.encoding = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('DictionaryPageHeader')
+    if self.num_values is not None:
+      oprot.writeFieldBegin('num_values', TType.I32, 1)
+      oprot.writeI32(self.num_values)
+      oprot.writeFieldEnd()
+    if self.encoding is not None:
+      oprot.writeFieldBegin('encoding', TType.I32, 2)
+      oprot.writeI32(self.encoding)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.num_values is None:
+      raise TProtocol.TProtocolException(message='Required field num_values is unset!')
+    if self.encoding is None:
+      raise TProtocol.TProtocolException(message='Required field encoding is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class PageHeader:
+  """
+  Attributes:
+   - type: the type of the page: indicates which of the *_header fields is set *
+   - uncompressed_page_size: Uncompressed page size in bytes (not including this header) *
+   - compressed_page_size: Compressed page size in bytes (not including this header) *
+   - crc: 32bit crc for the data below. This allows for disabling checksumming in HDFS
+  if only a few pages needs to be read
+
+   - data_page_header
+   - index_page_header
+   - dictionary_page_header
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'type', None, None, ), # 1
+    (2, TType.I32, 'uncompressed_page_size', None, None, ), # 2
+    (3, TType.I32, 'compressed_page_size', None, None, ), # 3
+    (4, TType.I32, 'crc', None, None, ), # 4
+    (5, TType.STRUCT, 'data_page_header', (DataPageHeader, DataPageHeader.thrift_spec), None, ), # 5
+    (6, TType.STRUCT, 'index_page_header', (IndexPageHeader, IndexPageHeader.thrift_spec), None, ), # 6
+    (7, TType.STRUCT, 'dictionary_page_header', (DictionaryPageHeader, DictionaryPageHeader.thrift_spec), None, ), # 7
+  )
+
+  def __init__(self, type=None, uncompressed_page_size=None, compressed_page_size=None, crc=None, data_page_header=None, index_page_header=None, dictionary_page_header=None,):
+    self.type = type
+    self.uncompressed_page_size = uncompressed_page_size
+    self.compressed_page_size = compressed_page_size
+    self.crc = crc
+    self.data_page_header = data_page_header
+    self.index_page_header = index_page_header
+    self.dictionary_page_header = dictionary_page_header
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.type = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.I32:
+          self.uncompressed_page_size = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.I32:
+          self.compressed_page_size = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.I32:
+          self.crc = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 5:
+        if ftype == TType.STRUCT:
+          self.data_page_header = DataPageHeader()
+          self.data_page_header.read(iprot)
+        else:
+          iprot.skip(ftype)
+      elif fid == 6:
+        if ftype == TType.STRUCT:
+          self.index_page_header = IndexPageHeader()
+          self.index_page_header.read(iprot)
+        else:
+          iprot.skip(ftype)
+      elif fid == 7:
+        if ftype == TType.STRUCT:
+          self.dictionary_page_header = DictionaryPageHeader()
+          self.dictionary_page_header.read(iprot)
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('PageHeader')
+    if self.type is not None:
+      oprot.writeFieldBegin('type', TType.I32, 1)
+      oprot.writeI32(self.type)
+      oprot.writeFieldEnd()
+    if self.uncompressed_page_size is not None:
+      oprot.writeFieldBegin('uncompressed_page_size', TType.I32, 2)
+      oprot.writeI32(self.uncompressed_page_size)
+      oprot.writeFieldEnd()
+    if self.compressed_page_size is not None:
+      oprot.writeFieldBegin('compressed_page_size', TType.I32, 3)
+      oprot.writeI32(self.compressed_page_size)
+      oprot.writeFieldEnd()
+    if self.crc is not None:
+      oprot.writeFieldBegin('crc', TType.I32, 4)
+      oprot.writeI32(self.crc)
+      oprot.writeFieldEnd()
+    if self.data_page_header is not None:
+      oprot.writeFieldBegin('data_page_header', TType.STRUCT, 5)
+      self.data_page_header.write(oprot)
+      oprot.writeFieldEnd()
+    if self.index_page_header is not None:
+      oprot.writeFieldBegin('index_page_header', TType.STRUCT, 6)
+      self.index_page_header.write(oprot)
+      oprot.writeFieldEnd()
+    if self.dictionary_page_header is not None:
+      oprot.writeFieldBegin('dictionary_page_header', TType.STRUCT, 7)
+      self.dictionary_page_header.write(oprot)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.type is None:
+      raise TProtocol.TProtocolException(message='Required field type is unset!')
+    if self.uncompressed_page_size is None:
+      raise TProtocol.TProtocolException(message='Required field uncompressed_page_size is unset!')
+    if self.compressed_page_size is None:
+      raise TProtocol.TProtocolException(message='Required field compressed_page_size is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class KeyValue:
+  """
+  Wrapper struct to store key values
+
+  Attributes:
+   - key
+   - value
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.STRING, 'key', None, None, ), # 1
+    (2, TType.STRING, 'value', None, None, ), # 2
+  )
+
+  def __init__(self, key=None, value=None,):
+    self.key = key
+    self.value = value
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.STRING:
+          self.key = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRING:
+          self.value = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('KeyValue')
+    if self.key is not None:
+      oprot.writeFieldBegin('key', TType.STRING, 1)
+      oprot.writeString(self.key)
+      oprot.writeFieldEnd()
+    if self.value is not None:
+      oprot.writeFieldBegin('value', TType.STRING, 2)
+      oprot.writeString(self.value)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.key is None:
+      raise TProtocol.TProtocolException(message='Required field key is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class SortingColumn:
+  """
+  Wrapper struct to specify sort order
+
+  Attributes:
+   - nulls_first: The column index (in this row group)
+  1: required i32 column_idx
+
+  /** If true, indicates this column is sorted in descending order.
+  2: required bool descending
+
+  /** If true, nulls will come before non-null values, otherwise,
+   * nulls go at the end.
+  """
+
+  thrift_spec = (
+    None, # 0
+    None, # 1
+    None, # 2
+    (3, TType.BOOL, 'nulls_first', None, None, ), # 3
+  )
+
+  def __init__(self, nulls_first=None,):
+    self.nulls_first = nulls_first
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 3:
+        if ftype == TType.BOOL:
+          self.nulls_first = iprot.readBool();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('SortingColumn')
+    if self.nulls_first is not None:
+      oprot.writeFieldBegin('nulls_first', TType.BOOL, 3)
+      oprot.writeBool(self.nulls_first)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.nulls_first is None:
+      raise TProtocol.TProtocolException(message='Required field nulls_first is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class ColumnMetaData:
+  """
+  Description for column metadata
+
+  Attributes:
+   - type: Type of this column *
+   - encodings: Set of all encodings used for this column. The purpose is to validate
+  whether we can decode those pages. *
+   - path_in_schema: Path in schema *
+   - codec: Compression codec *
+   - num_values: Number of values in this column *
+   - total_uncompressed_size: total byte size of all uncompressed pages in this column chunk (including the headers) *
+   - total_compressed_size: total byte size of all compressed pages in this column chunk (including the headers) *
+   - key_value_metadata: Optional key/value metadata *
+   - data_page_offset: Byte offset from beginning of file to first data page *
+   - index_page_offset: Byte offset from beginning of file to root index page *
+   - dictionary_page_offset: Byte offset from the beginning of file to first (only) dictionary page *
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'type', None, None, ), # 1
+    (2, TType.LIST, 'encodings', (TType.I32,None), None, ), # 2
+    (3, TType.LIST, 'path_in_schema', (TType.STRING,None), None, ), # 3
+    (4, TType.I32, 'codec', None, None, ), # 4
+    (5, TType.I64, 'num_values', None, None, ), # 5
+    (6, TType.I64, 'total_uncompressed_size', None, None, ), # 6
+    (7, TType.I64, 'total_compressed_size', None, None, ), # 7
+    (8, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 8
+    (9, TType.I64, 'data_page_offset', None, None, ), # 9
+    (10, TType.I64, 'index_page_offset', None, None, ), # 10
+    (11, TType.I64, 'dictionary_page_offset', None, None, ), # 11
+  )
+
+  def __init__(self, type=None, encodings=None, path_in_schema=None, codec=None, num_values=None, total_uncompressed_size=None, total_compressed_size=None, key_value_metadata=None, data_page_offset=None, index_page_offset=None, dictionary_page_offset=None,):
+    self.type = type
+    self.encodings = encodings
+    self.path_in_schema = path_in_schema
+    self.codec = codec
+    self.num_values = num_values
+    self.total_uncompressed_size = total_uncompressed_size
+    self.total_compressed_size = total_compressed_size
+    self.key_value_metadata = key_value_metadata
+    self.data_page_offset = data_page_offset
+    self.index_page_offset = index_page_offset
+    self.dictionary_page_offset = dictionary_page_offset
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.type = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.LIST:
+          self.encodings = []
+          (_etype3, _size0) = iprot.readListBegin()
+          for _i4 in xrange(_size0):
+            _elem5 = iprot.readI32();
+            self.encodings.append(_elem5)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.LIST:
+          self.path_in_schema = []
+          (_etype9, _size6) = iprot.readListBegin()
+          for _i10 in xrange(_size6):
+            _elem11 = iprot.readString();
+            self.path_in_schema.append(_elem11)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.I32:
+          self.codec = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 5:
+        if ftype == TType.I64:
+          self.num_values = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 6:
+        if ftype == TType.I64:
+          self.total_uncompressed_size = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 7:
+        if ftype == TType.I64:
+          self.total_compressed_size = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 8:
+        if ftype == TType.LIST:
+          self.key_value_metadata = []
+          (_etype15, _size12) = iprot.readListBegin()
+          for _i16 in xrange(_size12):
+            _elem17 = KeyValue()
+            _elem17.read(iprot)
+            self.key_value_metadata.append(_elem17)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 9:
+        if ftype == TType.I64:
+          self.data_page_offset = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 10:
+        if ftype == TType.I64:
+          self.index_page_offset = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 11:
+        if ftype == TType.I64:
+          self.dictionary_page_offset = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('ColumnMetaData')
+    if self.type is not None:
+      oprot.writeFieldBegin('type', TType.I32, 1)
+      oprot.writeI32(self.type)
+      oprot.writeFieldEnd()
+    if self.encodings is not None:
+      oprot.writeFieldBegin('encodings', TType.LIST, 2)
+      oprot.writeListBegin(TType.I32, len(self.encodings))
+      for iter18 in self.encodings:
+        oprot.writeI32(iter18)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.path_in_schema is not None:
+      oprot.writeFieldBegin('path_in_schema', TType.LIST, 3)
+      oprot.writeListBegin(TType.STRING, len(self.path_in_schema))
+      for iter19 in self.path_in_schema:
+        oprot.writeString(iter19)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.codec is not None:
+      oprot.writeFieldBegin('codec', TType.I32, 4)
+      oprot.writeI32(self.codec)
+      oprot.writeFieldEnd()
+    if self.num_values is not None:
+      oprot.writeFieldBegin('num_values', TType.I64, 5)
+      oprot.writeI64(self.num_values)
+      oprot.writeFieldEnd()
+    if self.total_uncompressed_size is not None:
+      oprot.writeFieldBegin('total_uncompressed_size', TType.I64, 6)
+      oprot.writeI64(self.total_uncompressed_size)
+      oprot.writeFieldEnd()
+    if self.total_compressed_size is not None:
+      oprot.writeFieldBegin('total_compressed_size', TType.I64, 7)
+      oprot.writeI64(self.total_compressed_size)
+      oprot.writeFieldEnd()
+    if self.key_value_metadata is not None:
+      oprot.writeFieldBegin('key_value_metadata', TType.LIST, 8)
+      oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
+      for iter20 in self.key_value_metadata:
+        iter20.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.data_page_offset is not None:
+      oprot.writeFieldBegin('data_page_offset', TType.I64, 9)
+      oprot.writeI64(self.data_page_offset)
+      oprot.writeFieldEnd()
+    if self.index_page_offset is not None:
+      oprot.writeFieldBegin('index_page_offset', TType.I64, 10)
+      oprot.writeI64(self.index_page_offset)
+      oprot.writeFieldEnd()
+    if self.dictionary_page_offset is not None:
+      oprot.writeFieldBegin('dictionary_page_offset', TType.I64, 11)
+      oprot.writeI64(self.dictionary_page_offset)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.type is None:
+      raise TProtocol.TProtocolException(message='Required field type is unset!')
+    if self.encodings is None:
+      raise TProtocol.TProtocolException(message='Required field encodings is unset!')
+    if self.path_in_schema is None:
+      raise TProtocol.TProtocolException(message='Required field path_in_schema is unset!')
+    if self.codec is None:
+      raise TProtocol.TProtocolException(message='Required field codec is unset!')
+    if self.num_values is None:
+      raise TProtocol.TProtocolException(message='Required field num_values is unset!')
+    if self.total_uncompressed_size is None:
+      raise TProtocol.TProtocolException(message='Required field total_uncompressed_size is unset!')
+    if self.total_compressed_size is None:
+      raise TProtocol.TProtocolException(message='Required field total_compressed_size is unset!')
+    if self.data_page_offset is None:
+      raise TProtocol.TProtocolException(message='Required field data_page_offset is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class ColumnChunk:
+  """
+  Attributes:
+   - file_path: File where column data is stored.  If not set, assumed to be same file as
+  metadata.  This path is relative to the current file.
+
+   - file_offset: Byte offset in file_path to the ColumnMetaData *
+   - meta_data: Column metadata for this chunk. This is the same content as what is at
+  file_path/file_offset.  Having it here has it replicated in the file
+  metadata.
+
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.STRING, 'file_path', None, None, ), # 1
+    (2, TType.I64, 'file_offset', None, None, ), # 2
+    (3, TType.STRUCT, 'meta_data', (ColumnMetaData, ColumnMetaData.thrift_spec), None, ), # 3
+  )
+
+  def __init__(self, file_path=None, file_offset=None, meta_data=None,):
+    self.file_path = file_path
+    self.file_offset = file_offset
+    self.meta_data = meta_data
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.STRING:
+          self.file_path = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.I64:
+          self.file_offset = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.STRUCT:
+          self.meta_data = ColumnMetaData()
+          self.meta_data.read(iprot)
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('ColumnChunk')
+    if self.file_path is not None:
+      oprot.writeFieldBegin('file_path', TType.STRING, 1)
+      oprot.writeString(self.file_path)
+      oprot.writeFieldEnd()
+    if self.file_offset is not None:
+      oprot.writeFieldBegin('file_offset', TType.I64, 2)
+      oprot.writeI64(self.file_offset)
+      oprot.writeFieldEnd()
+    if self.meta_data is not None:
+      oprot.writeFieldBegin('meta_data', TType.STRUCT, 3)
+      self.meta_data.write(oprot)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.file_offset is None:
+      raise TProtocol.TProtocolException(message='Required field file_offset is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class RowGroup:
+  """
+  Attributes:
+   - columns
+   - total_byte_size: Total byte size of all the uncompressed column data in this row group *
+   - num_rows: Number of rows in this row group *
+   - sorting_columns: If set, specifies a sort ordering of the rows in this RowGroup.
+  The sorting columns can be a subset of all the columns.
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.LIST, 'columns', (TType.STRUCT,(ColumnChunk, ColumnChunk.thrift_spec)), None, ), # 1
+    (2, TType.I64, 'total_byte_size', None, None, ), # 2
+    (3, TType.I64, 'num_rows', None, None, ), # 3
+    (4, TType.LIST, 'sorting_columns', (TType.STRUCT,(SortingColumn, SortingColumn.thrift_spec)), None, ), # 4
+  )
+
+  def __init__(self, columns=None, total_byte_size=None, num_rows=None, sorting_columns=None,):
+    self.columns = columns
+    self.total_byte_size = total_byte_size
+    self.num_rows = num_rows
+    self.sorting_columns = sorting_columns
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.LIST:
+          self.columns = []
+          (_etype24, _size21) = iprot.readListBegin()
+          for _i25 in xrange(_size21):
+            _elem26 = ColumnChunk()
+            _elem26.read(iprot)
+            self.columns.append(_elem26)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.I64:
+          self.total_byte_size = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.I64:
+          self.num_rows = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.LIST:
+          self.sorting_columns = []
+          (_etype30, _size27) = iprot.readListBegin()
+          for _i31 in xrange(_size27):
+            _elem32 = SortingColumn()
+            _elem32.read(iprot)
+            self.sorting_columns.append(_elem32)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('RowGroup')
+    if self.columns is not None:
+      oprot.writeFieldBegin('columns', TType.LIST, 1)
+      oprot.writeListBegin(TType.STRUCT, len(self.columns))
+      for iter33 in self.columns:
+        iter33.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.total_byte_size is not None:
+      oprot.writeFieldBegin('total_byte_size', TType.I64, 2)
+      oprot.writeI64(self.total_byte_size)
+      oprot.writeFieldEnd()
+    if self.num_rows is not None:
+      oprot.writeFieldBegin('num_rows', TType.I64, 3)
+      oprot.writeI64(self.num_rows)
+      oprot.writeFieldEnd()
+    if self.sorting_columns is not None:
+      oprot.writeFieldBegin('sorting_columns', TType.LIST, 4)
+      oprot.writeListBegin(TType.STRUCT, len(self.sorting_columns))
+      for iter34 in self.sorting_columns:
+        iter34.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.columns is None:
+      raise TProtocol.TProtocolException(message='Required field columns is unset!')
+    if self.total_byte_size is None:
+      raise TProtocol.TProtocolException(message='Required field total_byte_size is unset!')
+    if self.num_rows is None:
+      raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class FileMetaData:
+  """
+  Description for file metadata
+
+  Attributes:
+   - version: Version of this file *
+   - schema: Parquet schema for this file.  This schema contains metadata for all the columns.
+  The schema is represented as a tree with a single root.  The nodes of the tree
+  are flattened to a list by doing a depth-first traversal.
+  The column metadata contains the path in the schema for that column which can be
+  used to map columns to nodes in the schema.
+  The first element is the root *
+   - num_rows: Number of rows in this file *
+   - row_groups: Row groups in this file *
+   - key_value_metadata: Optional key/value metadata *
+   - created_by: String for application that wrote this file.  This should be in the format
+  <Application> version <App Version> (build <App Build Hash>).
+  e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
+
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'version', None, None, ), # 1
+    (2, TType.LIST, 'schema', (TType.STRUCT,(SchemaElement, SchemaElement.thrift_spec)), None, ), # 2
+    (3, TType.I64, 'num_rows', None, None, ), # 3
+    (4, TType.LIST, 'row_groups', (TType.STRUCT,(RowGroup, RowGroup.thrift_spec)), None, ), # 4
+    (5, TType.LIST, 'key_value_metadata', (TType.STRUCT,(KeyValue, KeyValue.thrift_spec)), None, ), # 5
+    (6, TType.STRING, 'created_by', None, None, ), # 6
+  )
+
+  def __init__(self, version=None, schema=None, num_rows=None, row_groups=None, key_value_metadata=None, created_by=None,):
+    self.version = version
+    self.schema = schema
+    self.num_rows = num_rows
+    self.row_groups = row_groups
+    self.key_value_metadata = key_value_metadata
+    self.created_by = created_by
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.version = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.LIST:
+          self.schema = []
+          (_etype38, _size35) = iprot.readListBegin()
+          for _i39 in xrange(_size35):
+            _elem40 = SchemaElement()
+            _elem40.read(iprot)
+            self.schema.append(_elem40)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.I64:
+          self.num_rows = iprot.readI64();
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.LIST:
+          self.row_groups = []
+          (_etype44, _size41) = iprot.readListBegin()
+          for _i45 in xrange(_size41):
+            _elem46 = RowGroup()
+            _elem46.read(iprot)
+            self.row_groups.append(_elem46)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 5:
+        if ftype == TType.LIST:
+          self.key_value_metadata = []
+          (_etype50, _size47) = iprot.readListBegin()
+          for _i51 in xrange(_size47):
+            _elem52 = KeyValue()
+            _elem52.read(iprot)
+            self.key_value_metadata.append(_elem52)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 6:
+        if ftype == TType.STRING:
+          self.created_by = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('FileMetaData')
+    if self.version is not None:
+      oprot.writeFieldBegin('version', TType.I32, 1)
+      oprot.writeI32(self.version)
+      oprot.writeFieldEnd()
+    if self.schema is not None:
+      oprot.writeFieldBegin('schema', TType.LIST, 2)
+      oprot.writeListBegin(TType.STRUCT, len(self.schema))
+      for iter53 in self.schema:
+        iter53.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.num_rows is not None:
+      oprot.writeFieldBegin('num_rows', TType.I64, 3)
+      oprot.writeI64(self.num_rows)
+      oprot.writeFieldEnd()
+    if self.row_groups is not None:
+      oprot.writeFieldBegin('row_groups', TType.LIST, 4)
+      oprot.writeListBegin(TType.STRUCT, len(self.row_groups))
+      for iter54 in self.row_groups:
+        iter54.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.key_value_metadata is not None:
+      oprot.writeFieldBegin('key_value_metadata', TType.LIST, 5)
+      oprot.writeListBegin(TType.STRUCT, len(self.key_value_metadata))
+      for iter55 in self.key_value_metadata:
+        iter55.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.created_by is not None:
+      oprot.writeFieldBegin('created_by', TType.STRING, 6)
+      oprot.writeString(self.created_by)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.version is None:
+      raise TProtocol.TProtocolException(message='Required field version is unset!')
+    if self.schema is None:
+      raise TProtocol.TProtocolException(message='Required field schema is unset!')
+    if self.num_rows is None:
+      raise TProtocol.TProtocolException(message='Required field num_rows is unset!')
+    if self.row_groups is None:
+      raise TProtocol.TProtocolException(message='Required field row_groups is unset!')
+    return
+
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)

+ 12 - 0
setup.py

@@ -0,0 +1,12 @@
+from distutils.core import setup
+
+setup(name='parquet',
+      version='1.0',
+      description='Python support for Parquet file format',
+      author='Joe Crobak',
+      author_email='joecrow@gmail.com',
+      packages=['parquet'],
+      requires=[
+      	'thrift',
+      ]
+     )

+ 35 - 0
test/test_read_support.py

@@ -0,0 +1,35 @@
+import tempfile
+import unittest
+
+import parquet
+
+class TestFileFormat(unittest.TestCase):
+	def test_header_magic_bytes(self):
+		with tempfile.NamedTemporaryFile() as t:
+			t.write("PAR1_some_bogus_data")
+			t.flush()
+			self.assertTrue(parquet._check_header_magic_bytes(t))
+
+	def test_footer_magic_bytes(self):
+		with tempfile.NamedTemporaryFile() as t:
+			t.write("PAR1_some_bogus_data_PAR1")
+			t.flush()
+			self.assertTrue(parquet._check_footer_magic_bytes(t))
+
+	def test_not_parquet_file(self):
+		with tempfile.NamedTemporaryFile() as t:
+			t.write("blah")
+			t.flush()
+			self.assertFalse(parquet._check_header_magic_bytes(t))
+			self.assertFalse(parquet._check_footer_magic_bytes(t))
+
+class TestMetadata(unittest.TestCase):
+
+	f = "/Users/joecrow/Code/parquet-compatibility/parquet-testdata/impala/1.0.4-SNAPPY/nation.impala.parquet"
+	
+	def testFooterBytes(self):
+		with open(self.f) as fo:
+			self.assertEquals(229, parquet._get_footer_size(fo))
+
+	def testReadFOoter(self):
+		parquet.read_footer(self.f)