Bläddra i källkod

HUE-9494 [lib] Upgrade parquet from 1.1 to 1.3.1

Ying Chen 5 år sedan
förälder
incheckning
7a6b61f09e
27 ändrade filer med 263 tillägg och 186 borttagningar
  1. 0 22
      desktop/core/ext-py/parquet-1.1/PKG-INFO
  2. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/gzip-nation.impala.parquet
  3. 0 25
      desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.csv
  4. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.dict.parquet
  5. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.impala.parquet
  6. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.plain.parquet
  7. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/snappy-nation.impala.parquet
  8. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/test-converted-type-null.parquet
  9. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/test-null-dictionary.parquet
  10. BIN
      desktop/core/ext-py/parquet-1.1/parquet/test-data/test-null.parquet
  11. 0 20
      desktop/core/ext-py/parquet-1.1/parquet/thrift_filetransport.py
  12. 0 35
      desktop/core/ext-py/parquet-1.1/tox.ini
  13. 0 0
      desktop/core/ext-py/parquet-1.3.1/MANIFEST.in
  14. 116 0
      desktop/core/ext-py/parquet-1.3.1/PKG-INFO
  15. 11 2
      desktop/core/ext-py/parquet-1.3.1/README.rst
  16. 18 27
      desktop/core/ext-py/parquet-1.3.1/parquet/__init__.py
  17. 4 5
      desktop/core/ext-py/parquet-1.3.1/parquet/__main__.py
  18. 21 23
      desktop/core/ext-py/parquet-1.3.1/parquet/converted_types.py
  19. 11 10
      desktop/core/ext-py/parquet-1.3.1/parquet/encoding.py
  20. 0 0
      desktop/core/ext-py/parquet-1.3.1/parquet/parquet.thrift
  21. 5 9
      desktop/core/ext-py/parquet-1.3.1/parquet/schema.py
  22. 43 0
      desktop/core/ext-py/parquet-1.3.1/parquet/thrift_filetransport.py
  23. 0 1
      desktop/core/ext-py/parquet-1.3.1/setup.cfg
  24. 12 5
      desktop/core/ext-py/parquet-1.3.1/setup.py
  25. 13 0
      desktop/core/ext-py/parquet-1.3.1/test/test_converted_types.py
  26. 9 2
      desktop/core/ext-py/parquet-1.3.1/test/test_encoding.py
  27. 0 0
      desktop/core/ext-py/parquet-1.3.1/test/test_read_support.py

+ 0 - 22
desktop/core/ext-py/parquet-1.1/PKG-INFO

@@ -1,22 +0,0 @@
-Metadata-Version: 1.1
-Name: parquet
-Version: 1.1
-Summary: Python support for Parquet file format
-Home-page: https://github.com/jcrobak/parquet-python
-Author: Joe Crobak
-Author-email: joecrow@gmail.com
-License: Apache License 2.0
-Description: UNKNOWN
-Platform: UNKNOWN
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Intended Audience :: System Administrators
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Programming Language :: Python :: Implementation :: PyPy

BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/gzip-nation.impala.parquet


+ 0 - 25
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.csv

@@ -1,25 +0,0 @@
-0|ALGERIA|0| haggle. carefully final deposits detect slyly agai
-1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon
-2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special 
-3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
-4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d
-5|ETHIOPIA|0|ven packages wake quickly. regu
-6|FRANCE|3|refully final requests. regular, ironi
-7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco
-8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun
-9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
-10|IRAN|4|efully alongside of the slyly final dependencies. 
-11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula
-12|JAPAN|2|ously. final, express gifts cajole a
-13|JORDAN|4|ic deposits are blithely about the carefully regular pa
-14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t
-15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold platelets?
-16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r
-17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun
-18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos
-19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
-20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely
-21|VIETNAM|2|hely enticingly express accounts. even, final 
-22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint
-23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull
-24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be

BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.dict.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.impala.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.plain.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/snappy-nation.impala.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/test-converted-type-null.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/test-null-dictionary.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/test-null.parquet


+ 0 - 20
desktop/core/ext-py/parquet-1.1/parquet/thrift_filetransport.py

@@ -1,20 +0,0 @@
-"""thrift_filetransport.py - read thrift encoded data from a file object."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from thriftpy.transport import TTransportBase
-
-
-class TFileTransport(TTransportBase):  # pylint: disable=too-few-public-methods
-    """TTransportBase implementation for decoding data from a file object."""
-
-    def __init__(self, fo):
-        """Initialize with `fo`, the file object to read from."""
-        self._fo = fo
-        self._pos = fo.tell()
-
-    def _read(self, sz):
-        """Read data `sz` bytes."""
-        return self._fo.read(sz)

+ 0 - 35
desktop/core/ext-py/parquet-1.1/tox.ini

@@ -1,35 +0,0 @@
-[tox]
-envlist = py27, py34, py35, pypy, flake8, pylint
-[testenv]
-deps=
-   nose
-   python-snappy
-   pymongo
-commands=nosetests
-
-# Linters
-[testenv:flake8]
-basepython = python3
-skip_install = true
-deps =
-    flake8
-    flake8-docstrings>=0.2.7
-    flake8-import-order>=0.9
-commands =
-    flake8 parquet/ test/ setup.py
-
-
-# flake8 config
-[flake8]
-max-line-length = 120
-ignore = F841
-exclude = build
-
-
-[testenv:pylint]
-basepython = python3
-deps =
-    pyflakes
-    pylint
-commands =
-    pylint parquet/ setup.py

+ 0 - 0
desktop/core/ext-py/parquet-1.1/MANIFEST.in → desktop/core/ext-py/parquet-1.3.1/MANIFEST.in


+ 116 - 0
desktop/core/ext-py/parquet-1.3.1/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.1
+Name: parquet
+Version: 1.3.1
+Summary: Python support for Parquet file format
+Home-page: https://github.com/jcrobak/parquet-python
+Author: Joe Crobak
+Author-email: joecrow@gmail.com
+License: Apache License 2.0
+Description: parquet-python
+        ==============
+        
+        .. image:: https://travis-ci.org/jcrobak/parquet-python.svg?branch=master
+            :target: https://travis-ci.org/jcrobak/parquet-python
+        
+        parquet-python is a pure-python implementation (currently with only
+        read-support) of the `parquet
+        format <https://github.com/Parquet/parquet-format>`_. It comes with a
+        script for reading parquet files and outputting the data to stdout as
+        JSON or TSV (without the overhead of JVM startup). Performance has not
+        yet been optimized, but it's useful for debugging and quick viewing of
+        data in files.
+        
+        Not all parts of the parquet-format have been implemented yet or tested
+        e.g. nested data—see Todos below for a full list. With that said,
+        parquet-python is capable of reading all the data files from the
+        `parquet-compatability <https://github.com/Parquet/parquet-compatibility>`_
+        project.
+        
+        requirements
+        ============
+        
+        parquet-python has been tested on python 2.7, 3.6, and 3.7. It depends
+        on ``pythrift2`` and optionally on ``python-snappy`` (for snappy compressed
+        files, please also install ``parquet-python[snappy]``).
+        
+        getting started
+        ===============
+        
+        parquet-python is available via PyPi and can be installed using
+        `pip install parquet`. The package includes the `parquet`
+        command for reading python files, e.g. `parquet test.parquet`.
+        See `parquet --help` for full usage.
+        
+        Example
+        -------
+        
+        parquet-python currently has two programatic interfaces with similar
+        functionality to Python's csv reader. First, it supports a DictReader
+        which returns a dictionary per row. Second, it has a reader which
+        returns a list of values for each row. Both function require a file-like
+        object and support an optional ``columns`` field to only read the
+        specified columns.
+        
+        .. code:: python
+        
+        
+            import parquet
+            import json
+        
+            ## assuming parquet file with two rows and three columns:
+            ## foo bar baz
+            ## 1   2   3
+            ## 4   5   6
+        
+            with open("test.parquet") as fo:
+               # prints:
+               # {"foo": 1, "bar": 2}
+               # {"foo": 4, "bar": 5}
+               for row in parquet.DictReader(fo, columns=['foo', 'bar']):
+                   print(json.dumps(row))
+        
+        
+            with open("test.parquet") as fo:
+               # prints:
+               # 1,2
+               # 4,5
+               for row in parquet.reader(fo, columns=['foo', 'bar]):
+                   print(",".join([str(r) for r in row]))
+        
+        Todos
+        =====
+        
+        -  Support the deprecated bitpacking
+        -  Fix handling of repetition-levels and definition-levels
+        -  Tests for nested schemas, null data
+        -  Support reading of data from HDFS via snakebite and/or webhdfs.
+        -  Implement writing
+        -  performance evaluation and optimization (i.e. how does it compare to
+           the c++, java implementations)
+        
+        Contributing
+        ============
+        
+        Is done via Pull Requests. Please include tests with your changes and
+        follow `pep8 <http://www.python.org/dev/peps/pep-0008/>`_.
+        
+        To run the tests you must install and execute ``tox`` (``pip install tox``) to
+        run for all supported versions. If you want to run just for your current
+        version, execute: ``pip install -r requirements-development.txt`` and then
+        ``nosetests``.
+        
+Platform: UNKNOWN
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Description-Content-Type: text/x-rst
+Provides-Extra: snappy

+ 11 - 2
desktop/core/ext-py/parquet-1.1/README.rst → desktop/core/ext-py/parquet-1.3.1/README.rst

@@ -1,6 +1,9 @@
 parquet-python
 ==============
 
+.. image:: https://travis-ci.org/jcrobak/parquet-python.svg?branch=master
+    :target: https://travis-ci.org/jcrobak/parquet-python
+
 parquet-python is a pure-python implementation (currently with only
 read-support) of the `parquet
 format <https://github.com/Parquet/parquet-format>`_. It comes with a
@@ -18,8 +21,9 @@ project.
 requirements
 ============
 
-parquet-python has been tested on python 2.7, 3.4, and 3.5. It depends
-on ``thrift`` (0.9) and ``python-snappy`` (for snappy compressed files).
+parquet-python has been tested on python 2.7, 3.6, and 3.7. It depends
+on ``pythrift2`` and optionally on ``python-snappy`` (for snappy compressed
+files, please also install ``parquet-python[snappy]``).
 
 getting started
 ===============
@@ -81,3 +85,8 @@ Contributing
 
 Is done via Pull Requests. Please include tests with your changes and
 follow `pep8 <http://www.python.org/dev/peps/pep-0008/>`_.
+
+To run the tests you must install and execute ``tox`` (``pip install tox``) to
+run for all supported versions. If you want to run just for your current
+version, execute: ``pip install -r requirements-development.txt`` and then
+``nosetests``.

+ 18 - 27
desktop/core/ext-py/parquet-1.1/parquet/__init__.py → desktop/core/ext-py/parquet-1.3.1/parquet/__init__.py

@@ -1,8 +1,6 @@
 """parquet - read parquet files."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
 
 import gzip
 import io
@@ -11,18 +9,12 @@ import logging
 import os
 import struct
 import sys
+from collections import OrderedDict, defaultdict
 
-from collections import defaultdict
-try:
-  from collections import OrderedDict
-except ImportError:
-  from ordereddict import OrderedDict # Python 2.6
-
-import thriftpy
-from thriftpy.protocol.compact import TCompactProtocolFactory
+import thriftpy2 as thriftpy
+from thriftpy2.protocol.compact import TCompactProtocolFactory
 
-from . import encoding
-from . import schema
+from . import encoding, schema
 from .converted_types import convert_column
 from .thrift_filetransport import TFileTransport
 
@@ -39,7 +31,6 @@ parquet_thrift = thriftpy.load(THRIFT_FILE, module_name="parquet_thrift")  # pyl
 
 logger = logging.getLogger("parquet")  # pylint: disable=invalid-name
 
-
 try:
     import snappy
 except ImportError:
@@ -49,7 +40,6 @@ except ImportError:
 
 class ParquetFormatException(Exception):
     """Generic Exception related to unexpected data format when reading parquet file."""
-    pass
 
 
 def _check_header_magic_bytes(file_obj):
@@ -139,7 +129,7 @@ def dump_metadata(filename, show_row_group_metadata, out=sys.stdout):
     println("  Version: {0}".format(footer.version))
     println("  Num Rows: {0}".format(footer.num_rows))
     println("  k/v metadata: ")
-    if footer.key_value_metadata and len(footer.key_value_metadata) > 0:
+    if footer.key_value_metadata:
         for item in footer.key_value_metadata:
             println("    {0}={1}".format(item.key, item.value))
     else:
@@ -281,12 +271,12 @@ def _read_data(file_obj, fo_encoding, value_count, bit_width):
 
 
 def read_data_page(file_obj, schema_helper, page_header, column_metadata,
-                       dictionary):
-
-
+                   dictionary):
     """Read the data page from the given file-like object based upon the parameters.
+
     Metadata in the the schema_helper, page_header, column_metadata, and (optional) dictionary
     are used for parsing data.
+
     Returns a list of values.
     """
     daph = page_header.data_page_header
@@ -383,13 +373,14 @@ def read_data_page(file_obj, schema_helper, page_header, column_metadata,
                          len(vals), num_nulls)
 
     else:
-        raise ParquetFormatException("Unsupported encoding: %s",
-                                     _get_name(parquet_thrift.Encoding, daph.encoding))
+        raise ParquetFormatException("Unsupported encoding: {}".format(
+            _get_name(parquet_thrift.Encoding, daph.encoding)))
     return vals
 
 
 def _read_dictionary_page(file_obj, schema_helper, page_header, column_metadata):
     """Read a page containing dictionary data.
+
     Consumes data using the plain encoding and returns an array of values.
     """
     raw_bytes = _read_page(file_obj, page_header, column_metadata)
@@ -413,7 +404,7 @@ def DictReader(file_obj, columns=None):  # pylint: disable=invalid-name
     top-level dict and can be referenced with '.' notation (e.g. 'foo' -> 'bar'
     is referenced as 'foo.bar')
 
-    :param fo: the file containing parquet data
+    :param file_obj: the file containing parquet data
     :param columns: the columns to include. If None (default), all columns
                     are included. Nested values are referenced with "." notation
     """
@@ -432,7 +423,7 @@ def reader(file_obj, columns=None):
     This function is a generator returning a list of values for each row
     of data in the parquet file.
 
-    :param fo: the file containing parquet data
+    :param file_obj: the file containing parquet data
     :param columns: the columns to include. If None (default), all columns
                     are included. Nested values are referenced with "." notation
     """
@@ -489,7 +480,7 @@ def reader(file_obj, columns=None):
             yield [res[k][i] for k in keys if res[k]]
 
 
-class JsonWriter(object):  # pylint: disable=too-few-public-methods
+class JsonWriter:  # pylint: disable=too-few-public-methods
     """Utility for dumping rows as JSON objects."""
 
     def __init__(self, out):
@@ -500,7 +491,7 @@ class JsonWriter(object):  # pylint: disable=too-few-public-methods
         """Write a single row."""
         json_text = json.dumps(row)
         if isinstance(json_text, bytes):
-            json_text = json_text.decode('utf-8', 'ignore')
+            json_text = json_text.decode('utf-8')
         self._out.write(json_text)
         self._out.write(u'\n')
 
@@ -524,7 +515,7 @@ def _dump(file_obj, options, out=sys.stdout):
             writer.writeheader()
         if options.limit != -1 and total_count >= options.limit:
             return
-        row_unicode = dict((k, (v.decode("utf-8", "ignore") if type(v) is bytes else v)) for k, v in row.items())
+        row_unicode = {k: v.decode("utf-8") if isinstance(v, bytes) else v for k, v in row.items()}
         writer.writerow(row_unicode)
         total_count += 1
 

+ 4 - 5
desktop/core/ext-py/parquet-1.1/parquet/__main__.py → desktop/core/ext-py/parquet-1.3.1/parquet/__main__.py

@@ -1,9 +1,6 @@
 """parquet - tool for inspecting parquet files."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
 
 import argparse
 import logging
@@ -53,6 +50,7 @@ def main(argv=None):
 
     setup_logging(args)
 
+    # pylint: disable=import-outside-toplevel
     import parquet
 
     if args.metadata:
@@ -60,5 +58,6 @@ def main(argv=None):
     if not args.no_data:
         parquet.dump(args.file, args)
 
+
 if __name__ == '__main__':
     main()

+ 21 - 23
desktop/core/ext-py/parquet-1.1/parquet/converted_types.py → desktop/core/ext-py/parquet-1.3.1/parquet/converted_types.py

@@ -6,10 +6,8 @@ The implementations in this class are pure python for the widest compatibility,
 but they're not necessarily the most performant.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
 
 import codecs
 import datetime
@@ -20,7 +18,7 @@ import struct
 import sys
 from decimal import Decimal
 
-import thriftpy
+import thriftpy2 as thriftpy
 
 THRIFT_FILE = os.path.join(os.path.dirname(__file__), "parquet.thrift")
 parquet_thrift = thriftpy.load(THRIFT_FILE, module_name=str("parquet_thrift"))  # pylint: disable=invalid-name
@@ -33,7 +31,7 @@ try:
 except ImportError:
     pass
 
-PY3 = sys.version_info > (3,)
+PY3 = sys.version_info.major > 2
 
 # define bytes->int for non 2, 4, 8 byte ints
 if PY3:
@@ -53,8 +51,8 @@ def _convert_unsigned(data, fmt):
     """Convert data from signed to unsigned in bulk."""
     num = len(data)
     return struct.unpack(
-        b"{0}{0}".format(num, fmt.upper()).encode("utf-8"),
-        struct.pack(b"{0}{0}".format(num, fmt).encode("utf-8"), *data)
+        "{}{}".format(num, fmt.upper()).encode("utf-8"),
+        struct.pack("{}{}".format(num, fmt).encode("utf-8"), *data)
     )
 
 
@@ -66,27 +64,27 @@ def convert_column(data, schemae):
         if schemae.type == parquet_thrift.Type.INT32 or schemae.type == parquet_thrift.Type.INT64:
             return [Decimal(unscaled) * scale_factor for unscaled in data]
         return [Decimal(intbig(unscaled)) * scale_factor for unscaled in data]
-    elif ctype == parquet_thrift.ConvertedType.DATE:
+    if ctype == parquet_thrift.ConvertedType.DATE:
         return [datetime.date.fromordinal(d) for d in data]
-    elif ctype == parquet_thrift.ConvertedType.TIME_MILLIS:
+    if ctype == parquet_thrift.ConvertedType.TIME_MILLIS:
         return [datetime.timedelta(milliseconds=d) for d in data]
-    elif ctype == parquet_thrift.ConvertedType.TIMESTAMP_MILLIS:
+    if ctype == parquet_thrift.ConvertedType.TIMESTAMP_MILLIS:
         return [datetime.datetime.utcfromtimestamp(d / 1000.0) for d in data]
-    elif ctype == parquet_thrift.ConvertedType.UTF8:
-        return list(codecs.iterdecode(data, "utf-8"))
-    elif ctype == parquet_thrift.ConvertedType.UINT_8:
+    if ctype == parquet_thrift.ConvertedType.UTF8:
+        return [codecs.decode(item, "utf-8") for item in data]
+    if ctype == parquet_thrift.ConvertedType.UINT_8:
         return _convert_unsigned(data, 'b')
-    elif ctype == parquet_thrift.ConvertedType.UINT_16:
+    if ctype == parquet_thrift.ConvertedType.UINT_16:
         return _convert_unsigned(data, 'h')
-    elif ctype == parquet_thrift.ConvertedType.UINT_32:
+    if ctype == parquet_thrift.ConvertedType.UINT_32:
         return _convert_unsigned(data, 'i')
-    elif ctype == parquet_thrift.ConvertedType.UINT_64:
+    if ctype == parquet_thrift.ConvertedType.UINT_64:
         return _convert_unsigned(data, 'q')
-    elif ctype == parquet_thrift.ConvertedType.JSON:
+    if ctype == parquet_thrift.ConvertedType.JSON:
         return [json.loads(s) for s in codecs.iterdecode(data, "utf-8")]
-    elif ctype == parquet_thrift.ConvertedType.BSON and bson:
+    if ctype == parquet_thrift.ConvertedType.BSON and bson:
         return [bson.BSON(s).decode() for s in data]
-    else:
-        logger.info("Converted type '%s'' not handled",
-                    parquet_thrift.ConvertedType._VALUES_TO_NAMES[ctype])  # pylint:disable=protected-access
-    return data
+
+    logger.info("Converted type '%s'' not handled",
+                parquet_thrift.ConvertedType._VALUES_TO_NAMES[ctype])  # pylint:disable=protected-access
+    return data

+ 11 - 10
desktop/core/ext-py/parquet-1.1/parquet/encoding.py → desktop/core/ext-py/parquet-1.3.1/parquet/encoding.py

@@ -13,14 +13,14 @@ import os
 import struct
 import sys
 
-import thriftpy
+import thriftpy2 as thriftpy
 
 THRIFT_FILE = os.path.join(os.path.dirname(__file__), "parquet.thrift")
 parquet_thrift = thriftpy.load(THRIFT_FILE, module_name=str("parquet_thrift"))  # pylint: disable=invalid-name
 
 logger = logging.getLogger("parquet")  # pylint: disable=invalid-name
 
-PY3 = sys.version_info > (3,)
+PY3 = sys.version_info.major > 2
 
 ARRAY_BYTE_STR = u'B' if PY3 else b'B'
 
@@ -38,31 +38,30 @@ def read_plain_int32(file_obj, count):
     length = 4 * count
     data = file_obj.read(length)
     if len(data) != length:
-        raise EOFError("Expected {0} bytes but got {1} bytes".format(length, len(data)))
-    res = struct.unpack(b"<{0}i".format(count).encode("utf-8"), data)
+        raise EOFError("Expected {} bytes but got {} bytes".format(length, len(data)))
+    res = struct.unpack("<{}i".format(count).encode("utf-8"), data)
     return res
 
 
 def read_plain_int64(file_obj, count):
     """Read `count` 64-bit ints using the plain encoding."""
-    return struct.unpack(b"<{0}q".format(count).encode("utf-8"), file_obj.read(8 * count))
+    return struct.unpack("<{}q".format(count).encode("utf-8"), file_obj.read(8 * count))
 
 
 def read_plain_int96(file_obj, count):
     """Read `count` 96-bit ints using the plain encoding."""
-    items = struct.unpack(b"<qi" * count, file_obj.read(12) * count)
-    args = [iter(items)] * 2
-    return [q << 32 | i for (q, i) in zip(*args)]
+    items = struct.unpack(b"<" + b"qi" * count, file_obj.read(12 * count))
+    return [q << 32 | i for (q, i) in zip(items[0::2], items[1::2])]
 
 
 def read_plain_float(file_obj, count):
     """Read `count` 32-bit floats using the plain encoding."""
-    return struct.unpack(b"<{0}f".format(count).encode("utf-8"), file_obj.read(4 * count))
+    return struct.unpack("<{}f".format(count).encode("utf-8"), file_obj.read(4 * count))
 
 
 def read_plain_double(file_obj, count):
     """Read `count` 64-bit float (double) using the plain encoding."""
-    return struct.unpack(b"<{0}d".format(count).encode("utf-8"), file_obj.read(8 * count))
+    return struct.unpack("<{}d".format(count).encode("utf-8"), file_obj.read(8 * count))
 
 
 def read_plain_byte_array(file_obj, count):
@@ -148,6 +147,8 @@ def read_bitpacked(file_obj, header, width, debug_logging):
     if debug_logging:
         logger.debug("Reading a bit-packed run with: %s groups, count %s, bytes %s",
                      num_groups, count, byte_count)
+    if width == 0:
+        return [0 for _ in range(count)]
     raw_bytes = array.array(ARRAY_BYTE_STR, file_obj.read(byte_count)).tolist()
     current_byte = 0
     data = raw_bytes[current_byte]

+ 0 - 0
desktop/core/ext-py/parquet-1.1/parquet/parquet.thrift → desktop/core/ext-py/parquet-1.3.1/parquet/parquet.thrift


+ 5 - 9
desktop/core/ext-py/parquet-1.1/parquet/schema.py → desktop/core/ext-py/parquet-1.3.1/parquet/schema.py

@@ -1,26 +1,22 @@
 """Utils for working with the parquet thrift models."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
 
 import os
 
-import thriftpy
-
+import thriftpy2 as thriftpy
 
 THRIFT_FILE = os.path.join(os.path.dirname(__file__), "parquet.thrift")
 parquet_thrift = thriftpy.load(THRIFT_FILE, module_name=str("parquet_thrift"))  # pylint: disable=invalid-name
 
 
-class SchemaHelper(object):
+class SchemaHelper:
     """Utility providing convenience methods for schema_elements."""
 
     def __init__(self, schema_elements):
         """Initialize with the specified schema_elements."""
         self.schema_elements = schema_elements
-        self.schema_elements_by_name = dict(
-            [(se.name, se) for se in schema_elements])
+        self.schema_elements_by_name = {se.name: se for se in schema_elements}
         assert len(self.schema_elements) == len(self.schema_elements_by_name)
 
     def schema_element(self, name):

+ 43 - 0
desktop/core/ext-py/parquet-1.3.1/parquet/thrift_filetransport.py

@@ -0,0 +1,43 @@
+"""thrift_filetransport.py - read thrift encoded data from a file object."""
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+from thriftpy2.transport import TTransportBase
+
+
+class TFileTransport(TTransportBase):  # pylint: disable=too-few-public-methods
+    """TTransportBase implementation for decoding data from a file object."""
+
+    def __init__(self, fo):
+        """Initialize with `fo`, the file object to read from."""
+        self._fo = fo
+        self._pos = fo.tell()
+
+    def _read(self, sz):
+        """Read data `sz` bytes."""
+        return self._fo.read(sz)
+
+    def open(self):
+        """Open which is a no-op."""
+        if not self.is_open():
+            raise ValueError("Already closed.")
+
+    def is_open(self):
+        """Return true if open."""
+        return not self._fo.closed
+
+    def close(self):
+        """Close the file object."""
+        self._fo.close()
+
+    def read(self, sz):
+        """Read data `sz` bytes."""
+        return self._fo.read(sz)
+
+    def write(self, buf):
+        """Write buf to the file object."""
+        self._fo.write(buf)
+
+    def flush(self):
+        """Flush the output."""
+        self._fo.flush()

+ 0 - 1
desktop/core/ext-py/parquet-1.1/setup.cfg → desktop/core/ext-py/parquet-1.3.1/setup.cfg

@@ -1,5 +1,4 @@
 [egg_info]
 tag_build = 
 tag_date = 0
-tag_svn_revision = 0
 

+ 12 - 5
desktop/core/ext-py/parquet-1.1/setup.py → desktop/core/ext-py/parquet-1.3.1/setup.py

@@ -5,10 +5,15 @@ try:
 except ImportError:
     from distutils.core import setup
 
+with open('README.rst') as f:
+    readme = f.read()
+
 setup(
     name='parquet',
-    version='1.1',
+    version='1.3.1',
     description='Python support for Parquet file format',
+    long_description_content_type="text/x-rst",
+    long_description=readme,
     author='Joe Crobak',
     author_email='joecrow@gmail.com',
     url='https://github.com/jcrobak/parquet-python',
@@ -22,20 +27,22 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
     ],
     packages=['parquet'],
     install_requires=[
-        'thriftpy>=0.3.6',
+        'thriftpy2',
     ],
     extras_require={
         ':python_version=="2.7"': [
             "backports.csv",
         ],
-        'snappy support': ['python-snappy']
+        'snappy': [
+            'python-snappy',
+        ],
     },
     entry_points={
         'console_scripts': [

+ 13 - 0
desktop/core/ext-py/parquet-1.1/test/test_converted_types.py → desktop/core/ext-py/parquet-1.3.1/test/test_converted_types.py

@@ -134,6 +134,19 @@ class TestSBytes(unittest.TestCase):
             'foo👾'
         )
 
+    def test_utf8_empty_string(self):
+        """Test bytes representing utf-8 string with empty strings."""
+        schema = pt.SchemaElement(
+            type=pt.Type.BYTE_ARRAY,
+            name="test",
+            converted_type=pt.ConvertedType.UTF8
+        )
+        data = [b'', b'foo\xf0\x9f\x91\xbe', b'']
+        self.assertEqual(
+            convert_column(data, schema),
+            ['', 'foo👾', '']
+        )
+
     def test_json(self):
         """Test bytes representing json."""
         schema = pt.SchemaElement(

+ 9 - 2
desktop/core/ext-py/parquet-1.1/test/test_encoding.py → desktop/core/ext-py/parquet-1.3.1/test/test_encoding.py

@@ -28,9 +28,9 @@ class TestPlain(unittest.TestCase):
     def test_int96(self):
         """Test reading bytes containing int96 data."""
         self.assertEqual(
-            999,
+            [999, 1 << 32 | 1000],
             parquet.encoding.read_plain_int96(
-                io.BytesIO(struct.pack(b"<qi", 0, 999)), 1)[0])
+                io.BytesIO(struct.pack(b"<qiqi", 0, 999, 1, 1000)), 2))
 
     def test_float(self):
         """Test reading bytes containing float data."""
@@ -117,6 +117,13 @@ class TestBitPacked(unittest.TestCase):
         res = parquet.encoding.read_bitpacked(fo, count, 3, True)
         self.assertEqual(list(range(8)), res)
 
+    def test_width_zero(self):
+        """Test a zero-width item."""
+        fo = io.BytesIO()
+        count = 1 << 1
+        res = parquet.encoding.read_bitpacked(fo, count, 0, True)
+        self.assertEqual([0] * 8, res)
+
 
 class TestBitPackedDeprecated(unittest.TestCase):
     """Test reading the deprecated bit-packed encoded data."""

+ 0 - 0
desktop/core/ext-py/parquet-1.1/test/test_read_support.py → desktop/core/ext-py/parquet-1.3.1/test/test_read_support.py