ソースを参照

HUE-5009 [core] Backport parquet-python Fix converted types for plain encoding.

Commit https://github.com/jcrobak/parquet-python/commit/00369f3e18e3b25d20420e8dd804eba04362ebf9

Jenny Kim 9 年 前
コミット
f15a44b044

+ 5 - 5
desktop/core/ext-py/parquet-1.1/parquet/__init__.py

@@ -339,8 +339,10 @@ def read_data_page(file_obj, schema_helper, page_header, column_metadata,
 
     # NOTE: The repetition levels aren't yet used.
     if daph.encoding == parquet_thrift.Encoding.PLAIN:
-        read_values = \
-            encoding.read_plain(io_obj, column_metadata.type, daph.num_values - num_nulls)
+        read_values = encoding.read_plain(io_obj, column_metadata.type, daph.num_values - num_nulls)
+        schema_element = schema_helper.schema_element(column_metadata.path_in_schema[-1])
+        read_values = convert_column(read_values, schema_element) \
+            if schema_element.converted_type is not None else read_values
         if definition_levels:
             itr = iter(read_values)
             vals.extend([next(itr) if level == max_definition_level else None for level in definition_levels])
@@ -451,9 +453,7 @@ def reader(file_obj, columns=None):
                 if page_header.type == parquet_thrift.PageType.DATA_PAGE:
                     values = read_data_page(file_obj, schema_helper, page_header, cmd,
                                             dict_items)
-                    schema_element = schema_helper.schema_element(cmd.path_in_schema[-1])
-                    res[".".join(cmd.path_in_schema)] += convert_column(values, schema_element) \
-                        if schema_element.converted_type else values
+                    res[".".join(cmd.path_in_schema)] += values
                     values_seen += page_header.data_page_header.num_values
                 elif page_header.type == parquet_thrift.PageType.DICTIONARY_PAGE:
                     if debug_logging:

BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/gzip-nation.impala.parquet


+ 25 - 0
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.csv

@@ -0,0 +1,25 @@
+0|ALGERIA|0| haggle. carefully final deposits detect slyly agai
+1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon
+2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special 
+3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
+4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d
+5|ETHIOPIA|0|ven packages wake quickly. regu
+6|FRANCE|3|refully final requests. regular, ironi
+7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco
+8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun
+9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
+10|IRAN|4|efully alongside of the slyly final dependencies. 
+11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula
+12|JAPAN|2|ously. final, express gifts cajole a
+13|JORDAN|4|ic deposits are blithely about the carefully regular pa
+14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t
+15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold platelets?
+16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r
+17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun
+18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos
+19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
+20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely
+21|VIETNAM|2|hely enticingly express accounts. even, final 
+22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint
+23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull
+24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be

BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.dict.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.impala.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/nation.plain.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/snappy-nation.impala.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/test-converted-type-null.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/test-null-dictionary.parquet


BIN
desktop/core/ext-py/parquet-1.1/parquet/test-data/test-null.parquet


+ 11 - 0
desktop/core/ext-py/parquet-1.1/test/test_read_support.py

@@ -217,3 +217,14 @@ class TestDefinitionLevel(unittest.TestCase):
             [{"foo": 1, "bar": 2}, {"foo": 1, "bar": None}],
             actual_data
         )
+
+    def test_converted_type_null(self):
+        """Test reading a file that contains null records for a plain column that is converted to utf-8."""
+        with open(os.path.join(TEST_DATA, "test-converted-type-null.parquet"), "rb") as parquet_fo:
+            actual_data = list(parquet.DictReader(parquet_fo))
+
+        self.assertListEqual(
+            # this is the contents of test-converted-type-null.parquet. 2 records.
+            [{"foo": "bar"}, {"foo": None}],
+            actual_data
+        )