Browse Source

Integration tests for parquet.

Testing all of the variations of the nation files from
parquet-compatibility.
Joe Crobak 12 years ago
parent
commit
80cbd47aed

+ 2 - 1
parquet/__init__.py

@@ -358,7 +358,8 @@ def dump(filename, options, out=sys.stdout):
                     else:
                     else:
                         logger.warn("Skipping unknown page type={0}".format(
                         logger.warn("Skipping unknown page type={0}".format(
                             _get_name(PageType, ph.type)))
                             _get_name(PageType, ph.type)))
-        keys = res.keys()
+        keys = options.col if options.col else [s.name for s in
+                                                footer.schema if s.name in res]
         if options.format == "csv" and not options.no_headers:
         if options.format == "csv" and not options.no_headers:
             println("\t".join(keys))
             println("\t".join(keys))
         for i in range(rg.num_rows):
         for i in range(rg.num_rows):

BIN
test-data/gzip-nation.impala.parquet


+ 25 - 0
test-data/nation.csv

@@ -0,0 +1,25 @@
+0|ALGERIA|0| haggle. carefully final deposits detect slyly agai
+1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon
+2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special 
+3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
+4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d
+5|ETHIOPIA|0|ven packages wake quickly. regu
+6|FRANCE|3|refully final requests. regular, ironi
+7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco
+8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun
+9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
+10|IRAN|4|efully alongside of the slyly final dependencies. 
+11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula
+12|JAPAN|2|ously. final, express gifts cajole a
+13|JORDAN|4|ic deposits are blithely about the carefully regular pa
+14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t
+15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold platelets?
+16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r
+17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun
+18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos
+19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
+20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely
+21|VIETNAM|2|hely enticingly express accounts. even, final 
+22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint
+23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull
+24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be

BIN
test-data/nation.dict.parquet


BIN
test-data/nation.impala.parquet


BIN
test-data/nation.plain.parquet


BIN
test-data/snappy-nation.impala.parquet


+ 29 - 6
test/test_read_support.py

@@ -1,3 +1,5 @@
+import csv
+import os
 import StringIO
 import StringIO
 import tempfile
 import tempfile
 import unittest
 import unittest
@@ -28,7 +30,7 @@ class TestFileFormat(unittest.TestCase):
 
 
 class TestMetadata(unittest.TestCase):
 class TestMetadata(unittest.TestCase):
 
 
-    f = "/Users/joecrow/Code/parquet-compatibility/parquet-testdata/impala/1.1.1-SNAPPY/nation.impala.parquet"
+    f = "test-data/nation.impala.parquet"
 
 
     def test_footer_bytes(self):
     def test_footer_bytes(self):
         with open(self.f) as fo:
         with open(self.f) as fo:
@@ -46,17 +48,38 @@ class TestMetadata(unittest.TestCase):
         parquet.dump_metadata(self.f, data)
         parquet.dump_metadata(self.f, data)
 
 
 
 
-class TestCompatibility(unittest.TestCase):
+class Options():
+    col = None
+    format = 'csv'
+    no_headers = True
+    limit = -1
 
 
-    files = []
 
 
-    def _test_file(self, parquet_file, csv_file):
+class TestCompatibility(object):
+
+    td = "test-data"
+    files = [(os.path.join(td, p), os.path.join(td, "nation.csv")) for p in
+             ["gzip-nation.impala.parquet", "nation.dict.parquet",
+              "nation.impala.parquet", "nation.plain.parquet",
+              "snappy-nation.impala.parquet"]]
+
+    def _test_file_csv(self, parquet_file, csv_file):
         """ Given the parquet_file and csv_file representation, converts the
         """ Given the parquet_file and csv_file representation, converts the
             parquet_file to a csv using the dump utility and then compares the
             parquet_file to a csv using the dump utility and then compares the
             result to the csv_file using column agnostic ordering.
             result to the csv_file using column agnostic ordering.
         """
         """
-        pass
+        expected_data = []
+        with open(csv_file, 'rb') as f:
+            expected_data = list(csv.reader(f, delimiter='|'))
+
+        actual_raw_data = StringIO.StringIO()
+        parquet.dump(parquet_file, Options(), out=actual_raw_data)
+        actual_raw_data.seek(0, 0)
+        actual_data = list(csv.reader(actual_raw_data, delimiter='\t'))
+
+        assert expected_data == actual_data, "{0} != {1}".format(
+            str(expected_data), str(actual_data))
 
 
     def test_all_files(self):
     def test_all_files(self):
         for parquet_file, csv_file in self.files:
         for parquet_file, csv_file in self.files:
-            yield self._test_file, parquet_file, csv_file
+            yield self._test_file_csv, parquet_file, csv_file