|
@@ -1,4 +1,5 @@
|
|
|
import csv
|
|
import csv
|
|
|
|
|
+import json
|
|
|
import os
|
|
import os
|
|
|
import StringIO
|
|
import StringIO
|
|
|
import tempfile
|
|
import tempfile
|
|
@@ -48,12 +49,22 @@ class TestMetadata(unittest.TestCase):
|
|
|
parquet.dump_metadata(self.f, data)
|
|
parquet.dump_metadata(self.f, data)
|
|
|
|
|
|
|
|
|
|
|
|
|
-class Options():
|
|
|
|
|
- col = None
|
|
|
|
|
- format = 'csv'
|
|
|
|
|
- no_headers = True
|
|
|
|
|
- limit = -1
|
|
|
|
|
|
|
+class Options(object):
|
|
|
|
|
|
|
|
|
|
+ def __init__(self, col=None, format='csv', no_headers=True, limit=-1):
|
|
|
|
|
+ self.col = col
|
|
|
|
|
+ self.format = format
|
|
|
|
|
+ self.no_headers = no_headers
|
|
|
|
|
+ self.limit = limit
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class TestReadApi(unittest.TestCase):
|
|
|
|
|
+
|
|
|
|
|
+ def test_projection(self):
|
|
|
|
|
+ pass
|
|
|
|
|
+
|
|
|
|
|
+ def test_limit(self):
|
|
|
|
|
+ pass
|
|
|
|
|
|
|
|
class TestCompatibility(object):
|
|
class TestCompatibility(object):
|
|
|
|
|
|
|
@@ -66,7 +77,7 @@ class TestCompatibility(object):
|
|
|
def _test_file_csv(self, parquet_file, csv_file):
|
|
def _test_file_csv(self, parquet_file, csv_file):
|
|
|
""" Given the parquet_file and csv_file representation, converts the
|
|
""" Given the parquet_file and csv_file representation, converts the
|
|
|
parquet_file to a csv using the dump utility and then compares the
|
|
parquet_file to a csv using the dump utility and then compares the
|
|
|
- result to the csv_file using column agnostic ordering.
|
|
|
|
|
|
|
+ result to the csv_file.
|
|
|
"""
|
|
"""
|
|
|
expected_data = []
|
|
expected_data = []
|
|
|
with open(csv_file, 'rb') as f:
|
|
with open(csv_file, 'rb') as f:
|
|
@@ -80,6 +91,43 @@ class TestCompatibility(object):
|
|
|
assert expected_data == actual_data, "{0} != {1}".format(
|
|
assert expected_data == actual_data, "{0} != {1}".format(
|
|
|
str(expected_data), str(actual_data))
|
|
str(expected_data), str(actual_data))
|
|
|
|
|
|
|
|
|
|
+ actual_raw_data = StringIO.StringIO()
|
|
|
|
|
+ parquet.dump(parquet_file, Options(no_headers=False),
|
|
|
|
|
+ out=actual_raw_data)
|
|
|
|
|
+ actual_raw_data.seek(0, 0)
|
|
|
|
|
+ actual_data = list(csv.reader(actual_raw_data, delimiter='\t'))[1:]
|
|
|
|
|
+
|
|
|
|
|
+ assert expected_data == actual_data, "{0} != {1}".format(
|
|
|
|
|
+ str(expected_data), str(actual_data))
|
|
|
|
|
+
|
|
|
|
|
+ def _test_file_json(self, parquet_file, csv_file):
|
|
|
|
|
+ """ Given the parquet_file and csv_file representation, converts the
|
|
|
|
|
+ parquet_file to json using the dump utility and then compares the
|
|
|
|
|
+ result to the csv_file using column agnostic ordering.
|
|
|
|
|
+ """
|
|
|
|
|
+ expected_data = []
|
|
|
|
|
+ with open(csv_file, 'rb') as f:
|
|
|
|
|
+ expected_data = list(csv.reader(f, delimiter='|'))
|
|
|
|
|
+
|
|
|
|
|
+ actual_raw_data = StringIO.StringIO()
|
|
|
|
|
+ parquet.dump(parquet_file, Options(format='json'),
|
|
|
|
|
+ out=actual_raw_data)
|
|
|
|
|
+ actual_raw_data.seek(0, 0)
|
|
|
|
|
+ actual_data = [json.loads(x.rstrip()) for x in
|
|
|
|
|
+ actual_raw_data.read().split("\n") if len(x) > 0]
|
|
|
|
|
+
|
|
|
|
|
+ assert len(expected_data) == len(actual_data)
|
|
|
|
|
+ footer = parquet.read_footer(parquet_file)
|
|
|
|
|
+ cols = [s.name for s in footer.schema]
|
|
|
|
|
+ for expected, actual in zip(expected_data, actual_raw_data):
|
|
|
|
|
+ assert len(expected) == len(actual)
|
|
|
|
|
+ for i, c in enumerate(cols):
|
|
|
|
|
+ if c in actual:
|
|
|
|
|
+ assert expected[i] == actual[c]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def test_all_files(self):
|
|
def test_all_files(self):
|
|
|
for parquet_file, csv_file in self.files:
|
|
for parquet_file, csv_file in self.files:
|
|
|
yield self._test_file_csv, parquet_file, csv_file
|
|
yield self._test_file_csv, parquet_file, csv_file
|
|
|
|
|
+ yield self._test_file_json, parquet_file, csv_file
|