test_read_support.py 1.9 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. import StringIO
  2. import tempfile
  3. import unittest
  4. import parquet
  5. class TestFileFormat(unittest.TestCase):
  6. def test_header_magic_bytes(self):
  7. with tempfile.NamedTemporaryFile() as t:
  8. t.write("PAR1_some_bogus_data")
  9. t.flush()
  10. self.assertTrue(parquet._check_header_magic_bytes(t))
  11. def test_footer_magic_bytes(self):
  12. with tempfile.NamedTemporaryFile() as t:
  13. t.write("PAR1_some_bogus_data_PAR1")
  14. t.flush()
  15. self.assertTrue(parquet._check_footer_magic_bytes(t))
  16. def test_not_parquet_file(self):
  17. with tempfile.NamedTemporaryFile() as t:
  18. t.write("blah")
  19. t.flush()
  20. self.assertFalse(parquet._check_header_magic_bytes(t))
  21. self.assertFalse(parquet._check_footer_magic_bytes(t))
  22. class TestMetadata(unittest.TestCase):
  23. f = "/Users/joecrow/Code/parquet-compatibility/parquet-testdata/impala/1.1.1-SNAPPY/nation.impala.parquet"
  24. def test_footer_bytes(self):
  25. with open(self.f) as fo:
  26. self.assertEquals(327, parquet._get_footer_size(fo))
  27. def test_read_footer(self):
  28. footer = parquet.read_footer(self.f)
  29. self.assertEquals(
  30. set([s.name for s in footer.schema]),
  31. set(["schema", "n_regionkey", "n_name", "n_nationkey",
  32. "n_comment"]))
  33. def test_dump_metadata(self):
  34. data = StringIO.StringIO()
  35. parquet.dump_metadata(self.f, data)
  36. class TestCompatibility(unittest.TestCase):
  37. files = []
  38. def _test_file(self, parquet_file, csv_file):
  39. """ Given the parquet_file and csv_file representation, converts the
  40. parquet_file to a csv using the dump utility and then compares the
  41. result to the csv_file using column agnostic ordering.
  42. """
  43. pass
  44. def test_all_files(self):
  45. for parquet_file, csv_file in self.files:
  46. yield self._test_file, parquet_file, csv_file