test_encoding.py 4.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127
  1. import array
  2. import struct
  3. import StringIO
  4. import unittest
  5. import parquet.encoding
  6. from parquet.ttypes import Type
  7. from nose import SkipTest
  8. class TestPlain(unittest.TestCase):
  9. def test_int32(self):
  10. self.assertEquals(
  11. 999,
  12. parquet.encoding.read_plain_int32(
  13. StringIO.StringIO(struct.pack("<i", 999))))
  14. def test_int64(self):
  15. self.assertEquals(
  16. 999,
  17. parquet.encoding.read_plain_int64(
  18. StringIO.StringIO(struct.pack("<q", 999))))
  19. def test_int96(self):
  20. self.assertEquals(
  21. 999,
  22. parquet.encoding.read_plain_int96(
  23. StringIO.StringIO(struct.pack("<qi", 0, 999))))
  24. def test_float(self):
  25. self.assertAlmostEquals(
  26. 9.99,
  27. parquet.encoding.read_plain_float(
  28. StringIO.StringIO(struct.pack("<f", 9.99))),
  29. 2)
  30. def test_double(self):
  31. self.assertEquals(
  32. 9.99,
  33. parquet.encoding.read_plain_double(
  34. StringIO.StringIO(struct.pack("<d", 9.99))))
  35. def test_fixed(self):
  36. data = "foobar"
  37. fo = StringIO.StringIO(data)
  38. self.assertEquals(
  39. data[:3],
  40. parquet.encoding.read_plain_byte_array_fixed(
  41. fo, 3))
  42. self.assertEquals(
  43. data[3:],
  44. parquet.encoding.read_plain_byte_array_fixed(
  45. fo, 3))
  46. def test_fixed_read_plain(self):
  47. data = "foobar"
  48. fo = StringIO.StringIO(data)
  49. self.assertEquals(
  50. data[:3],
  51. parquet.encoding.read_plain(
  52. fo, Type.FIXED_LEN_BYTE_ARRAY, 3))
  53. class TestRle(unittest.TestCase):
  54. def testFourByteValue(self):
  55. fo = StringIO.StringIO(struct.pack("<i", 1 << 30))
  56. out = parquet.encoding.read_rle(fo, 2 << 1, 30)
  57. self.assertEquals([1 << 30] * 2, list(out))
  58. class TestVarInt(unittest.TestCase):
  59. def testSingleByte(self):
  60. fo = StringIO.StringIO(struct.pack("<B", 0x7F))
  61. out = parquet.encoding.read_unsigned_var_int(fo)
  62. self.assertEquals(0x7F, out)
  63. def testFourByte(self):
  64. fo = StringIO.StringIO(struct.pack("<BBBB", 0xFF, 0xFF, 0xFF, 0x7F))
  65. out = parquet.encoding.read_unsigned_var_int(fo)
  66. self.assertEquals(0x0FFFFFFF, out)
  67. class TestBitPacked(unittest.TestCase):
  68. def testFromExample(self):
  69. raw_data_in = [0b10001000, 0b11000110, 0b11111010]
  70. encoded_bitstring = array.array('B', raw_data_in).tostring()
  71. fo = StringIO.StringIO(encoded_bitstring)
  72. count = 3 << 1
  73. res = parquet.encoding.read_bitpacked(fo, count, 3)
  74. self.assertEquals(range(8), res)
  75. class TestBitPackedDeprecated(unittest.TestCase):
  76. def testFromExample(self):
  77. encoded_bitstring = array.array(
  78. 'B', [0b00000101, 0b00111001, 0b01110111]).tostring()
  79. fo = StringIO.StringIO(encoded_bitstring)
  80. res = parquet.encoding.read_bitpacked_deprecated(fo, 3, 8, 3)
  81. self.assertEquals(range(8), res)
  82. class TestWidthFromMaxInt(unittest.TestCase):
  83. def testWidths(self):
  84. self.assertEquals(0, parquet.encoding.width_from_max_int(0))
  85. self.assertEquals(1, parquet.encoding.width_from_max_int(1))
  86. self.assertEquals(2, parquet.encoding.width_from_max_int(2))
  87. self.assertEquals(2, parquet.encoding.width_from_max_int(3))
  88. self.assertEquals(3, parquet.encoding.width_from_max_int(4))
  89. self.assertEquals(3, parquet.encoding.width_from_max_int(5))
  90. self.assertEquals(3, parquet.encoding.width_from_max_int(6))
  91. self.assertEquals(3, parquet.encoding.width_from_max_int(7))
  92. self.assertEquals(4, parquet.encoding.width_from_max_int(8))
  93. self.assertEquals(4, parquet.encoding.width_from_max_int(15))
  94. self.assertEquals(5, parquet.encoding.width_from_max_int(16))
  95. self.assertEquals(5, parquet.encoding.width_from_max_int(31))
  96. self.assertEquals(6, parquet.encoding.width_from_max_int(32))
  97. self.assertEquals(6, parquet.encoding.width_from_max_int(63))
  98. self.assertEquals(7, parquet.encoding.width_from_max_int(64))
  99. self.assertEquals(7, parquet.encoding.width_from_max_int(127))
  100. self.assertEquals(8, parquet.encoding.width_from_max_int(128))
  101. self.assertEquals(8, parquet.encoding.width_from_max_int(255))