|
|
@@ -2,7 +2,7 @@ import gzip
|
|
|
import json
|
|
|
import logging
|
|
|
import struct
|
|
|
-import StringIO
|
|
|
+import cStringIO
|
|
|
import sys
|
|
|
from collections import defaultdict
|
|
|
from ttypes import (FileMetaData, CompressionCodec, Encoding,
|
|
|
@@ -204,7 +204,7 @@ def _read_page(fo, page_header, column_metadata):
|
|
|
if column_metadata.codec == CompressionCodec.SNAPPY:
|
|
|
raw_bytes = snappy.decompress(bytes_from_file)
|
|
|
elif column_metadata.codec == CompressionCodec.GZIP:
|
|
|
- io_obj = StringIO.StringIO(bytes_from_file)
|
|
|
+ io_obj = cStringIO.StringIO(bytes_from_file)
|
|
|
with gzip.GzipFile(fileobj=io_obj, mode='rb') as f:
|
|
|
raw_bytes = f.read()
|
|
|
else:
|
|
|
@@ -252,7 +252,7 @@ def read_data_page(fo, schema_helper, page_header, column_metadata,
|
|
|
"""
|
|
|
daph = page_header.data_page_header
|
|
|
raw_bytes = _read_page(fo, page_header, column_metadata)
|
|
|
- io_obj = StringIO.StringIO(raw_bytes)
|
|
|
+ io_obj = cStringIO.StringIO(raw_bytes)
|
|
|
vals = []
|
|
|
|
|
|
logger.debug(" definition_level_encoding: %s",
|
|
|
@@ -301,11 +301,13 @@ def read_data_page(fo, schema_helper, page_header, column_metadata,
|
|
|
logger.debug("bit_width: %d", bit_width)
|
|
|
total_seen = 0
|
|
|
dict_values_bytes = io_obj.read()
|
|
|
- dict_values_io_obj = StringIO.StringIO(dict_values_bytes)
|
|
|
+ dict_values_io_obj = cStringIO.StringIO(dict_values_bytes)
|
|
|
# TODO jcrobak -- not sure that this loop is needed?
|
|
|
while total_seen < daph.num_values:
|
|
|
values = encoding.read_rle_bit_packed_hybrid(
|
|
|
dict_values_io_obj, bit_width, len(dict_values_bytes))
|
|
|
+ if len(values) + total_seen > daph.num_values:
|
|
|
+ values = values[0: daph.num_values - total_seen]
|
|
|
vals += [dictionary[v] for v in values]
|
|
|
total_seen += len(values)
|
|
|
else:
|
|
|
@@ -316,7 +318,7 @@ def read_data_page(fo, schema_helper, page_header, column_metadata,
|
|
|
|
|
|
def read_dictionary_page(fo, page_header, column_metadata):
|
|
|
raw_bytes = _read_page(fo, page_header, column_metadata)
|
|
|
- io_obj = StringIO.StringIO(raw_bytes)
|
|
|
+ io_obj = cStringIO.StringIO(raw_bytes)
|
|
|
dict_items = []
|
|
|
while io_obj.tell() < len(raw_bytes):
|
|
|
# TODO - length for fixed byte array
|