|
|
@@ -771,8 +771,8 @@ def _read_avro(fhandle, path, offset, length, stats):
|
|
|
|
|
|
def _read_parquet(fhandle, path, offset, length, stats):
|
|
|
try:
|
|
|
- size = 1 * 1024 * 1024 # Limit readable file, ParquetOptions.limit does not help
|
|
|
- data = StringIO(fhandle.read(size)) # There is a footer
|
|
|
+ size = 1 * 128 * 1024 * 1024 # Buffer file stream to 128 MB chunks
|
|
|
+ data = StringIO(fhandle.read(size))
|
|
|
|
|
|
dumped_data = StringIO()
|
|
|
parquet._dump(data, ParquetOptions(limit=1000), out=dumped_data)
|