|
|
@@ -688,10 +688,11 @@ def read_contents(codec_type, path, fs, offset, length):
|
|
|
fhandle = fs.open(path)
|
|
|
stats = fs.stats(path)
|
|
|
|
|
|
- # Auto codec detection for [gzip, avro, none]
|
|
|
- # Only done when codec_type is unset
|
|
|
- contents = fhandle.read(3)
|
|
|
- if not codec_type:
|
|
|
+ # Auto codec detection for [gzip, avro, snappy, snappy avro, none]
|
|
|
+ if codec_type == 'avro' and snappy_installed() and detect_snappy(fhandle.read()):
|
|
|
+ codec_type = 'snappy_avro'
|
|
|
+ elif not codec_type:
|
|
|
+ contents = fhandle.read(3)
|
|
|
codec_type = 'none'
|
|
|
if path.endswith('.gz') and detect_gzip(contents):
|
|
|
codec_type = 'gzip'
|
|
|
@@ -699,24 +700,23 @@ def read_contents(codec_type, path, fs, offset, length):
|
|
|
elif path.endswith('.avro'):
|
|
|
if detect_avro(contents):
|
|
|
codec_type = 'avro'
|
|
|
- elif snappy_installed():
|
|
|
- if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
|
|
|
- raise PopupException(_('Failed to validate snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
|
|
|
+ if snappy_installed() and stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
|
|
|
+ codec_type = 'snappy_avro'
|
|
|
+ elif snappy_installed() and path.endswith('.snappy'):
|
|
|
+ codec_type = 'snappy'
|
|
|
+ elif snappy_installed() and stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
|
|
|
+ codec_type = 'snappy'
|
|
|
|
|
|
- if detect_snappy(contents + fhandle.read()):
|
|
|
- codec_type = 'snappy_avro'
|
|
|
fhandle.seek(0)
|
|
|
|
|
|
- if codec_type == 'avro' and snappy_installed() and detect_snappy(fhandle.read()):
|
|
|
- fhandle.seek(0)
|
|
|
- codec_type = 'snappy_avro'
|
|
|
-
|
|
|
if codec_type == 'gzip':
|
|
|
contents = _read_gzip(fhandle, path, offset, length, stats)
|
|
|
elif codec_type == 'avro':
|
|
|
contents = _read_avro(fhandle, path, offset, length, stats)
|
|
|
elif codec_type == 'snappy_avro':
|
|
|
contents = _read_snappy_avro(fhandle, path, offset, length, stats)
|
|
|
+ elif codec_type == 'snappy':
|
|
|
+ contents = _read_snappy(fhandle, path, offset, length, stats)
|
|
|
else:
|
|
|
# for 'none' type.
|
|
|
contents = _read_simple(fhandle, path, offset, length, stats)
|
|
|
@@ -735,6 +735,16 @@ def _decompress_snappy(compressed_content):
|
|
|
raise PopupException(_('Failed to decompress snappy compressed file.'), detail=e)
|
|
|
|
|
|
|
|
|
+def _read_snappy(fhandle, path, offset, length, stats):
|
|
|
+ if not snappy_installed():
|
|
|
+ raise PopupException(_('Failed to decompress snappy compressed file. Snappy is not installed.'))
|
|
|
+
|
|
|
+ if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
|
|
|
+ raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
|
|
|
+
|
|
|
+ return _read_simple(StringIO(_decompress_snappy(fhandle.read())), path, offset, length, stats)
|
|
|
+
|
|
|
+
|
|
|
def _read_snappy_avro(fhandle, path, offset, length, stats):
|
|
|
if not snappy_installed():
|
|
|
raise PopupException(_('Failed to decompress snappy compressed file. Snappy is not installed.'))
|