|
@@ -582,7 +582,7 @@ def test_chooser():
|
|
|
|
|
|
|
|
|
|
|
|
|
@attr('requires_hadoop')
|
|
@attr('requires_hadoop')
|
|
|
-def test_view_snappy_compressed_avro():
|
|
|
|
|
|
|
+def test_view_snappy_compressed():
|
|
|
if not snappy_installed():
|
|
if not snappy_installed():
|
|
|
raise SkipTest
|
|
raise SkipTest
|
|
|
import snappy
|
|
import snappy
|
|
@@ -592,60 +592,40 @@ def test_view_snappy_compressed_avro():
|
|
|
try:
|
|
try:
|
|
|
c = make_logged_in_client()
|
|
c = make_logged_in_client()
|
|
|
cluster.fs.setuser(cluster.superuser)
|
|
cluster.fs.setuser(cluster.superuser)
|
|
|
- if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
|
|
|
|
|
- cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
|
|
|
|
|
|
|
+ if cluster.fs.isdir('/tmp/test-snappy-filebrowser'):
|
|
|
|
|
+ cluster.fs.rmtree('/tmp/test-snappy-filebrowser')
|
|
|
|
|
|
|
|
- cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
|
|
|
|
|
|
|
+ cluster.fs.mkdir('/tmp/test-snappy-avro-filebrowser/')
|
|
|
|
|
|
|
|
- test_schema = schema.parse("""
|
|
|
|
|
- {
|
|
|
|
|
- "name": "test",
|
|
|
|
|
- "type": "record",
|
|
|
|
|
- "fields": [
|
|
|
|
|
- { "name": "name", "type": "string" },
|
|
|
|
|
- { "name": "integer", "type": "int" }
|
|
|
|
|
- ]
|
|
|
|
|
- }
|
|
|
|
|
- """)
|
|
|
|
|
|
|
+ f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.snappy', "w")
|
|
|
|
|
+ f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
|
|
|
|
|
+ f.close()
|
|
|
|
|
|
|
|
- # Cannot use StringIO with datafile writer!
|
|
|
|
|
- f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', "w")
|
|
|
|
|
- data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
|
|
|
|
|
- writers_schema=test_schema,
|
|
|
|
|
- codec='deflate')
|
|
|
|
|
- dummy_datum = {
|
|
|
|
|
- 'name': 'Test',
|
|
|
|
|
- 'integer': 10,
|
|
|
|
|
- }
|
|
|
|
|
- data_file_writer.append(dummy_datum)
|
|
|
|
|
- data_file_writer.close()
|
|
|
|
|
|
|
+ f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.stillsnappy', "w")
|
|
|
|
|
+ f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
|
|
|
|
|
+ f.close()
|
|
|
|
|
|
|
|
- fh = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', 'r')
|
|
|
|
|
- f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
|
|
|
|
|
- f.write(snappy.compress(fh.read()))
|
|
|
|
|
|
|
+ f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.notsnappy', "w")
|
|
|
|
|
+ f.write('foobar')
|
|
|
f.close()
|
|
f.close()
|
|
|
- fh.close()
|
|
|
|
|
|
|
|
|
|
# Snappy compressed fail
|
|
# Snappy compressed fail
|
|
|
- response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
|
|
|
|
|
|
|
+ response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.notsnappy?compression=snappy')
|
|
|
assert_true('Failed to decompress' in response.context['message'], response)
|
|
assert_true('Failed to decompress' in response.context['message'], response)
|
|
|
|
|
|
|
|
# Snappy compressed succeed
|
|
# Snappy compressed succeed
|
|
|
- response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
|
|
|
|
|
- assert_equal('snappy_avro', response.context['view']['compression'])
|
|
|
|
|
- assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
|
|
|
|
|
- response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=snappy_avro')
|
|
|
|
|
- assert_equal('snappy_avro', response.context['view']['compression'])
|
|
|
|
|
- assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
|
|
|
|
|
|
|
+ response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.snappy')
|
|
|
|
|
+ assert_equal('snappy', response.context['view']['compression'])
|
|
|
|
|
+ assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
|
|
|
|
|
|
|
|
- # Avro should also decompress snappy
|
|
|
|
|
- response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=avro')
|
|
|
|
|
- assert_equal('snappy_avro', response.context['view']['compression'])
|
|
|
|
|
- assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
|
|
|
|
|
|
|
+ # Snappy compressed succeed
|
|
|
|
|
+ response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy')
|
|
|
|
|
+ assert_equal('snappy', response.context['view']['compression'])
|
|
|
|
|
+ assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
|
|
|
|
|
|
|
|
# Largest snappy compressed file
|
|
# Largest snappy compressed file
|
|
|
finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
|
|
finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
|
|
|
- response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
|
|
|
|
|
|
|
+ response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy?compression=snappy')
|
|
|
assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
|
|
assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
|
|
|
|
|
|
|
|
finally:
|
|
finally:
|
|
@@ -658,7 +638,7 @@ def test_view_snappy_compressed_avro():
|
|
|
|
|
|
|
|
|
|
|
|
|
@attr('requires_hadoop')
|
|
@attr('requires_hadoop')
|
|
|
-def test_view_snappy_compressed():
|
|
|
|
|
|
|
+def test_view_snappy_compressed_avro():
|
|
|
if not snappy_installed():
|
|
if not snappy_installed():
|
|
|
raise SkipTest
|
|
raise SkipTest
|
|
|
import snappy
|
|
import snappy
|
|
@@ -668,41 +648,44 @@ def test_view_snappy_compressed():
|
|
|
try:
|
|
try:
|
|
|
c = make_logged_in_client()
|
|
c = make_logged_in_client()
|
|
|
cluster.fs.setuser(cluster.superuser)
|
|
cluster.fs.setuser(cluster.superuser)
|
|
|
- if cluster.fs.isdir('/tmp/test-snappy-filebrowser'):
|
|
|
|
|
- cluster.fs.rmtree('/tmp/test-snappy-filebrowser')
|
|
|
|
|
|
|
+ if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
|
|
|
|
|
+ cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
|
|
|
|
|
|
|
|
- cluster.fs.mkdir('/tmp/test-snappy-avro-filebrowser/')
|
|
|
|
|
|
|
+ cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
|
|
|
|
|
|
|
|
- f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.snappy', "w")
|
|
|
|
|
- f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
|
|
|
|
|
- f.close()
|
|
|
|
|
|
|
+ test_schema = schema.parse("""
|
|
|
|
|
+ {
|
|
|
|
|
+ "name": "test",
|
|
|
|
|
+ "type": "record",
|
|
|
|
|
+ "fields": [
|
|
|
|
|
+ { "name": "name", "type": "string" },
|
|
|
|
|
+ { "name": "integer", "type": "int" }
|
|
|
|
|
+ ]
|
|
|
|
|
+ }
|
|
|
|
|
+ """)
|
|
|
|
|
|
|
|
- f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.stillsnappy', "w")
|
|
|
|
|
- f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
|
|
|
|
|
|
|
+ # Cannot use StringIO with datafile writer!
|
|
|
|
|
+ f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
|
|
|
|
|
+ data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
|
|
|
|
|
+ writers_schema=test_schema,
|
|
|
|
|
+ codec='snappy')
|
|
|
|
|
+ dummy_datum = {
|
|
|
|
|
+ 'name': 'Test',
|
|
|
|
|
+ 'integer': 10,
|
|
|
|
|
+ }
|
|
|
|
|
+ data_file_writer.append(dummy_datum)
|
|
|
|
|
+ data_file_writer.close()
|
|
|
f.close()
|
|
f.close()
|
|
|
|
|
|
|
|
- f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.notsnappy', "w")
|
|
|
|
|
- f.write('foobar')
|
|
|
|
|
|
|
+ # Check to see if snappy is the codec
|
|
|
|
|
+ f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "r")
|
|
|
|
|
+ assert_true('snappy' in f.read())
|
|
|
f.close()
|
|
f.close()
|
|
|
|
|
|
|
|
- # Snappy compressed fail
|
|
|
|
|
- response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.notsnappy?compression=snappy')
|
|
|
|
|
- assert_true('Failed to decompress' in response.context['message'], response)
|
|
|
|
|
-
|
|
|
|
|
- # Snappy compressed succeed
|
|
|
|
|
- response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.snappy')
|
|
|
|
|
- assert_equal('snappy', response.context['view']['compression'])
|
|
|
|
|
- assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
|
|
|
|
|
-
|
|
|
|
|
# Snappy compressed succeed
|
|
# Snappy compressed succeed
|
|
|
- response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy')
|
|
|
|
|
- assert_equal('snappy', response.context['view']['compression'])
|
|
|
|
|
- assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
|
|
|
|
|
-
|
|
|
|
|
- # Largest snappy compressed file
|
|
|
|
|
- finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
|
|
|
|
|
- response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy?compression=snappy')
|
|
|
|
|
- assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
|
|
|
|
|
|
|
+ response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
|
|
|
|
|
+ assert_equal('avro', response.context['view']['compression'])
|
|
|
|
|
+ assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
|
|
|
|
|
|
|
|
finally:
|
|
finally:
|
|
|
for done in finish:
|
|
for done in finish:
|