Browse Source

HUE-577 File viewer should work with snappy-compressed Avro data files

Revert some of the original changes.
Abraham Elmahrek 11 years ago
parent
commit
4d7628c

+ 4 - 21
apps/filebrowser/src/filebrowser/views.py

@@ -612,20 +612,15 @@ def read_contents(codec_type, path, fs, offset, length):
         fhandle = fs.open(path)
         stats = fs.stats(path)
 
-        # Auto codec detection for [gzip, avro, snappy, snappy avro, none]
-        if codec_type == 'avro' and snappy_installed() and detect_snappy(fhandle.read()):
-            codec_type = 'snappy_avro'
-        elif not codec_type:
+        # Auto codec detection for [gzip, avro, snappy, none]
+        if not codec_type:
             contents = fhandle.read(3)
             codec_type = 'none'
             if path.endswith('.gz') and detect_gzip(contents):
                 codec_type = 'gzip'
                 offset = 0
-            elif path.endswith('.avro'):
-                if detect_avro(contents):
-                    codec_type = 'avro'
-                if snappy_installed() and stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
-                    codec_type = 'snappy_avro'
+            elif path.endswith('.avro') and detect_avro(contents):
+                codec_type = 'avro'
             elif snappy_installed() and path.endswith('.snappy'):
                 codec_type = 'snappy'
             elif snappy_installed() and stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
@@ -637,8 +632,6 @@ def read_contents(codec_type, path, fs, offset, length):
             contents = _read_gzip(fhandle, path, offset, length, stats)
         elif codec_type == 'avro':
             contents = _read_avro(fhandle, path, offset, length, stats)
-        elif codec_type == 'snappy_avro':
-            contents = _read_snappy_avro(fhandle, path, offset, length, stats)
         elif codec_type == 'snappy':
             contents = _read_snappy(fhandle, path, offset, length, stats)
         else:
@@ -669,16 +662,6 @@ def _read_snappy(fhandle, path, offset, length, stats):
     return _read_simple(StringIO(_decompress_snappy(fhandle.read())), path, offset, length, stats)
 
 
-def _read_snappy_avro(fhandle, path, offset, length, stats):
-    if not snappy_installed():
-        raise PopupException(_('Failed to decompress snappy compressed file. Snappy is not installed.'))
-
-    if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
-        raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
-
-    return _read_avro(StringIO(_decompress_snappy(fhandle.read())), path, offset, length, stats)
-
-
 def _read_avro(fhandle, path, offset, length, stats):
     contents = ''
     try:

+ 52 - 69
apps/filebrowser/src/filebrowser/views_test.py

@@ -582,7 +582,7 @@ def test_chooser():
 
 
 @attr('requires_hadoop')
-def test_view_snappy_compressed_avro():
+def test_view_snappy_compressed():
   if not snappy_installed():
     raise SkipTest
   import snappy
@@ -592,60 +592,40 @@ def test_view_snappy_compressed_avro():
   try:
     c = make_logged_in_client()
     cluster.fs.setuser(cluster.superuser)
-    if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
-      cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
+    if cluster.fs.isdir('/tmp/test-snappy-filebrowser'):
+      cluster.fs.rmtree('/tmp/test-snappy-filebrowser')
 
-    cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
+    cluster.fs.mkdir('/tmp/test-snappy-avro-filebrowser/')
 
-    test_schema = schema.parse("""
-      {
-        "name": "test",
-        "type": "record",
-        "fields": [
-          { "name": "name", "type": "string" },
-          { "name": "integer", "type": "int" }
-        ]
-      }
-    """)
+    f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.snappy', "w")
+    f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
+    f.close()
 
-    # Cannot use StringIO with datafile writer!
-    f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', "w")
-    data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
-                                                writers_schema=test_schema,
-                                                codec='deflate')
-    dummy_datum = {
-      'name': 'Test',
-      'integer': 10,
-    }
-    data_file_writer.append(dummy_datum)
-    data_file_writer.close()
+    f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.stillsnappy', "w")
+    f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
+    f.close()
 
-    fh = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', 'r')
-    f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
-    f.write(snappy.compress(fh.read()))
+    f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.notsnappy', "w")
+    f.write('foobar')
     f.close()
-    fh.close()
 
     # Snappy compressed fail
-    response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
+    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.notsnappy?compression=snappy')
     assert_true('Failed to decompress' in response.context['message'], response)
 
     # Snappy compressed succeed
-    response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
-    assert_equal('snappy_avro', response.context['view']['compression'])
-    assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
-    response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=snappy_avro')
-    assert_equal('snappy_avro', response.context['view']['compression'])
-    assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
+    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.snappy')
+    assert_equal('snappy', response.context['view']['compression'])
+    assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
 
-    # Avro should also decompress snappy
-    response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=avro')
-    assert_equal('snappy_avro', response.context['view']['compression'])
-    assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
+    # Snappy compressed succeed
+    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy')
+    assert_equal('snappy', response.context['view']['compression'])
+    assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
 
     # Largest snappy compressed file
     finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
-    response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
+    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy?compression=snappy')
     assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
 
   finally:
@@ -658,7 +638,7 @@ def test_view_snappy_compressed_avro():
 
 
 @attr('requires_hadoop')
-def test_view_snappy_compressed():
+def test_view_snappy_compressed_avro():
   if not snappy_installed():
     raise SkipTest
   import snappy
@@ -668,41 +648,44 @@ def test_view_snappy_compressed():
   try:
     c = make_logged_in_client()
     cluster.fs.setuser(cluster.superuser)
-    if cluster.fs.isdir('/tmp/test-snappy-filebrowser'):
-      cluster.fs.rmtree('/tmp/test-snappy-filebrowser')
+    if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
+      cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
 
-    cluster.fs.mkdir('/tmp/test-snappy-avro-filebrowser/')
+    cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
 
-    f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.snappy', "w")
-    f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
-    f.close()
+    test_schema = schema.parse("""
+      {
+        "name": "test",
+        "type": "record",
+        "fields": [
+          { "name": "name", "type": "string" },
+          { "name": "integer", "type": "int" }
+        ]
+      }
+    """)
 
-    f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.stillsnappy', "w")
-    f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
+    # Cannot use StringIO with datafile writer!
+    f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
+    data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
+                                                writers_schema=test_schema,
+                                                codec='snappy')
+    dummy_datum = {
+      'name': 'Test',
+      'integer': 10,
+    }
+    data_file_writer.append(dummy_datum)
+    data_file_writer.close()
     f.close()
 
-    f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.notsnappy', "w")
-    f.write('foobar')
+    # Check to see if snappy is the codec
+    f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "r")
+    assert_true('snappy' in f.read())
     f.close()
 
-    # Snappy compressed fail
-    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.notsnappy?compression=snappy')
-    assert_true('Failed to decompress' in response.context['message'], response)
-
-    # Snappy compressed succeed
-    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.snappy')
-    assert_equal('snappy', response.context['view']['compression'])
-    assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
-
     # Snappy compressed succeed
-    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy')
-    assert_equal('snappy', response.context['view']['compression'])
-    assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
-
-    # Largest snappy compressed file
-    finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
-    response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy?compression=snappy')
-    assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
+    response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
+    assert_equal('avro', response.context['view']['compression'])
+    assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
 
   finally:
     for done in finish: