فهرست منبع

HUE-3164 [fb] bzip2 files cannot be viewed properly in HUE file browser

Adrian Yavorskyy 8 سال پیش
والد
کامیت
39c9c1d

+ 5 - 1
apps/filebrowser/src/filebrowser/templates/display.mako

@@ -56,6 +56,10 @@ ${ fb_components.menubar() }
               <li><a class="pointer" data-bind="click: function(){ switchCompression('gzip'); }"><i class="fa fa-youtube-play"></i> ${_('Preview as Gzip')}</a></li>
             <!-- /ko -->
 
+            <!-- ko if: $root.file().view.compression() !== "bz2" && ($root.file().path().toLowerCase().endsWith('.bz2') || $root.file().path().toLowerCase().endsWith('.bzip2'))-->
+              <li><a class="pointer" data-bind="click: function(){ switchCompression('bz2'); }"><i class="fa fa-youtube-play"></i> ${_('Preview as Bzip2')}</a></li>
+            <!-- /ko -->
+
             <!-- ko if: $root.file().view.compression() !== "avro" && $root.file().view.compression() !== "snappy_avro" && $root.file().path().toLowerCase().endsWith('.avro') -->
               <li><a class="pointer" data-bind="click: function(){ switchCompression('avro'); }"><i class="fa fa-youtube-play"></i> ${_('Preview as Avro')}</a></li>
             <!-- /ko -->
@@ -135,7 +139,7 @@ ${ fb_components.menubar() }
                 <!-- ko if: $root.file().view.contents && $root.file().view.masked_binary_data() -->
                 <div class="alert alert-warning">${_("Warning: some binary data has been masked out with '&#xfffd'.")}</div>
                 <!-- /ko -->
-                <!-- ko if: ['avro', 'gzip', 'parquet', 'snappy'].indexOf($root.file().view.compression()) > -1 -->
+                <!-- ko if: ['avro', 'bz2', 'gzip', 'parquet', 'snappy'].indexOf($root.file().view.compression()) > -1 -->
                 <div class="alert alert-warning"><i class="fa fa-info-circle"></i> ${_('Output rendered from compressed %s file.') % view['compression']}</div>
                 <!-- /ko -->
               <!-- /ko -->

+ 20 - 0
apps/filebrowser/src/filebrowser/views.py

@@ -27,6 +27,7 @@ import shutil
 import stat as stat_module
 import urllib
 
+from bz2 import decompress
 from datetime import datetime
 from cStringIO import StringIO
 from gzip import GzipFile
@@ -687,6 +688,8 @@ def read_contents(codec_type, path, fs, offset, length):
             if path.endswith('.gz') and detect_gzip(contents):
                 codec_type = 'gzip'
                 offset = 0
+            elif (path.endswith('.bz2') or path.endswith('.bzip2')) and detect_bz2(contents):
+                codec_type = 'bz2'
             elif path.endswith('.avro') and detect_avro(contents):
                 codec_type = 'avro'
             elif detect_parquet(fhandle):
@@ -702,6 +705,8 @@ def read_contents(codec_type, path, fs, offset, length):
 
         if codec_type == 'gzip':
             contents = _read_gzip(fhandle, path, offset, length, stats)
+        elif codec_type == 'bz2':
+            contents = _read_bz2(fhandle, path, offset, length, stats)
         elif codec_type == 'avro':
             contents = _read_avro(fhandle, path, offset, length, stats)
         elif codec_type == 'parquet':
@@ -790,6 +795,16 @@ def _read_gzip(fhandle, path, offset, length, stats):
     return contents
 
 
+def _read_bz2(fhandle, path, offset, length, stats):
+    contents = ''
+    try:
+        contents = decompress(fhandle.read(length))
+    except Exception, e:
+        logging.exception('Could not decompress file at "%s": %s' % (path, e))
+        raise PopupException(_("Failed to decompress file."))
+    return contents
+
+
 def _read_simple(fhandle, path, offset, length, stats):
     contents = ''
     try:
@@ -806,6 +821,11 @@ def detect_gzip(contents):
     return contents[:2] == '\x1f\x8b'
 
 
+def detect_bz2(contents):
+    '''This is a silly small function which checks to see if the file is Bz2'''
+    return contents[:3] == 'BZh'
+
+
 def detect_avro(contents):
     '''This is a silly small function which checks to see if the file is Avro'''
     # Check if the first three bytes are 'O', 'b' and 'j'

+ 18 - 0
apps/filebrowser/src/filebrowser/views_test.py

@@ -687,6 +687,24 @@ class TestFileBrowserWithHadoop(object):
     assert_true('SR3_ndw_otlt_cmf_xref_INA' in response.context['view']['contents'], response.context['view']['contents'])
 
 
+  def test_view_bz2(self):
+    prefix = self.cluster.fs_prefix + '/test_view_bz2'
+    self.cluster.fs.mkdir(prefix)
+
+    # Bz2 file encoded as hex.
+    test_data = "425a6839314159265359338bcfac000001018002000c00200021981984185dc914e14240ce2f3eb0"
+
+    f = self.cluster.fs.open(prefix + '/test-view.bz2', "w")
+    f.write(test_data.decode('hex'))
+
+    # autodetect
+    response = self.c.get('/filebrowser/view=%s/test-view.bz2?compression=bz2' % prefix)
+    assert_true('test' in response.context['view']['contents'])
+
+    response = self.c.get('/filebrowser/view=%s/test-view.bz2' % prefix)
+    assert_true('test' in response.context['view']['contents'])
+
+
   def test_view_gz(self):
     prefix = self.cluster.fs_prefix + '/test_view_gz'
     self.cluster.fs.mkdir(prefix)