Browse Source

HUE-2602 [fb] Add bzip2 support when uploading archive

krish 10 years ago
parent
commit
0da36b95ad

+ 49 - 0
apps/filebrowser/src/filebrowser/lib/archives.py

@@ -17,6 +17,7 @@
 #
 # Utilities for dealing with file modes.
 
+import bz2
 import os
 import posixpath
 import tarfile
@@ -171,11 +172,59 @@ class TarballArchive(Archive):
       new_file.close()
 
 
+class BZ2Archive(Archive):
+  """
+  Acts on a bzip2 file in memory or in a temporary location.
+  Python's BZ2File class inherently buffers all reading.
+  """
+
+  def __init__(self, file):
+    # bzip2 only compresses single files and there is no direct method in the bz2 library to get the file name
+    self.name = file.name[:-6] if file.name.lower().endswith('.bzip2') else file.name[:-4]
+
+    if isinstance(file, basestring):
+      self.path = file
+    else:
+      f = tempfile.NamedTemporaryFile(delete=False)
+      f.write(file.read())
+      self.path = f.name
+      f.close()
+    self.fh = bz2.BZ2File(self.path)
+
+  def extract(self):
+    """
+    Extracts a bz2 file.
+    Opens the file for writing and meta pipe the contents bz2file to the new file.
+    """
+    # Store all extracted files in a temporary directory.
+    if ARCHIVE_UPLOAD_TEMPDIR.get():
+      directory = tempfile.mkdtemp(dir=ARCHIVE_UPLOAD_TEMPDIR.get())
+    else:
+      directory = tempfile.mkdtemp()
+
+    files = [self.name]
+    self._create_files(directory, files)
+
+    return directory
+
+  def _create_files(self, basepath, files=[]):
+    """
+    Files are written to a temporary directory immediately after being decompressed.
+    """
+    for f in files:
+      new_path = os.path.join(basepath, f)
+      new_file = open(new_path, 'w')
+      new_file.write(self.fh.read())
+      new_file.close()
+
+
 def archive_factory(path, archive_type='zip'):
   if archive_type == 'zip':
     return ZipArchive(path)
   elif archive_type == 'tarball' or archive_type == 'tar.gz' or archive_type == 'tgz':
     return TarballArchive(path)
+  elif archive_type == 'bz2' or archive_type == 'bzip2':
+    return BZ2Archive(path)
 
 class IllegalPathException(PopupException):
 

+ 1 - 1
apps/filebrowser/src/filebrowser/templates/listdir.mako

@@ -108,7 +108,7 @@ ${ fb_components.menubar() }
             </a>
             <ul class="dropdown-menu">
               <li><a href="#" class="upload-link" title="${_('Files')}" data-bind="click: uploadFile"><i class="fa fa-file-o"></i> ${_('Files')}</a></li>
-              <li><a href="#" class="upload-link" title="${_('Archive')}" data-bind="click: uploadArchive"><i class="fa fa-gift"></i> ${_('Zip/Tgz file')}</a></li>
+              <li><a href="#" class="upload-link" title="${_('Archive')}" data-bind="click: uploadArchive"><i class="fa fa-gift"></i> ${_('Zip/Tgz/Bz2 file')}</a></li>
             </ul>
           </div>
           <div class="btn-group" style="vertical-align: middle">

+ 1 - 1
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -1314,7 +1314,7 @@ from django.utils.translation import ugettext as _
           action:"/filebrowser/upload/archive",
           template:'<div class="qq-uploader">' +
                   '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
-                  '<div class="qq-upload-button">${_('Upload a zip file')}</div>' +
+                  '<div class="qq-upload-button">${_('Upload an Archive')}</div>' +
                   '<ul class="qq-upload-list"></ul>' +
                   '</div>',
           fileTemplate:'<li>' +

BIN
apps/filebrowser/src/filebrowser/test_data/test.txt.bz2


+ 7 - 1
apps/filebrowser/src/filebrowser/views.py

@@ -1245,7 +1245,13 @@ def _upload_archive(request):
                 if not temp_path:
                     raise PopupException(_('Could not extract contents of file.'))
                 # Move the file to where it belongs
-                dest = dest[:-7]
+                dest = dest[:-7] if dest.lower().endswith('.tar.gz') else dest[:-4]
+            elif dest.lower().endswith('.bz2') or dest.lower().endswith('.bzip2'):
+              temp_path = archive_factory(uploaded_file, 'bz2').extract()
+              if not temp_path:
+                  raise PopupException(_('Could not extract contents of file.'))
+                # Move the file to where it belongs
+              dest = dest[:-6] if dest.lower().endswith('.bzip2') else dest[:-4]
             else:
                 raise PopupException(_('Could not interpret archive type.'))
 

+ 41 - 0
apps/filebrowser/src/filebrowser/views_test.py

@@ -1183,6 +1183,47 @@ def test_upload_tgz():
     except:
       pass
 
+@attr('requires_hadoop')
+def test_upload_bz2():
+  """Test archive upload"""
+  cluster = pseudo_hdfs4.shared_cluster()
+
+  try:
+    USER_NAME = 'test'
+    HDFS_DEST_DIR = "/tmp/fb-upload-test"
+    BZ2_FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.txt.bz2')
+    HDFS_BZ2_FILE = HDFS_DEST_DIR + '/test.txt.bz2'
+    HDFS_DECOMPRESSED_FILE = HDFS_DEST_DIR + '/test'
+
+    cluster.fs.setuser(USER_NAME)
+    client = make_logged_in_client(USER_NAME)
+
+    cluster.fs.mkdir(HDFS_DEST_DIR)
+    cluster.fs.chown(HDFS_DEST_DIR, USER_NAME)
+    cluster.fs.chmod(HDFS_DEST_DIR, 0700)
+
+    # Upload and decompress archive
+    resp = client.post('/filebrowser/upload/archive?dest=%s' % HDFS_DEST_DIR,
+                       dict(dest=HDFS_DEST_DIR, archive=file(BZ2_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_false(cluster.fs.exists(HDFS_BZ2_FILE))
+    assert_true(cluster.fs.isdir(HDFS_DECOMPRESSED_FILE))
+    assert_true(cluster.fs.isfile(HDFS_DECOMPRESSED_FILE + '/test.txt'))
+    assert_equal(cluster.fs.read(HDFS_DECOMPRESSED_FILE + '/test.txt', 0, 4), "test")
+
+    # Upload archive
+    resp = client.post('/filebrowser/upload/file?dest=%s' % HDFS_DEST_DIR,
+                       dict(dest=HDFS_DEST_DIR, hdfs_file=file(BZ2_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_true(cluster.fs.exists(HDFS_BZ2_FILE))
+  finally:
+    try:
+      cluster.fs.remove(HDFS_DEST_DIR)
+    except:
+      pass
+
 def test_location_to_url():
   assert_equal('/filebrowser/view/var/lib/hadoop-hdfs', location_to_url('/var/lib/hadoop-hdfs', False))
   assert_equal('/filebrowser/view/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs'))