Bläddra i källkod

HUE-1324 [fb] Support upload of tar.gz archive

Abraham Elmahrek 11 år sedan
förälder
incheckning
59c54f9

+ 69 - 10
apps/filebrowser/src/filebrowser/lib/archives.py

@@ -19,6 +19,7 @@
 
 import os
 import posixpath
+import tarfile
 import tempfile
 from zipfile import ZipFile
 
@@ -39,6 +40,17 @@ class Archive(object):
     """
     raise NotImplemented(_("Must implement 'extract' method."))
 
+  def _create_dirs(self, basepath, dirs=[]):
+    """
+    Creates all directories passed at the given basepath.
+    """
+    for directory in dirs:
+      directory = os.path.join(basepath, directory)
+      try:
+        os.makedirs(directory)
+      except OSError:
+        pass
+
 class ZipArchive(Archive):
   """
   Acts on a zip file in memory or in a temporary location.
@@ -76,16 +88,60 @@ class ZipArchive(Archive):
         files.append(name)
     return (dirs, files)
 
-  def _create_dirs(self, basepath, dirs=[]):
+  def _create_files(self, basepath, files=[]):
     """
-    Creates all directories passed at the given basepath.
+    Extract files to their rightful place.
+    Files are written to a temporary directory immediately after being decompressed.
     """
-    for directory in dirs:
-      directory = os.path.join(basepath, directory)
-      try:
-        os.makedirs(directory)
-      except OSError:
-        pass
+    for f in files:
+      new_path = os.path.join(basepath, f)
+      new_file = open(new_path, 'w')
+      new_file.write(self.zfh.read(f))
+      new_file.close()
+
+
+class TarballArchive(Archive):
+  """
+  Acts on a tarball (tar.gz) file in memory or in a temporary location.
+  Python's ZipFile class inherently buffers all reading.
+  """
+  def __init__(self, file):
+    if isinstance(file, basestring):
+      self.path = file
+    else:
+      f = tempfile.NamedTemporaryFile(delete=False)
+      f.write(file.read())
+      self.path = f.name
+      f.close()
+    self.fh = tarfile.open(self.path)
+
+  def extract(self):
+    """
+    Extracts a zip file.
+    If a 'file' ends with '/', then it is a directory and we must create it.
+    Else, open a file for writing and meta pipe the contents zipfile to the new file.
+    """
+    # Store all extracted files in a temporary directory.
+    directory = tempfile.mkdtemp()
+
+    dirs, files = self._filenames()
+    self._create_dirs(directory, dirs)
+    self._create_files(directory, files)
+
+    return directory
+
+  def _filenames(self):
+    """
+    List all dirs and files by reading the table of contents of the Zipfile.
+    """
+    dirs = []
+    files = []
+    for tarinfo in self.fh.getmembers():
+      if tarinfo.isdir():
+        dirs.append(tarinfo.name)
+      else:
+        files.append(tarinfo.name)
+    return (dirs, files)
 
   def _create_files(self, basepath, files=[]):
     """
@@ -95,9 +151,12 @@ class ZipArchive(Archive):
     for f in files:
       new_path = os.path.join(basepath, f)
       new_file = open(new_path, 'w')
-      new_file.write(self.zfh.read(f))
+      new_file.write(self.fh.extractfile(f).read())
       new_file.close()
 
 
 def archive_factory(path, archive_type='zip'):
-  return ZipArchive(path)
+  if archive_type == 'zip':
+    return ZipArchive(path)
+  elif archive_type == 'tarball' or archive_type == 'tar.gz' or archive_type == 'tgz':
+    return TarballArchive(path)

+ 11 - 0
apps/filebrowser/src/filebrowser/lib/archives_test.py

@@ -36,6 +36,17 @@ class ArchiveTest(unittest.TestCase):
     assert_true(os.path.isfile(directory + '/test.txt'))
     assert_equal(os.path.getsize(directory + '/test.txt'), 4)
 
+  def test_tgz(self):
+    FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.tar.gz')
+
+    # Extract the file
+    # This file should only have 'test.txt' in it
+    directory = archives.archive_factory(FILE, 'tgz').extract()
+    assert_true(os.path.exists(directory))
+    assert_true(os.path.isdir(directory))
+    assert_true(os.path.isfile(directory + '/test.txt'))
+    assert_equal(os.path.getsize(directory + '/test.txt'), 4)
+
 
 if __name__ == "__main__":
   unittest.main()

+ 1 - 1
apps/filebrowser/src/filebrowser/templates/listdir.mako

@@ -85,7 +85,7 @@ ${ fb_components.menubar() }
               </a>
               <ul class="dropdown-menu">
                 <li><a href="#" class="upload-link" title="${_('Files')}" data-bind="click: uploadFile"><i class="fa fa-file-o"></i> ${_('Files')}</a></li>
-                <li><a href="#" class="upload-link" title="${_('Archive')}" data-bind="click: uploadArchive"><i class="fa fa-gift"></i> ${_('Zip file')}</a></li>
+                <li><a href="#" class="upload-link" title="${_('Archive')}" data-bind="click: uploadArchive"><i class="fa fa-gift"></i> ${_('Zip/Tgz file')}</a></li>
               </ul>
             </div>
             <div class="btn-group" style="vertical-align: middle">

BIN
apps/filebrowser/src/filebrowser/test_data/test.tar.gz


+ 13 - 5
apps/filebrowser/src/filebrowser/views.py

@@ -1195,19 +1195,27 @@ def _upload_archive(request):
         dest = request.fs.join(form.cleaned_data['dest'], uploaded_file.name)
         try:
             # Extract if necessary
-            # Make sure dest path is without '.zip' extension
+            # Make sure dest path is without the extension
             if dest.endswith('.zip'):
-                temp_path = archive_factory(uploaded_file).extract()
+                temp_path = archive_factory(uploaded_file, 'zip').extract()
                 if not temp_path:
                     raise PopupException(_('Could not extract contents of file.'))
                 # Move the file to where it belongs
                 dest = dest[:-4]
-                request.fs.copyFromLocal(temp_path, dest)
-                shutil.rmtree(temp_path)
-                response['status'] = 0
+            elif dest.endswith('.tar.gz'):
+                print uploaded_file
+                temp_path = archive_factory(uploaded_file, 'tgz').extract()
+                if not temp_path:
+                    raise PopupException(_('Could not extract contents of file.'))
+                # Move the file to where it belongs
+                dest = dest[:-7]
             else:
                 raise PopupException(_('Could not interpret archive type.'))
 
+            request.fs.copyFromLocal(temp_path, dest)
+            shutil.rmtree(temp_path)
+            response['status'] = 0
+
         except IOError, ex:
             already_exists = False
             try:

+ 43 - 2
apps/filebrowser/src/filebrowser/views_test.py

@@ -1079,7 +1079,7 @@ def test_upload_file():
       pass
 
 @attr('requires_hadoop')
-def test_upload_archive():
+def test_upload_zip():
   """Test archive upload"""
   cluster = pseudo_hdfs4.shared_cluster()
 
@@ -1115,7 +1115,48 @@ def test_upload_archive():
   finally:
     try:
       cluster.fs.remove(HDFS_DEST_DIR)
-    except Exception, ex:
+    except:
+      pass
+
+@attr('requires_hadoop')
+def test_upload_tgz():
+  """Test archive upload"""
+  cluster = pseudo_hdfs4.shared_cluster()
+
+  try:
+    USER_NAME = 'test'
+    HDFS_DEST_DIR = "/tmp/fb-upload-test"
+    TGZ_FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.tar.gz')
+    HDFS_TGZ_FILE = HDFS_DEST_DIR + '/test.tar.gz'
+    HDFS_DECOMPRESSED_FILE = HDFS_DEST_DIR + '/test'
+
+    cluster.fs.setuser(USER_NAME)
+    client = make_logged_in_client(USER_NAME)
+
+    cluster.fs.mkdir(HDFS_DEST_DIR)
+    cluster.fs.chown(HDFS_DEST_DIR, USER_NAME)
+    cluster.fs.chmod(HDFS_DEST_DIR, 0700)
+
+    # Upload and decompress archive
+    resp = client.post('/filebrowser/upload/archive?dest=%s' % HDFS_DEST_DIR,
+                       dict(dest=HDFS_DEST_DIR, archive=file(TGZ_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_false(cluster.fs.exists(HDFS_TGZ_FILE))
+    assert_true(cluster.fs.isdir(HDFS_DECOMPRESSED_FILE))
+    assert_true(cluster.fs.isfile(HDFS_DECOMPRESSED_FILE + '/test.txt'))
+    assert_equal(cluster.fs.read(HDFS_DECOMPRESSED_FILE + '/test.txt', 0, 4), "test")
+
+    # Upload archive
+    resp = client.post('/filebrowser/upload/file?dest=%s' % HDFS_DEST_DIR,
+                       dict(dest=HDFS_DEST_DIR, hdfs_file=file(TGZ_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_true(cluster.fs.exists(HDFS_TGZ_FILE))
+  finally:
+    try:
+      cluster.fs.remove(HDFS_DEST_DIR)
+    except:
       pass
 
 def test_location_to_url():