Sfoglia il codice sorgente

HUE-775 [fb] Upload zip file through filebrowser.

- Added test cases to validate unzipping
- Added upload scenarios to upload test case to ensure proper uploading and extraction
- Added a hackish solution for explicitly requiring zip extraction
- New 'uplaod archive' button.
abec 13 anni fa
parent
commit
be9f6467a2

+ 6 - 1
apps/filebrowser/src/filebrowser/forms.py

@@ -53,12 +53,17 @@ class RenameForm(forms.Form):
   src_path = CharField(label=_("File to rename"), help_text=_("The file to rename."))
   dest_path = CharField(label=_("New name"), help_text=_("Rename the file to:"))
 
-class UploadForm(forms.Form):
+class UploadFileForm(forms.Form):
   op = "upload"
   # The "hdfs" prefix in "hdfs_file" triggers the HDFSfileUploadHandler
   hdfs_file = FileField(forms.Form, label=_("File to Upload"))
   dest = PathField(label=_("Destination Path"), help_text=_("Filename or directory to upload to."))
 
+class UploadArchiveForm(forms.Form):
+  op = "upload"
+  archive = FileField(forms.Form, label=_("Archive to Upload"))
+  dest = PathField(label=_("Destination Path"), help_text=_("Archive to upload to."))
+
 class RemoveForm(forms.Form):
   op = "remove"
   path = PathField(label=_("File to remove"))

+ 102 - 0
apps/filebrowser/src/filebrowser/lib/archives.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Utilities for dealing with file modes.
+
+import os
+import posixpath
+import tempfile
+import uuid
+from zipfile import ZipFile
+
+from django.utils.translation import ugettext as _
+
+
+__all__ = ['archive_factory']
+
+
+class Archive(object):
+  """
+  Acrchive interface.
+  """
+  def extract(self, path):
+    """
+    Extract an Archive.
+    Should return a directory where the extracted contents live.
+    """
+    raise NotImplemented(_("Must implement 'extract' method."))
+
+class ZipArchive(Archive):
+  """
+  Acts on a zip file in memory or in a temporary location.
+  Python's ZipFile class inherently buffers all reading.
+  """
+  def __init__(self, file):
+    self.file = isinstance(file, basestring) and open(file) or file
+    self.zfh = ZipFile(self.file)
+
+  def extract(self):
+    """
+    Extracts a zip file.
+    If a 'file' ends with '/', then it is a directory and we must create it.
+    Else, open a file for writing and meta pipe the contents zipfile to the new file.
+    """
+    # Store all extracted files in a temporary directory.
+    directory = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
+    os.mkdir(directory)
+
+    dirs, files = self._filenames()
+    self._create_dirs(directory, dirs)
+    self._create_files(directory, files)
+
+    return directory
+
+  def _filenames(self):
+    """
+    List all dirs and files by reading the table of contents of the Zipfile.
+    """
+    dirs = []
+    files = []
+    for name in self.zfh.namelist():
+      if name.endswith(posixpath.sep):
+        dirs.append(name)
+      else:
+        files.append(name)
+    return (dirs, files)
+
+  def _create_dirs(self, basepath, dirs=[]):
+    """
+    Creates all directories passed at the given basepath.
+    """
+    for directory in dirs:
+      directory = os.path.join(basepath, directory)
+      os.mkdir(directory)
+
+  def _create_files(self, basepath, files=[]):
+    """
+    Extract files to their rightful place.
+    Files are written to a temporary directory immediately after being decompressed.
+    """
+    for f in files:
+      new_path = os.path.join(basepath, f)
+      new_file = open(new_path, 'w')
+      new_file.write(self.zfh.read(f))
+      new_file.close()
+
+
+def archive_factory(path, archive_type='zip'):
+  return ZipArchive(path)

+ 41 - 0
apps/filebrowser/src/filebrowser/lib/archives_test.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import os
+
+from nose.tools import assert_true, assert_equal
+
+import archives
+
+
+class ArchiveTest(unittest.TestCase):
+
+  def test_zip(self):
+    FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.zip')
+
+    # Extract the file
+    # This file should only have 'test.txt' in it
+    directory = archives.archive_factory(FILE, 'zip').extract()
+    assert_true(os.path.exists(directory))
+    assert_true(os.path.isdir(directory))
+    assert_true(os.path.isfile(directory + '/test.txt'))
+    assert_equal(os.path.getsize(directory + '/test.txt'), 4)
+
+
+if __name__ == "__main__":
+  unittest.main()

+ 8 - 1
apps/filebrowser/src/filebrowser/templates/listdir.mako

@@ -41,7 +41,14 @@ ${commonheader(_('File Browser'), 'filebrowser', user)}
             <button class="btn fileToolbarBtn" title="${_('Delete')}" data-bind="click: deleteSelected, enable: selectedFiles().length == 1"><i class="icon-trash"></i> ${_('Delete')}</button>
         </%def>
         <%def name="creation()">
-            <a href="#" class="btn upload-link" title="${_('Upload files')}"><i class="icon-upload"></i> ${_('Upload files')}</a>
+            <div id="upload-dropdown" class="btn-group" style="display: inline-block;">
+                <a href="#" class="btn upload-link dropdown-toggle" title="${_('Upload')}" data-toggle="dropdown"><i class="icon-upload"></i> ${_('Upload')}</a>
+                <ul class="dropdown-menu">
+                  <li><a href="#" tabindex="-1" class="upload-link" title="${_('Upload files')}" data-bind="click: uploadFile">${_('Upload files')}</a></li>
+                  <li class="divider"></li>
+                  <li><a href="#" tabindex="-1" class="upload-link" title="${_('Upload archive')}" data-bind="click: uploadArchive">${_('Upload archive')}</a></li>
+                </ul>
+            </div>
             <a href="#" class="btn create-directory-link" title="${_('New directory')}"><i class="icon-folder-close"></i> ${_('New directory')}</a>
         </%def>
     </%actionbar:render>

+ 120 - 56
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -166,14 +166,31 @@ from django.utils.translation import ugettext as _
 
     <div id="moveModal" class="modal hide fade"></div>
 
-    <!-- upload modal -->
-    <div id="uploadModal" class="modal hide fade">
+    <!-- upload file modal -->
+    <div id="uploadFileModal" class="modal hide fade">
         <div class="modal-header">
             <a href="#" class="close" data-dismiss="modal">&times;</a>
             <h3>${_('Uploading to:')} <span id="uploadDirName" data-bind="text: currentPath"></span></h3>
         </div>
-        <div class="modal-body">
-            <div id="fileUploader">
+        <div class="modal-body form-inline">
+            <div id="fileUploader" class="uploader">
+            <noscript>
+                <p>${_('Please enable JavaScript to use the file uploader.')}</p>
+            </noscript>
+            </div>
+        </div>
+        <div class="modal-footer"></div>
+    </div>
+
+    <!-- upload archive modal -->
+    <div id="uploadArchiveModal" class="modal hide fade">
+        <div class="modal-header">
+            <a href="#" class="close" data-dismiss="modal">&times;</a>
+            <h3>${_('Uploading to:')} <span id="uploadDirName" data-bind="text: currentPath"></span></h3>
+            <p>${_('The file will then be extracted in the path specified above.')}</p>
+        </div>
+        <div class="modal-body form-inline">
+            <div id="archiveUploader" class="uploader">
             <noscript>
                 <p>${_('Please enable JavaScript to use the file uploader.')}</p>
             </noscript>
@@ -259,46 +276,8 @@ from django.utils.translation import ugettext as _
             });
         }
 
-        //uploader
-        var num_of_pending_uploads = 0;
-        var uploader = null;
-        function createUploader(){
-            uploader = new qq.FileUploader({
-                element: document.getElementById("fileUploader"),
-                action: "/filebrowser/upload",
-                template: '<div class="qq-uploader">' +
-                        '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
-                        '<div class="qq-upload-button">${_('Upload a file')}</div>' +
-                        '<ul class="qq-upload-list"></ul>' +
-                        '</div>',
-                fileTemplate: '<li>' +
-                        '<span class="qq-upload-file"></span>' +
-                        '<span class="qq-upload-spinner"></span>' +
-                        '<span class="qq-upload-size"></span>' +
-                        '<a class="qq-upload-cancel" href="#">${_('Cancel')}</a>' +
-                        '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
-                        '</li>',
-                params:{
-                    dest: viewModel.currentPath(),
-                    fileFieldLabel: "hdfs_file"
-                },
-                onComplete:function(id, fileName, responseJSON){
-                    num_of_pending_uploads--;
-                    if(num_of_pending_uploads == 0){
-                        window.location = "/filebrowser/view" + viewModel.currentPath();
-                    }
-                },
-                onSubmit:function(id, fileName, responseJSON){
-                    num_of_pending_uploads++;
-                },
-                debug: false
-            });
-        }
-
         $(document).ready(function(){
 
-            createUploader();
-
             $("#cancelDeleteBtn").click(function(){
                 $("#deleteModal").modal("hide");
             });
@@ -334,14 +313,6 @@ from django.utils.translation import ugettext as _
                 $("#moveForm").find("input[name='dest_path']").removeClass("fieldError");
             });
 
-            //upload handlers
-            $(".upload-link").click(function(){
-                $("#uploadModal").modal({
-                    keyboard: true,
-                    show: true
-                });
-            });
-
             //create directory handlers
             $(".create-directory-link").click(function(){
                 $("#createDirectoryModal").modal({
@@ -594,12 +565,9 @@ from django.utils.translation import ugettext as _
                     return new Breadcrumb(breadcrumb);
                 }));
                 self.currentPath(currentDirPath);
-                if (uploader != null){
-                    uploader.setParams({
-                        dest: self.currentPath(),
-                        fileFieldLabel: "hdfs_file"
-                    });
-                }
+
+                $('.uploader').trigger('fb:updatePath', {dest: self.currentPath()});
+ 
                 self.isLoading(false);
                 $(".scrollable").jHueTableScroller();
             };
@@ -700,6 +668,102 @@ from django.utils.translation import ugettext as _
                 $(formElement).attr("action", "/filebrowser/mkdir?next=${url('filebrowser.views.view', path=urlencode('/'))}"+ "." + self.currentPath());
                 return true;
             };
+
+            self.uploadFile = (function() {
+                var num_of_pending_uploads = 0;
+                var uploader = new qq.FileUploader({
+                    element: document.getElementById("fileUploader"),
+                    action: "/filebrowser/upload/file",
+                    template: '<div class="qq-uploader">' +
+                            '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
+                            '<div class="qq-upload-button">${_('Upload a file')}</div>' +
+                            '<ul class="qq-upload-list"></ul>' +
+                            '</div>',
+                    fileTemplate: '<li>' +
+                            '<span class="qq-upload-file"></span>' +
+                            '<span class="qq-upload-spinner"></span>' +
+                            '<span class="qq-upload-size"></span>' +
+                            '<a class="qq-upload-cancel" href="#">${_('Cancel')}</a>' +
+                            '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
+                            '</li>',
+                    params:{
+                        dest: self.currentPath(),
+                        fileFieldLabel: "hdfs_file"
+                    },
+                    onComplete:function(id, fileName, responseJSON){
+                        num_of_pending_uploads--;
+                        if(num_of_pending_uploads == 0){
+                            window.location = "/filebrowser/view" + self.currentPath();
+                        }
+                    },
+                    onSubmit:function(id, fileName, responseJSON){
+                        num_of_pending_uploads++;
+                    },
+                    debug: false
+                });
+
+                $("#archiveUploader").on('fb:updatePath', function(e, options) {
+                    uploader.setParams({
+                        dest: options.dest,
+                        fileFieldLabel: "hdfs_file"
+                    });
+                });
+
+                return function() {
+                    $("#uploadFileModal").modal({
+                        keyboard: true,
+                        show: true
+                    });
+                };
+            })();
+
+            self.uploadArchive = (function() {
+                var num_of_pending_uploads = 0;
+                var uploader = new qq.FileUploader({
+                    element: document.getElementById("archiveUploader"),
+                    action: "/filebrowser/upload/archive",
+                    template: '<div class="qq-uploader">' +
+                            '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
+                            '<div class="qq-upload-button">${_('Upload an archive')}</div>' +
+                            '<ul class="qq-upload-list"></ul>' +
+                            '</div>',
+                    fileTemplate: '<li>' +
+                            '<span class="qq-upload-file"></span>' +
+                            '<span class="qq-upload-spinner"></span>' +
+                            '<span class="qq-upload-size"></span>' +
+                            '<a class="qq-upload-cancel" href="#">${_('Cancel')}</a>' +
+                            '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
+                            '</li>',
+                    params:{
+                        dest: self.currentPath(),
+                        fileFieldLabel: "archive"
+                    },
+                    onComplete:function(id, fileName, responseJSON){
+                        num_of_pending_uploads--;
+                        if(num_of_pending_uploads == 0){
+                            window.location = "/filebrowser/view" + self.currentPath();
+                        }
+                    },
+                    onSubmit:function(id, fileName, responseJSON){
+                        num_of_pending_uploads++;
+                    },
+                    debug: false
+                });
+
+                $("#archiveUploader").on('fb:updatePath', function(e, options) {
+                    uploader.setParams({
+                        dest: options.dest,
+                        fileFieldLabel: "archive"
+                    });
+                });
+
+                return function() {
+                    $("#uploadArchiveModal").modal({
+                        keyboard: true,
+                        show: true
+                    });
+                };
+            })();
         };
 
         var viewModel = new FileBrowserModel([], null, [], "/");

BIN
apps/filebrowser/src/filebrowser/test_data/test.zip


+ 2 - 1
apps/filebrowser/src/filebrowser/urls.py

@@ -35,7 +35,8 @@ urlpatterns = patterns('filebrowser.views',
 
   # POST operations
   url(r'upload_flash$', 'upload_flash', name='upload_flash'),
-  url(r'upload$', 'upload', name='upload'),
+  url(r'upload/file$', 'upload_file', name='upload_file'),
+  url(r'upload/archive$', 'upload_archive', name='upload_archive'),
   url(r'rename', 'rename', name='rename'),
   url(r'mkdir', 'mkdir', name='mkdir'),
   url(r'^move', 'move', name='move'),

+ 91 - 9
apps/filebrowser/src/filebrowser/views.py

@@ -25,6 +25,7 @@ import logging
 import mimetypes
 import operator
 import posixpath
+import shutil
 import stat as stat_module
 import os
 from types import MethodType
@@ -50,10 +51,11 @@ from desktop.lib import i18n, paginator
 from desktop.lib.conf import coerce_bool
 from desktop.lib.django_util import make_absolute, render, render_json, format_preserving_redirect
 from desktop.lib.exceptions import PopupException
+from filebrowser.lib.archives import archive_factory
 from filebrowser.lib.rwx import filetype, rwx
 from filebrowser.lib import xxd
-from filebrowser.forms import RenameForm, UploadForm, MkDirForm, RmDirForm, RmTreeForm,\
-    RemoveForm, ChmodForm, ChownForm, EditorForm
+from filebrowser.forms import RenameForm, UploadFileForm, UploadArchiveForm, MkDirForm,\
+    RmDirForm, RmTreeForm, RemoveForm, ChmodForm, ChownForm, EditorForm
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.exceptions import WebHdfsException
 
@@ -874,7 +876,7 @@ def upload_flash(request):
     special signifier.
     """
     try:
-        r = upload(request)
+        r = upload_file(request)
         if r.status_code == 200:
             return HttpResponse("{}", content_type="application/json")
         else:
@@ -884,7 +886,7 @@ def upload_flash(request):
                             content_type="application/json")
 
 
-def upload(request):
+def upload_file(request):
     """
     A wrapper around the actual upload view function to clean up the temporary file afterwards.
 
@@ -896,7 +898,7 @@ def upload(request):
     if request.method == 'POST':
         try:
             try:
-                resp = _upload(request)
+                resp = _upload_file(request)
                 response.update(resp)
             except Exception, ex:
                 response['data'] = str(ex)
@@ -914,13 +916,12 @@ def upload(request):
 
     return HttpResponse(json.dumps(response), content_type="text/plain")
 
-
-def _upload(request):
+def _upload_file(request):
     """
     Handles file uploaded by HDFSfileUploadHandler.
     The uploaded file is stored in HDFS. We just need to rename it to the destination path.
     """
-    form = UploadForm(request.POST, request.FILES)
+    form = UploadFileForm(request.POST, request.FILES)
 
     if form.is_valid():
         uploaded_file = request.FILES['hdfs_file']
@@ -939,7 +940,7 @@ def _upload(request):
             request.fs.do_as_superuser(request.fs.chown, tmp_file, username, username)
 
             # Move the file to where it belongs
-            request.fs.rename(uploaded_file.get_temp_path(), dest)
+            request.fs.rename(tmp_file, dest)
         except IOError, ex:
             already_exists = False
             try:
@@ -962,6 +963,87 @@ def _upload(request):
         raise PopupException(_("Error in upload form: %s") % (form.errors,))
 
 
+def upload_archive(request):
+    """
+    A wrapper around the actual upload view function to clean up the temporary file afterwards.
+
+    Returns JSON.
+    e.g. {'status' 0/1, data:'message'...}
+    """
+    response = {'status': -1, 'data': ''}
+
+    if request.method == 'POST':
+        try:
+            try:
+                resp = _upload_archive(request)
+                response.update(resp)
+            except Exception, ex:
+                response['data'] = str(ex)
+        finally:
+            hdfs_file = request.FILES.get('hdfs_file')
+            if hdfs_file:
+                hdfs_file.remove()
+    else:
+        response['data'] = _('A POST request is required.')
+
+    if response['status'] == 0:
+        request.info(_('%(destination)s upload succeeded') % {'destination': response['path']})
+    else:
+        request.error(_('Upload failed: %(data)s') % {'data': response['data']})
+
+    return HttpResponse(json.dumps(response), content_type="text/plain")
+
+
+def _upload_archive(request):
+    """
+    Handles archive upload.
+    The uploaded file is stored in memory.
+    We need to extract it and rename it.
+    """
+    form = UploadArchiveForm(request.POST, request.FILES)
+
+    if form.is_valid():
+        uploaded_file = request.FILES['archive']
+
+        # Always a dir
+        if request.fs.isdir(form.cleaned_data['dest']) and posixpath.sep in uploaded_file.name:
+            raise PopupException(_('Sorry, no "%(sep)s" in the filename %(name)s.' % {'sep': posixpath.sep, 'name': uploaded_file.name}))
+
+        dest = request.fs.join(form.cleaned_data['dest'], uploaded_file.name)
+        try:
+            # Extract if necessary
+            # Make sure dest path is without '.zip' extension
+            if dest.endswith('.zip'):
+                temp_path = archive_factory(uploaded_file).extract()
+                if not temp_path:
+                    raise PopupException(_('Could not extract contents of file.'))
+                # Move the file to where it belongs
+                dest = dest[:-4]
+                request.fs.copyFromLocal(temp_path, dest)
+                shutil.rmtree(temp_path)
+            else:
+                raise PopupException(_('Could not interpret archive type.'))
+
+        except IOError, ex:
+            already_exists = False
+            try:
+                already_exists = request.fs.exists(dest)
+            except Exception:
+              pass
+            if already_exists:
+                msg = _('Destination %(name)s already exists.' % {'name': dest})
+            else:
+                msg = _('Copy to "%(name)s failed: %(error)s') % {'name': dest, 'error': ex}
+            raise PopupException(msg)
+
+        return {
+          'status': 0,
+          'path': dest,
+          'result': _massage_stats(request, request.fs.stats(dest)),
+          'next': request.GET.get("next")
+          }
+    else:
+        raise PopupException(_("Error in upload form: %s") % (form.errors,))
 
 def status(request):
     status = request.fs.status()

+ 46 - 4
apps/filebrowser/src/filebrowser/views_test.py

@@ -37,6 +37,7 @@ from desktop.lib.test_utils import grant_access
 from hadoop import pseudo_hdfs4
 
 from avro import schema, datafile, io
+from lib.archives import archive_factory
 from lib.rwx import expand_mode
 
 
@@ -672,7 +673,7 @@ def edit_helper(cluster, encoding, contents_pass_1, contents_pass_2):
 
 
 @attr('requires_hadoop')
-def test_upload():
+def test_upload_file():
   """Test file upload"""
   cluster = pseudo_hdfs4.shared_cluster()
 
@@ -683,6 +684,7 @@ def test_upload():
     LOCAL_FILE = __file__
     HDFS_FILE = HDFS_DEST_DIR + '/' + os.path.basename(__file__)
 
+    cluster.fs.setuser(USER_NAME)
     client = make_logged_in_client(USER_NAME)
 
     client_not_me = make_logged_in_client(username=USER_NAME_NOT_ME, is_superuser=False, groupname='test')
@@ -693,7 +695,7 @@ def test_upload():
     cluster.fs.do_as_superuser(cluster.fs.chmod, HDFS_DEST_DIR, 0700)
 
     # Just upload the current python file
-    resp = client.post('/filebrowser/upload',
+    resp = client.post('/filebrowser/upload/file',
                        dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
     response = json.loads(resp.content)
 
@@ -708,13 +710,13 @@ def test_upload():
     assert_equal(actual, expected)
 
     # Upload again and so fails because file already exits
-    resp = client.post('/filebrowser/upload',
+    resp = client.post('/filebrowser/upload/file',
                        dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
     response = json.loads(resp.content)
     assert_equal(-1, response['status'], response)
 
     # Upload in tmp and fails because of missing permissions
-    resp = client_not_me.post('/filebrowser/upload',
+    resp = client_not_me.post('/filebrowser/upload/file',
                               dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
     response = json.loads(resp.content)
     assert_equal(-1, response['status'], response)
@@ -723,3 +725,43 @@ def test_upload():
       cluster.fs.remove(HDFS_DEST_DIR)
     except Exception, ex:
       pass
+
+@attr('requires_hadoop')
+def test_upload_archive():
+  """Test archive upload"""
+  cluster = pseudo_hdfs4.shared_cluster()
+
+  try:
+    USER_NAME = 'test'
+    HDFS_DEST_DIR = "/tmp/fb-upload-test"
+    ZIP_FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.zip')
+    HDFS_ZIP_FILE = HDFS_DEST_DIR + '/test.zip'
+    HDFS_UNZIPPED_FILE = HDFS_DEST_DIR + '/test'
+
+    cluster.fs.setuser(USER_NAME)
+    client = make_logged_in_client(USER_NAME)
+
+    cluster.fs.mkdir(HDFS_DEST_DIR)
+    cluster.fs.chown(HDFS_DEST_DIR, USER_NAME)
+    cluster.fs.chmod(HDFS_DEST_DIR, 0700)
+
+    # Upload and unzip archive
+    resp = client.post('/filebrowser/upload/archive',
+                       dict(dest=HDFS_DEST_DIR, archive=file(ZIP_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_false(cluster.fs.exists(HDFS_ZIP_FILE))
+    assert_true(cluster.fs.isdir(HDFS_UNZIPPED_FILE))
+    assert_true(cluster.fs.isfile(HDFS_UNZIPPED_FILE + '/test.txt'))
+
+    # Upload archive
+    resp = client.post('/filebrowser/upload/file',
+                       dict(dest=HDFS_DEST_DIR, hdfs_file=file(ZIP_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_true(cluster.fs.exists(HDFS_ZIP_FILE))
+  finally:
+    try:
+      cluster.fs.remove(HDFS_DEST_DIR)
+    except Exception, ex:
+      pass

+ 2 - 2
desktop/core/src/desktop/templates/actionbar.mako

@@ -19,11 +19,11 @@
 
 <%def name="render()">
     <div class="well" style="padding-top: 10px; padding-bottom: 0">
-        <p class="pull-right" style="margin:0">
+        <div class="pull-right" style="margin:0">
             %if hasattr(caller, "creation"):
                 ${caller.creation()}
             %endif
-        </p>
+        </div>
         <p>
             %if hasattr(caller, "search"):
                 ${caller.search()}

+ 21 - 0
desktop/libs/hadoop/src/hadoop/fs/fs_test.py

@@ -115,6 +115,27 @@ def test_hdfs_copy():
   sb = minifs.stats('/copy_test_dst')
   assert_equal(0646, stat.S_IMODE(sb.mode))
 
+@attr('requires_hadoop')
+def test_hdfs_copy_from_local():
+  minicluster = pseudo_hdfs4.shared_cluster()
+  minifs = minicluster.fs
+
+  olduser = minifs.setuser(minifs.superuser)
+  minifs.chmod('/', 0777)
+  minifs.setuser(olduser)
+
+  path = os.path.join(tempfile.gettempdir(), 'copy_test_src')
+  logging.info(path)
+
+  data = "I will not make flatuent noises in class\n" * 2000
+  f = open(path, 'w')
+  f.write(data)
+  f.close()
+
+  minifs.copyFromLocal(path, '/copy_test_dst')
+  actual = minifs.read('/copy_test_dst', 0, len(data) + 100)
+  assert_equal(data, actual)
+
 
 if __name__ == "__main__":
   logging.basicConfig()

+ 66 - 0
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -31,6 +31,7 @@ import threading
 from thrift.transport import TTransport
 
 from django.utils.encoding import smart_str, force_unicode
+from django.utils.translation import ugettext as _
 from desktop.lib import thrift_util, i18n
 from desktop.lib.conf import validate_port
 from hadoop.api.hdfs import Namenode, Datanode
@@ -215,6 +216,71 @@ class Hdfs(object):
     path = url[i:]
     return (schema, netloc, normpath(path), '', '')
 
+  def exists(self):
+    raise NotImplementedError(_("exists has not been implemented."_))
+
+  def do_as_user(self):
+    raise NotImplementedError(_("do_as_user has not been implemented."))
+
+  def create(self):
+    raise NotImplementedError(_("exists has not been implemented."))
+
+  def append(self):
+    raise NotImplementedError(_("do_as_user has not been implemented."))
+
+  def mkdir(self):
+    raise NotImplementedError(_("mkdir has not been implemented."))
+
+  def isdir(self):
+    raise NotImplementedError(_("isdir has not been implemented."))
+
+  def copyFromLocal(self, local_src, remote_dst, mode=0755):
+    remote_dst = remote_dst.endswith(posixpath.sep) and remote_dst[:-1] or remote_dst
+    local_src = local_src.endswith(posixpath.sep) and local_src[:-1] or local_src
+
+    if os.path.isdir(local_src):
+      self._copy_dir(local_src, remote_dst, mode)
+    else:
+      (basename, filename) = os.path.split(local_src)
+      self._copy_file(local_src, self.isdir(remote_dst) and self.join(remote_dst, filename) or remote_dst)
+
+  def _copy_dir(self, local_dir, remote_dir, mode=0755):
+    self.mkdir(remote_dir, mode=mode)
+
+    for f in os.listdir(local_dir):
+      local_src = os.path.join(local_dir, f)
+      remote_dst = self.join(remote_dir, f)
+
+      if os.path.isdir(local_src):
+        self._copy_dir(local_src, remote_dst, mode)
+      else:
+        self._copy_file(local_src, remote_dst)
+
+  def _copy_file(self, local_src, remote_dst, chunk_size=1024 * 1024):
+    if os.path.isfile(local_src):
+      if self.exists(remote_dst):
+        LOG.info(_('%(remote_dst)s already exists. Skipping.') % {'remote_dst': remote_dst})
+        return
+      else:
+        LOG.info(_('%(remote_dst)s does not exist. Trying to copy') % {'remote_dst': remote_dst})
+
+      src = file(local_src)
+      try:
+        try:
+          self.create(remote_dst, permission=01755)
+          chunk = src.read(chunk_size)
+          while chunk:
+            self.append(remote_dst, chunk)
+            chunk = src.read(chunk_size)
+          LOG.info(_('Copied %s -> %s') % (local_src, remote_dst))
+        except:
+          LOG.error(_('Copying %s -> %s failed') % (local_src, remote_dst))
+          raise
+      finally:
+        src.close()
+    else:
+      LOG.info(_('Skipping %s (not a file)') % local_src)
+
 class HadoopFileSystem(Hdfs):
   """
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.