Browse Source

HUE-775 [fb] Upload zip file through filebrowser.

- Added test cases to validate unzipping
- Added upload scenarios to upload test case to ensure proper uploading and extraction
- Added a hackish solution for explicitly requiring zip extraction
- New 'uplaod archive' button.
abec 13 years ago
parent
commit
be9f6467a2

+ 6 - 1
apps/filebrowser/src/filebrowser/forms.py

@@ -53,12 +53,17 @@ class RenameForm(forms.Form):
   src_path = CharField(label=_("File to rename"), help_text=_("The file to rename."))
   src_path = CharField(label=_("File to rename"), help_text=_("The file to rename."))
   dest_path = CharField(label=_("New name"), help_text=_("Rename the file to:"))
   dest_path = CharField(label=_("New name"), help_text=_("Rename the file to:"))
 
 
-class UploadForm(forms.Form):
+class UploadFileForm(forms.Form):
   op = "upload"
   op = "upload"
   # The "hdfs" prefix in "hdfs_file" triggers the HDFSfileUploadHandler
   # The "hdfs" prefix in "hdfs_file" triggers the HDFSfileUploadHandler
   hdfs_file = FileField(forms.Form, label=_("File to Upload"))
   hdfs_file = FileField(forms.Form, label=_("File to Upload"))
   dest = PathField(label=_("Destination Path"), help_text=_("Filename or directory to upload to."))
   dest = PathField(label=_("Destination Path"), help_text=_("Filename or directory to upload to."))
 
 
+class UploadArchiveForm(forms.Form):
+  op = "upload"
+  archive = FileField(forms.Form, label=_("Archive to Upload"))
+  dest = PathField(label=_("Destination Path"), help_text=_("Archive to upload to."))
+
 class RemoveForm(forms.Form):
 class RemoveForm(forms.Form):
   op = "remove"
   op = "remove"
   path = PathField(label=_("File to remove"))
   path = PathField(label=_("File to remove"))

+ 102 - 0
apps/filebrowser/src/filebrowser/lib/archives.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Utilities for dealing with file modes.
+
+import os
+import posixpath
+import tempfile
+import uuid
+from zipfile import ZipFile
+
+from django.utils.translation import ugettext as _
+
+
+__all__ = ['archive_factory']
+
+
+class Archive(object):
+  """
+  Acrchive interface.
+  """
+  def extract(self, path):
+    """
+    Extract an Archive.
+    Should return a directory where the extracted contents live.
+    """
+    raise NotImplemented(_("Must implement 'extract' method."))
+
+class ZipArchive(Archive):
+  """
+  Acts on a zip file in memory or in a temporary location.
+  Python's ZipFile class inherently buffers all reading.
+  """
+  def __init__(self, file):
+    self.file = isinstance(file, basestring) and open(file) or file
+    self.zfh = ZipFile(self.file)
+
+  def extract(self):
+    """
+    Extracts a zip file.
+    If a 'file' ends with '/', then it is a directory and we must create it.
+    Else, open a file for writing and meta pipe the contents zipfile to the new file.
+    """
+    # Store all extracted files in a temporary directory.
+    directory = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
+    os.mkdir(directory)
+
+    dirs, files = self._filenames()
+    self._create_dirs(directory, dirs)
+    self._create_files(directory, files)
+
+    return directory
+
+  def _filenames(self):
+    """
+    List all dirs and files by reading the table of contents of the Zipfile.
+    """
+    dirs = []
+    files = []
+    for name in self.zfh.namelist():
+      if name.endswith(posixpath.sep):
+        dirs.append(name)
+      else:
+        files.append(name)
+    return (dirs, files)
+
+  def _create_dirs(self, basepath, dirs=[]):
+    """
+    Creates all directories passed at the given basepath.
+    """
+    for directory in dirs:
+      directory = os.path.join(basepath, directory)
+      os.mkdir(directory)
+
+  def _create_files(self, basepath, files=[]):
+    """
+    Extract files to their rightful place.
+    Files are written to a temporary directory immediately after being decompressed.
+    """
+    for f in files:
+      new_path = os.path.join(basepath, f)
+      new_file = open(new_path, 'w')
+      new_file.write(self.zfh.read(f))
+      new_file.close()
+
+
+def archive_factory(path, archive_type='zip'):
+  return ZipArchive(path)

+ 41 - 0
apps/filebrowser/src/filebrowser/lib/archives_test.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import os
+
+from nose.tools import assert_true, assert_equal
+
+import archives
+
+
+class ArchiveTest(unittest.TestCase):
+
+  def test_zip(self):
+    FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.zip')
+
+    # Extract the file
+    # This file should only have 'test.txt' in it
+    directory = archives.archive_factory(FILE, 'zip').extract()
+    assert_true(os.path.exists(directory))
+    assert_true(os.path.isdir(directory))
+    assert_true(os.path.isfile(directory + '/test.txt'))
+    assert_equal(os.path.getsize(directory + '/test.txt'), 4)
+
+
+if __name__ == "__main__":
+  unittest.main()

+ 8 - 1
apps/filebrowser/src/filebrowser/templates/listdir.mako

@@ -41,7 +41,14 @@ ${commonheader(_('File Browser'), 'filebrowser', user)}
             <button class="btn fileToolbarBtn" title="${_('Delete')}" data-bind="click: deleteSelected, enable: selectedFiles().length == 1"><i class="icon-trash"></i> ${_('Delete')}</button>
             <button class="btn fileToolbarBtn" title="${_('Delete')}" data-bind="click: deleteSelected, enable: selectedFiles().length == 1"><i class="icon-trash"></i> ${_('Delete')}</button>
         </%def>
         </%def>
         <%def name="creation()">
         <%def name="creation()">
-            <a href="#" class="btn upload-link" title="${_('Upload files')}"><i class="icon-upload"></i> ${_('Upload files')}</a>
+            <div id="upload-dropdown" class="btn-group" style="display: inline-block;">
+                <a href="#" class="btn upload-link dropdown-toggle" title="${_('Upload')}" data-toggle="dropdown"><i class="icon-upload"></i> ${_('Upload')}</a>
+                <ul class="dropdown-menu">
+                  <li><a href="#" tabindex="-1" class="upload-link" title="${_('Upload files')}" data-bind="click: uploadFile">${_('Upload files')}</a></li>
+                  <li class="divider"></li>
+                  <li><a href="#" tabindex="-1" class="upload-link" title="${_('Upload archive')}" data-bind="click: uploadArchive">${_('Upload archive')}</a></li>
+                </ul>
+            </div>
             <a href="#" class="btn create-directory-link" title="${_('New directory')}"><i class="icon-folder-close"></i> ${_('New directory')}</a>
             <a href="#" class="btn create-directory-link" title="${_('New directory')}"><i class="icon-folder-close"></i> ${_('New directory')}</a>
         </%def>
         </%def>
     </%actionbar:render>
     </%actionbar:render>

+ 120 - 56
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -166,14 +166,31 @@ from django.utils.translation import ugettext as _
 
 
     <div id="moveModal" class="modal hide fade"></div>
     <div id="moveModal" class="modal hide fade"></div>
 
 
-    <!-- upload modal -->
-    <div id="uploadModal" class="modal hide fade">
+    <!-- upload file modal -->
+    <div id="uploadFileModal" class="modal hide fade">
         <div class="modal-header">
         <div class="modal-header">
             <a href="#" class="close" data-dismiss="modal">&times;</a>
             <a href="#" class="close" data-dismiss="modal">&times;</a>
             <h3>${_('Uploading to:')} <span id="uploadDirName" data-bind="text: currentPath"></span></h3>
             <h3>${_('Uploading to:')} <span id="uploadDirName" data-bind="text: currentPath"></span></h3>
         </div>
         </div>
-        <div class="modal-body">
-            <div id="fileUploader">
+        <div class="modal-body form-inline">
+            <div id="fileUploader" class="uploader">
+            <noscript>
+                <p>${_('Please enable JavaScript to use the file uploader.')}</p>
+            </noscript>
+            </div>
+        </div>
+        <div class="modal-footer"></div>
+    </div>
+
+    <!-- upload archive modal -->
+    <div id="uploadArchiveModal" class="modal hide fade">
+        <div class="modal-header">
+            <a href="#" class="close" data-dismiss="modal">&times;</a>
+            <h3>${_('Uploading to:')} <span id="uploadDirName" data-bind="text: currentPath"></span></h3>
+            <p>${_('The file will then be extracted in the path specified above.')}</p>
+        </div>
+        <div class="modal-body form-inline">
+            <div id="archiveUploader" class="uploader">
             <noscript>
             <noscript>
                 <p>${_('Please enable JavaScript to use the file uploader.')}</p>
                 <p>${_('Please enable JavaScript to use the file uploader.')}</p>
             </noscript>
             </noscript>
@@ -259,46 +276,8 @@ from django.utils.translation import ugettext as _
             });
             });
         }
         }
 
 
-        //uploader
-        var num_of_pending_uploads = 0;
-        var uploader = null;
-        function createUploader(){
-            uploader = new qq.FileUploader({
-                element: document.getElementById("fileUploader"),
-                action: "/filebrowser/upload",
-                template: '<div class="qq-uploader">' +
-                        '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
-                        '<div class="qq-upload-button">${_('Upload a file')}</div>' +
-                        '<ul class="qq-upload-list"></ul>' +
-                        '</div>',
-                fileTemplate: '<li>' +
-                        '<span class="qq-upload-file"></span>' +
-                        '<span class="qq-upload-spinner"></span>' +
-                        '<span class="qq-upload-size"></span>' +
-                        '<a class="qq-upload-cancel" href="#">${_('Cancel')}</a>' +
-                        '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
-                        '</li>',
-                params:{
-                    dest: viewModel.currentPath(),
-                    fileFieldLabel: "hdfs_file"
-                },
-                onComplete:function(id, fileName, responseJSON){
-                    num_of_pending_uploads--;
-                    if(num_of_pending_uploads == 0){
-                        window.location = "/filebrowser/view" + viewModel.currentPath();
-                    }
-                },
-                onSubmit:function(id, fileName, responseJSON){
-                    num_of_pending_uploads++;
-                },
-                debug: false
-            });
-        }
-
         $(document).ready(function(){
         $(document).ready(function(){
 
 
-            createUploader();
-
             $("#cancelDeleteBtn").click(function(){
             $("#cancelDeleteBtn").click(function(){
                 $("#deleteModal").modal("hide");
                 $("#deleteModal").modal("hide");
             });
             });
@@ -334,14 +313,6 @@ from django.utils.translation import ugettext as _
                 $("#moveForm").find("input[name='dest_path']").removeClass("fieldError");
                 $("#moveForm").find("input[name='dest_path']").removeClass("fieldError");
             });
             });
 
 
-            //upload handlers
-            $(".upload-link").click(function(){
-                $("#uploadModal").modal({
-                    keyboard: true,
-                    show: true
-                });
-            });
-
             //create directory handlers
             //create directory handlers
             $(".create-directory-link").click(function(){
             $(".create-directory-link").click(function(){
                 $("#createDirectoryModal").modal({
                 $("#createDirectoryModal").modal({
@@ -594,12 +565,9 @@ from django.utils.translation import ugettext as _
                     return new Breadcrumb(breadcrumb);
                     return new Breadcrumb(breadcrumb);
                 }));
                 }));
                 self.currentPath(currentDirPath);
                 self.currentPath(currentDirPath);
-                if (uploader != null){
-                    uploader.setParams({
-                        dest: self.currentPath(),
-                        fileFieldLabel: "hdfs_file"
-                    });
-                }
+
+                $('.uploader').trigger('fb:updatePath', {dest: self.currentPath()});
+ 
                 self.isLoading(false);
                 self.isLoading(false);
                 $(".scrollable").jHueTableScroller();
                 $(".scrollable").jHueTableScroller();
             };
             };
@@ -700,6 +668,102 @@ from django.utils.translation import ugettext as _
                 $(formElement).attr("action", "/filebrowser/mkdir?next=${url('filebrowser.views.view', path=urlencode('/'))}"+ "." + self.currentPath());
                 $(formElement).attr("action", "/filebrowser/mkdir?next=${url('filebrowser.views.view', path=urlencode('/'))}"+ "." + self.currentPath());
                 return true;
                 return true;
             };
             };
+
+            self.uploadFile = (function() {
+                var num_of_pending_uploads = 0;
+                var uploader = new qq.FileUploader({
+                    element: document.getElementById("fileUploader"),
+                    action: "/filebrowser/upload/file",
+                    template: '<div class="qq-uploader">' +
+                            '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
+                            '<div class="qq-upload-button">${_('Upload a file')}</div>' +
+                            '<ul class="qq-upload-list"></ul>' +
+                            '</div>',
+                    fileTemplate: '<li>' +
+                            '<span class="qq-upload-file"></span>' +
+                            '<span class="qq-upload-spinner"></span>' +
+                            '<span class="qq-upload-size"></span>' +
+                            '<a class="qq-upload-cancel" href="#">${_('Cancel')}</a>' +
+                            '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
+                            '</li>',
+                    params:{
+                        dest: self.currentPath(),
+                        fileFieldLabel: "hdfs_file"
+                    },
+                    onComplete:function(id, fileName, responseJSON){
+                        num_of_pending_uploads--;
+                        if(num_of_pending_uploads == 0){
+                            window.location = "/filebrowser/view" + self.currentPath();
+                        }
+                    },
+                    onSubmit:function(id, fileName, responseJSON){
+                        num_of_pending_uploads++;
+                    },
+                    debug: false
+                });
+
+                $("#archiveUploader").on('fb:updatePath', function(e, options) {
+                    uploader.setParams({
+                        dest: options.dest,
+                        fileFieldLabel: "hdfs_file"
+                    });
+                });
+
+                return function() {
+                    $("#uploadFileModal").modal({
+                        keyboard: true,
+                        show: true
+                    });
+                };
+            })();
+
+            self.uploadArchive = (function() {
+                var num_of_pending_uploads = 0;
+                var uploader = new qq.FileUploader({
+                    element: document.getElementById("archiveUploader"),
+                    action: "/filebrowser/upload/archive",
+                    template: '<div class="qq-uploader">' +
+                            '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
+                            '<div class="qq-upload-button">${_('Upload an archive')}</div>' +
+                            '<ul class="qq-upload-list"></ul>' +
+                            '</div>',
+                    fileTemplate: '<li>' +
+                            '<span class="qq-upload-file"></span>' +
+                            '<span class="qq-upload-spinner"></span>' +
+                            '<span class="qq-upload-size"></span>' +
+                            '<a class="qq-upload-cancel" href="#">${_('Cancel')}</a>' +
+                            '<span class="qq-upload-failed-text">${_('Failed')}</span>' +
+                            '</li>',
+                    params:{
+                        dest: self.currentPath(),
+                        fileFieldLabel: "archive"
+                    },
+                    onComplete:function(id, fileName, responseJSON){
+                        num_of_pending_uploads--;
+                        if(num_of_pending_uploads == 0){
+                            window.location = "/filebrowser/view" + self.currentPath();
+                        }
+                    },
+                    onSubmit:function(id, fileName, responseJSON){
+                        num_of_pending_uploads++;
+                    },
+                    debug: false
+                });
+
+                $("#archiveUploader").on('fb:updatePath', function(e, options) {
+                    uploader.setParams({
+                        dest: options.dest,
+                        fileFieldLabel: "archive"
+                    });
+                });
+
+                return function() {
+                    $("#uploadArchiveModal").modal({
+                        keyboard: true,
+                        show: true
+                    });
+                };
+            })();
         };
         };
 
 
         var viewModel = new FileBrowserModel([], null, [], "/");
         var viewModel = new FileBrowserModel([], null, [], "/");

BIN
apps/filebrowser/src/filebrowser/test_data/test.zip


+ 2 - 1
apps/filebrowser/src/filebrowser/urls.py

@@ -35,7 +35,8 @@ urlpatterns = patterns('filebrowser.views',
 
 
   # POST operations
   # POST operations
   url(r'upload_flash$', 'upload_flash', name='upload_flash'),
   url(r'upload_flash$', 'upload_flash', name='upload_flash'),
-  url(r'upload$', 'upload', name='upload'),
+  url(r'upload/file$', 'upload_file', name='upload_file'),
+  url(r'upload/archive$', 'upload_archive', name='upload_archive'),
   url(r'rename', 'rename', name='rename'),
   url(r'rename', 'rename', name='rename'),
   url(r'mkdir', 'mkdir', name='mkdir'),
   url(r'mkdir', 'mkdir', name='mkdir'),
   url(r'^move', 'move', name='move'),
   url(r'^move', 'move', name='move'),

+ 91 - 9
apps/filebrowser/src/filebrowser/views.py

@@ -25,6 +25,7 @@ import logging
 import mimetypes
 import mimetypes
 import operator
 import operator
 import posixpath
 import posixpath
+import shutil
 import stat as stat_module
 import stat as stat_module
 import os
 import os
 from types import MethodType
 from types import MethodType
@@ -50,10 +51,11 @@ from desktop.lib import i18n, paginator
 from desktop.lib.conf import coerce_bool
 from desktop.lib.conf import coerce_bool
 from desktop.lib.django_util import make_absolute, render, render_json, format_preserving_redirect
 from desktop.lib.django_util import make_absolute, render, render_json, format_preserving_redirect
 from desktop.lib.exceptions import PopupException
 from desktop.lib.exceptions import PopupException
+from filebrowser.lib.archives import archive_factory
 from filebrowser.lib.rwx import filetype, rwx
 from filebrowser.lib.rwx import filetype, rwx
 from filebrowser.lib import xxd
 from filebrowser.lib import xxd
-from filebrowser.forms import RenameForm, UploadForm, MkDirForm, RmDirForm, RmTreeForm,\
-    RemoveForm, ChmodForm, ChownForm, EditorForm
+from filebrowser.forms import RenameForm, UploadFileForm, UploadArchiveForm, MkDirForm,\
+    RmDirForm, RmTreeForm, RemoveForm, ChmodForm, ChownForm, EditorForm
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.exceptions import WebHdfsException
 from hadoop.fs.exceptions import WebHdfsException
 
 
@@ -874,7 +876,7 @@ def upload_flash(request):
     special signifier.
     special signifier.
     """
     """
     try:
     try:
-        r = upload(request)
+        r = upload_file(request)
         if r.status_code == 200:
         if r.status_code == 200:
             return HttpResponse("{}", content_type="application/json")
             return HttpResponse("{}", content_type="application/json")
         else:
         else:
@@ -884,7 +886,7 @@ def upload_flash(request):
                             content_type="application/json")
                             content_type="application/json")
 
 
 
 
-def upload(request):
+def upload_file(request):
     """
     """
     A wrapper around the actual upload view function to clean up the temporary file afterwards.
     A wrapper around the actual upload view function to clean up the temporary file afterwards.
 
 
@@ -896,7 +898,7 @@ def upload(request):
     if request.method == 'POST':
     if request.method == 'POST':
         try:
         try:
             try:
             try:
-                resp = _upload(request)
+                resp = _upload_file(request)
                 response.update(resp)
                 response.update(resp)
             except Exception, ex:
             except Exception, ex:
                 response['data'] = str(ex)
                 response['data'] = str(ex)
@@ -914,13 +916,12 @@ def upload(request):
 
 
     return HttpResponse(json.dumps(response), content_type="text/plain")
     return HttpResponse(json.dumps(response), content_type="text/plain")
 
 
-
-def _upload(request):
+def _upload_file(request):
     """
     """
     Handles file uploaded by HDFSfileUploadHandler.
     Handles file uploaded by HDFSfileUploadHandler.
     The uploaded file is stored in HDFS. We just need to rename it to the destination path.
     The uploaded file is stored in HDFS. We just need to rename it to the destination path.
     """
     """
-    form = UploadForm(request.POST, request.FILES)
+    form = UploadFileForm(request.POST, request.FILES)
 
 
     if form.is_valid():
     if form.is_valid():
         uploaded_file = request.FILES['hdfs_file']
         uploaded_file = request.FILES['hdfs_file']
@@ -939,7 +940,7 @@ def _upload(request):
             request.fs.do_as_superuser(request.fs.chown, tmp_file, username, username)
             request.fs.do_as_superuser(request.fs.chown, tmp_file, username, username)
 
 
             # Move the file to where it belongs
             # Move the file to where it belongs
-            request.fs.rename(uploaded_file.get_temp_path(), dest)
+            request.fs.rename(tmp_file, dest)
         except IOError, ex:
         except IOError, ex:
             already_exists = False
             already_exists = False
             try:
             try:
@@ -962,6 +963,87 @@ def _upload(request):
         raise PopupException(_("Error in upload form: %s") % (form.errors,))
         raise PopupException(_("Error in upload form: %s") % (form.errors,))
 
 
 
 
+def upload_archive(request):
+    """
+    A wrapper around the actual upload view function to clean up the temporary file afterwards.
+
+    Returns JSON.
+    e.g. {'status' 0/1, data:'message'...}
+    """
+    response = {'status': -1, 'data': ''}
+
+    if request.method == 'POST':
+        try:
+            try:
+                resp = _upload_archive(request)
+                response.update(resp)
+            except Exception, ex:
+                response['data'] = str(ex)
+        finally:
+            hdfs_file = request.FILES.get('hdfs_file')
+            if hdfs_file:
+                hdfs_file.remove()
+    else:
+        response['data'] = _('A POST request is required.')
+
+    if response['status'] == 0:
+        request.info(_('%(destination)s upload succeeded') % {'destination': response['path']})
+    else:
+        request.error(_('Upload failed: %(data)s') % {'data': response['data']})
+
+    return HttpResponse(json.dumps(response), content_type="text/plain")
+
+
+def _upload_archive(request):
+    """
+    Handles archive upload.
+    The uploaded file is stored in memory.
+    We need to extract it and rename it.
+    """
+    form = UploadArchiveForm(request.POST, request.FILES)
+
+    if form.is_valid():
+        uploaded_file = request.FILES['archive']
+
+        # Always a dir
+        if request.fs.isdir(form.cleaned_data['dest']) and posixpath.sep in uploaded_file.name:
+            raise PopupException(_('Sorry, no "%(sep)s" in the filename %(name)s.' % {'sep': posixpath.sep, 'name': uploaded_file.name}))
+
+        dest = request.fs.join(form.cleaned_data['dest'], uploaded_file.name)
+        try:
+            # Extract if necessary
+            # Make sure dest path is without '.zip' extension
+            if dest.endswith('.zip'):
+                temp_path = archive_factory(uploaded_file).extract()
+                if not temp_path:
+                    raise PopupException(_('Could not extract contents of file.'))
+                # Move the file to where it belongs
+                dest = dest[:-4]
+                request.fs.copyFromLocal(temp_path, dest)
+                shutil.rmtree(temp_path)
+            else:
+                raise PopupException(_('Could not interpret archive type.'))
+
+        except IOError, ex:
+            already_exists = False
+            try:
+                already_exists = request.fs.exists(dest)
+            except Exception:
+              pass
+            if already_exists:
+                msg = _('Destination %(name)s already exists.' % {'name': dest})
+            else:
+                msg = _('Copy to "%(name)s failed: %(error)s') % {'name': dest, 'error': ex}
+            raise PopupException(msg)
+
+        return {
+          'status': 0,
+          'path': dest,
+          'result': _massage_stats(request, request.fs.stats(dest)),
+          'next': request.GET.get("next")
+          }
+    else:
+        raise PopupException(_("Error in upload form: %s") % (form.errors,))
 
 
 def status(request):
 def status(request):
     status = request.fs.status()
     status = request.fs.status()

+ 46 - 4
apps/filebrowser/src/filebrowser/views_test.py

@@ -37,6 +37,7 @@ from desktop.lib.test_utils import grant_access
 from hadoop import pseudo_hdfs4
 from hadoop import pseudo_hdfs4
 
 
 from avro import schema, datafile, io
 from avro import schema, datafile, io
+from lib.archives import archive_factory
 from lib.rwx import expand_mode
 from lib.rwx import expand_mode
 
 
 
 
@@ -672,7 +673,7 @@ def edit_helper(cluster, encoding, contents_pass_1, contents_pass_2):
 
 
 
 
 @attr('requires_hadoop')
 @attr('requires_hadoop')
-def test_upload():
+def test_upload_file():
   """Test file upload"""
   """Test file upload"""
   cluster = pseudo_hdfs4.shared_cluster()
   cluster = pseudo_hdfs4.shared_cluster()
 
 
@@ -683,6 +684,7 @@ def test_upload():
     LOCAL_FILE = __file__
     LOCAL_FILE = __file__
     HDFS_FILE = HDFS_DEST_DIR + '/' + os.path.basename(__file__)
     HDFS_FILE = HDFS_DEST_DIR + '/' + os.path.basename(__file__)
 
 
+    cluster.fs.setuser(USER_NAME)
     client = make_logged_in_client(USER_NAME)
     client = make_logged_in_client(USER_NAME)
 
 
     client_not_me = make_logged_in_client(username=USER_NAME_NOT_ME, is_superuser=False, groupname='test')
     client_not_me = make_logged_in_client(username=USER_NAME_NOT_ME, is_superuser=False, groupname='test')
@@ -693,7 +695,7 @@ def test_upload():
     cluster.fs.do_as_superuser(cluster.fs.chmod, HDFS_DEST_DIR, 0700)
     cluster.fs.do_as_superuser(cluster.fs.chmod, HDFS_DEST_DIR, 0700)
 
 
     # Just upload the current python file
     # Just upload the current python file
-    resp = client.post('/filebrowser/upload',
+    resp = client.post('/filebrowser/upload/file',
                        dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
                        dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
     response = json.loads(resp.content)
     response = json.loads(resp.content)
 
 
@@ -708,13 +710,13 @@ def test_upload():
     assert_equal(actual, expected)
     assert_equal(actual, expected)
 
 
     # Upload again and so fails because file already exits
     # Upload again and so fails because file already exits
-    resp = client.post('/filebrowser/upload',
+    resp = client.post('/filebrowser/upload/file',
                        dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
                        dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
     response = json.loads(resp.content)
     response = json.loads(resp.content)
     assert_equal(-1, response['status'], response)
     assert_equal(-1, response['status'], response)
 
 
     # Upload in tmp and fails because of missing permissions
     # Upload in tmp and fails because of missing permissions
-    resp = client_not_me.post('/filebrowser/upload',
+    resp = client_not_me.post('/filebrowser/upload/file',
                               dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
                               dict(dest=HDFS_DEST_DIR, hdfs_file=file(LOCAL_FILE)))
     response = json.loads(resp.content)
     response = json.loads(resp.content)
     assert_equal(-1, response['status'], response)
     assert_equal(-1, response['status'], response)
@@ -723,3 +725,43 @@ def test_upload():
       cluster.fs.remove(HDFS_DEST_DIR)
       cluster.fs.remove(HDFS_DEST_DIR)
     except Exception, ex:
     except Exception, ex:
       pass
       pass
+
+@attr('requires_hadoop')
+def test_upload_archive():
+  """Test archive upload"""
+  cluster = pseudo_hdfs4.shared_cluster()
+
+  try:
+    USER_NAME = 'test'
+    HDFS_DEST_DIR = "/tmp/fb-upload-test"
+    ZIP_FILE = os.path.realpath('apps/filebrowser/src/filebrowser/test_data/test.zip')
+    HDFS_ZIP_FILE = HDFS_DEST_DIR + '/test.zip'
+    HDFS_UNZIPPED_FILE = HDFS_DEST_DIR + '/test'
+
+    cluster.fs.setuser(USER_NAME)
+    client = make_logged_in_client(USER_NAME)
+
+    cluster.fs.mkdir(HDFS_DEST_DIR)
+    cluster.fs.chown(HDFS_DEST_DIR, USER_NAME)
+    cluster.fs.chmod(HDFS_DEST_DIR, 0700)
+
+    # Upload and unzip archive
+    resp = client.post('/filebrowser/upload/archive',
+                       dict(dest=HDFS_DEST_DIR, archive=file(ZIP_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_false(cluster.fs.exists(HDFS_ZIP_FILE))
+    assert_true(cluster.fs.isdir(HDFS_UNZIPPED_FILE))
+    assert_true(cluster.fs.isfile(HDFS_UNZIPPED_FILE + '/test.txt'))
+
+    # Upload archive
+    resp = client.post('/filebrowser/upload/file',
+                       dict(dest=HDFS_DEST_DIR, hdfs_file=file(ZIP_FILE)))
+    response = json.loads(resp.content)
+    assert_equal(0, response['status'], response)
+    assert_true(cluster.fs.exists(HDFS_ZIP_FILE))
+  finally:
+    try:
+      cluster.fs.remove(HDFS_DEST_DIR)
+    except Exception, ex:
+      pass

+ 2 - 2
desktop/core/src/desktop/templates/actionbar.mako

@@ -19,11 +19,11 @@
 
 
 <%def name="render()">
 <%def name="render()">
     <div class="well" style="padding-top: 10px; padding-bottom: 0">
     <div class="well" style="padding-top: 10px; padding-bottom: 0">
-        <p class="pull-right" style="margin:0">
+        <div class="pull-right" style="margin:0">
             %if hasattr(caller, "creation"):
             %if hasattr(caller, "creation"):
                 ${caller.creation()}
                 ${caller.creation()}
             %endif
             %endif
-        </p>
+        </div>
         <p>
         <p>
             %if hasattr(caller, "search"):
             %if hasattr(caller, "search"):
                 ${caller.search()}
                 ${caller.search()}

+ 21 - 0
desktop/libs/hadoop/src/hadoop/fs/fs_test.py

@@ -115,6 +115,27 @@ def test_hdfs_copy():
   sb = minifs.stats('/copy_test_dst')
   sb = minifs.stats('/copy_test_dst')
   assert_equal(0646, stat.S_IMODE(sb.mode))
   assert_equal(0646, stat.S_IMODE(sb.mode))
 
 
+@attr('requires_hadoop')
+def test_hdfs_copy_from_local():
+  minicluster = pseudo_hdfs4.shared_cluster()
+  minifs = minicluster.fs
+
+  olduser = minifs.setuser(minifs.superuser)
+  minifs.chmod('/', 0777)
+  minifs.setuser(olduser)
+
+  path = os.path.join(tempfile.gettempdir(), 'copy_test_src')
+  logging.info(path)
+
+  data = "I will not make flatuent noises in class\n" * 2000
+  f = open(path, 'w')
+  f.write(data)
+  f.close()
+
+  minifs.copyFromLocal(path, '/copy_test_dst')
+  actual = minifs.read('/copy_test_dst', 0, len(data) + 100)
+  assert_equal(data, actual)
+
 
 
 if __name__ == "__main__":
 if __name__ == "__main__":
   logging.basicConfig()
   logging.basicConfig()

+ 66 - 0
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -31,6 +31,7 @@ import threading
 from thrift.transport import TTransport
 from thrift.transport import TTransport
 
 
 from django.utils.encoding import smart_str, force_unicode
 from django.utils.encoding import smart_str, force_unicode
+from django.utils.translation import ugettext as _
 from desktop.lib import thrift_util, i18n
 from desktop.lib import thrift_util, i18n
 from desktop.lib.conf import validate_port
 from desktop.lib.conf import validate_port
 from hadoop.api.hdfs import Namenode, Datanode
 from hadoop.api.hdfs import Namenode, Datanode
@@ -215,6 +216,71 @@ class Hdfs(object):
     path = url[i:]
     path = url[i:]
     return (schema, netloc, normpath(path), '', '')
     return (schema, netloc, normpath(path), '', '')
 
 
+  def exists(self):
+    raise NotImplementedError(_("exists has not been implemented."_))
+
+  def do_as_user(self):
+    raise NotImplementedError(_("do_as_user has not been implemented."))
+
+  def create(self):
+    raise NotImplementedError(_("exists has not been implemented."))
+
+  def append(self):
+    raise NotImplementedError(_("do_as_user has not been implemented."))
+
+  def mkdir(self):
+    raise NotImplementedError(_("mkdir has not been implemented."))
+
+  def isdir(self):
+    raise NotImplementedError(_("isdir has not been implemented."))
+
+  def copyFromLocal(self, local_src, remote_dst, mode=0755):
+    remote_dst = remote_dst.endswith(posixpath.sep) and remote_dst[:-1] or remote_dst
+    local_src = local_src.endswith(posixpath.sep) and local_src[:-1] or local_src
+
+    if os.path.isdir(local_src):
+      self._copy_dir(local_src, remote_dst, mode)
+    else:
+      (basename, filename) = os.path.split(local_src)
+      self._copy_file(local_src, self.isdir(remote_dst) and self.join(remote_dst, filename) or remote_dst)
+
+  def _copy_dir(self, local_dir, remote_dir, mode=0755):
+    self.mkdir(remote_dir, mode=mode)
+
+    for f in os.listdir(local_dir):
+      local_src = os.path.join(local_dir, f)
+      remote_dst = self.join(remote_dir, f)
+
+      if os.path.isdir(local_src):
+        self._copy_dir(local_src, remote_dst, mode)
+      else:
+        self._copy_file(local_src, remote_dst)
+
+  def _copy_file(self, local_src, remote_dst, chunk_size=1024 * 1024):
+    if os.path.isfile(local_src):
+      if self.exists(remote_dst):
+        LOG.info(_('%(remote_dst)s already exists. Skipping.') % {'remote_dst': remote_dst})
+        return
+      else:
+        LOG.info(_('%(remote_dst)s does not exist. Trying to copy') % {'remote_dst': remote_dst})
+
+      src = file(local_src)
+      try:
+        try:
+          self.create(remote_dst, permission=01755)
+          chunk = src.read(chunk_size)
+          while chunk:
+            self.append(remote_dst, chunk)
+            chunk = src.read(chunk_size)
+          LOG.info(_('Copied %s -> %s') % (local_src, remote_dst))
+        except:
+          LOG.error(_('Copying %s -> %s failed') % (local_src, remote_dst))
+          raise
+      finally:
+        src.close()
+    else:
+      LOG.info(_('Skipping %s (not a file)') % local_src)
+
 class HadoopFileSystem(Hdfs):
 class HadoopFileSystem(Hdfs):
   """
   """
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.