Sfoglia il codice sorgente

[fb] Upload to federated cluster

The location for the temporary file being uploaded should be
the destination of the final, renamed file. The temporary file
will be named <filename>.tmp.

No new tests were necessary since we are not changing upload
functionality. Simply changing where the temp files exist.

HACK: In order to get the destination when uploading in the
custom upload handler, needed to use a GET parameter. This
is due to the obscurity in Django's builtin POST data
parser. Basically, the 'dest' POST parameter is not accessible
until the file contents has been parsed.
abec 13 anni fa
parent
commit
eb38948ee2

+ 3 - 2
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -916,9 +916,10 @@ from django.utils.translation import ugettext as _
 
             self.uploadFile = (function() {
                 var num_of_pending_uploads = 0;
+                var action = "/filebrowser/upload/file";
                 var uploader = new qq.FileUploader({
                     element: document.getElementById("fileUploader"),
-                    action: "/filebrowser/upload/file",
+                    action: action,
                     template: '<div class="qq-uploader">' +
                             '<div class="qq-upload-drop-area"><span>${_('Drop files here to upload')}</span></div>' +
                             '<div class="qq-upload-button">${_('Upload a file')}</div>' +
@@ -947,7 +948,7 @@ from django.utils.translation import ugettext as _
                     debug: false
                 });
 
-                $("#archiveUploader").on('fb:updatePath', function(e, options) {
+                $("#fileUploader").on('fb:updatePath', function(e, options) {
                     uploader.setParams({
                         dest: options.dest,
                         fileFieldLabel: "hdfs_file"

+ 2 - 1
desktop/core/static/ext/js/fileuploader.js

@@ -1231,7 +1231,8 @@ qq.extend(qq.UploadHandlerXhr.prototype, {
         formData.append(params.fileFieldLabel, file);
         formData.append('dest', params.dest);
 
-        xhr.open("POST", this._options.action, true);
+        var action = this._options.action + "?dest=" + params.dest;
+        xhr.open("POST", action, true);
         xhr.send(formData);
     },
     _onComplete: function(id, xhr){

+ 32 - 18
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -287,6 +287,38 @@ class Hdfs(object):
     else:
       LOG.info(_('Skipping %s (not a file).') % local_src)
 
+  @_coerce_exceptions
+  def mktemp(self, subdir='', prefix='tmp', basedir=None):
+    """
+    mktemp(prefix) ->  <temp_dir or basedir>/<subdir>/prefix.<rand>
+    Return a unique temporary filename with prefix in the cluster's temp dir.
+    """
+    RANDOM_BITS = 64
+
+    base = self.join(basedir or self._temp_dir, subdir)
+    if not self.isdir(base):
+      self.mkdir(base)
+
+    while True:
+      name = prefix + '.' + str(random.getrandbits(RANDOM_BITS))
+      candidate = self.join(base, name)
+      if not self.exists(candidate):
+        return candidate
+
+  def mkswap(self, filename, subdir='', suffix='swp', basedir=None):
+    """
+    mkswap(filename, suffix) ->  <temp_dir or basedir>/<subdir>/filename.<suffix>
+    Return a unique temporary filename with prefix in the cluster's temp dir.
+    """
+    RANDOM_BITS = 64
+
+    base = self.join(basedir or self._temp_dir, subdir)
+    if not self.isdir(base):
+      self.mkdir(base)
+
+    candidate = self.join(base, "%s.%s" % (filename, suffix))
+    return candidate
+
   def exists(self):
     raise NotImplementedError(_("%(function)s has not been implemented.") % {'function': 'exists'})
 
@@ -548,24 +580,6 @@ class HadoopFileSystem(Hdfs):
     path = encode_fs_path(path)
     self.nn_client.chown(self.request_context, normpath(path), user, group)
 
-  @_coerce_exceptions
-  def mktemp(self, subdir='', prefix='tmp'):
-    """
-    mktemp(prefix) ->  <temp_dir>/subdir/prefix.<rand>
-    Return a unique temporary filename with prefix in the cluster's temp dir.
-    """
-    RANDOM_BITS = 64
-
-    base = self.join(self._temp_dir, subdir)
-    if not self.isdir(base):
-      self.mkdir(base)
-
-    while True:
-      name = prefix + '.' + str(random.getrandbits(RANDOM_BITS))
-      candidate = self.join(base, name)
-      if not self.exists(candidate):
-        return candidate
-
   @_coerce_exceptions
   def get_namenode_info(self):
     (capacity, used, available) = self.nn_client.df(self.request_context)

+ 4 - 7
desktop/libs/hadoop/src/hadoop/fs/upload.py

@@ -46,7 +46,7 @@ class HDFStemporaryUploadedFile(object):
   A temporary HDFS file to store upload data.
   This class does not have any file read methods.
   """
-  def __init__(self, request, name):
+  def __init__(self, request, name, destination):
     self.name = name
     self.size = None
     self._do_cleanup = False
@@ -65,12 +65,8 @@ class HDFStemporaryUploadedFile(object):
     self._fs.setuser(self._fs.DEFAULT_USER)
     self._fs.setuser(self._fs.superuser)
 
-    self._path = self._fs.mktemp(
-        subdir='hue-uploads',
-        prefix='tmp.%s' % (request.environ['REMOTE_ADDR'],))
+    self._path = self._fs.mkswap(name, suffix='tmp', basedir=destination)
 
-    # Make the tmp dir 0777
-    self._fs.chmod(self._fs.dirname(self._path), 0777)
     self._file = self._fs.open(self._path, 'w')
     self._do_cleanup = True
 
@@ -122,6 +118,7 @@ class HDFSfileUploadHandler(FileUploadHandler):
     self._file = None
     self._starttime = 0
     self._activated = False
+    self._destination = request.GET.get('dest', None)
     # Need to directly modify FileUploadHandler.chunk_size
     FileUploadHandler.chunk_size = UPLOAD_CHUNK_SIZE.get()
 
@@ -132,7 +129,7 @@ class HDFSfileUploadHandler(FileUploadHandler):
     #       running the auth middleware.
     if field_name.upper().startswith('HDFS'):
       try:
-        self._file = HDFStemporaryUploadedFile(self.request, file_name)
+        self._file = HDFStemporaryUploadedFile(self.request, file_name, self._destination)
       except (HDFSerror, IOError), ex:
         LOG.error("Not using HDFS upload handler: %s" % (ex,))
         return

+ 0 - 17
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -248,23 +248,6 @@ class WebHdfs(Hdfs):
     if not success:
       raise IOError("Mkdir failed: %s" % (smart_str(path),))
 
-  def mktemp(self, subdir='', prefix='tmp'):
-    """
-    mktemp(subdir, prefix) ->  <temp_dir>/subdir/prefix.<rand>
-    Return a unique temporary filename with prefix in the cluster's temp dir.
-    """
-    RANDOM_BITS = 64
-
-    base = self.join(self._temp_dir, subdir)
-    if not self.isdir(base):
-      self.mkdir(base)
-
-    while True:
-      name = "%s.%s" % (prefix, random.getrandbits(RANDOM_BITS))
-      candidate = self.join(base, name)
-      if not self.exists(candidate):
-        return candidate
-
   def rename(self, old, new):
     """rename(old, new)"""
     old = Hdfs.normpath(old)