Browse Source

[core] Increase upload chunks from 64KB to 64MB

Make it configurable
Romain Rigaux 13 years ago
parent
commit
2b44f592c9

+ 6 - 0
desktop/libs/hadoop/src/hadoop/conf.py

@@ -59,6 +59,12 @@ CREDENTIALS_MERGER_JAR = Config("hadoop_credentials_merger_jar",
                 root=os.path.join(os.path.dirname(__file__), '..', '..', 'credentials-merger', 'java-lib')),
   private=True)
 
+UPLOAD_CHUNK_SIZE = Config(
+  key="upload_chunk_size",
+  help="Size, in bytes, of the 'chunks' Django should store into memory and feed into the handler. Default is 64MB.",
+  type=int,
+  default=1024 * 1024 * 64)
+
 
 HDFS_CLUSTERS = UnspecifiedConfigSection(
   "hdfs_clusters",

+ 3 - 0
desktop/libs/hadoop/src/hadoop/fs/upload.py

@@ -32,6 +32,7 @@ import time
 from django.core.files.uploadhandler import \
     FileUploadHandler, StopFutureHandlers, StopUpload
 import hadoop.cluster
+from hadoop.conf import UPLOAD_CHUNK_SIZE
 
 UPLOAD_SUBDIR = 'hue-uploads'
 LOG = logging.getLogger(__name__)
@@ -121,6 +122,8 @@ class HDFSfileUploadHandler(FileUploadHandler):
     self._file = None
     self._starttime = 0
     self._activated = False
+    # Need to directly modify FileUploadHandler.chunk_size
+    FileUploadHandler.chunk_size = UPLOAD_CHUNK_SIZE.get()
 
   def new_file(self, field_name, file_name, *args, **kwargs):
     # Detect "HDFS" in the field name.

+ 2 - 2
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -32,6 +32,7 @@ from hadoop.fs import normpath, SEEK_SET, SEEK_CUR, SEEK_END
 from hadoop.fs.hadoopfs import Hdfs
 from hadoop.fs.exceptions import WebHdfsException
 from hadoop.fs.webhdfs_types import WebHdfsStat, WebHdfsContentSummary
+from hadoop.conf import UPLOAD_CHUNK_SIZE
 
 import hadoop.conf
 
@@ -420,11 +421,10 @@ class WebHdfs(Hdfs):
     if self.isdir(dst):
       raise IOError(errno.INVAL, "Copy dst '%s' is a directory" % (dst,))
 
-    CHUNK_SIZE = 65536
     offset = 0
 
     while True:
-      data = self.read(src, offset, CHUNK_SIZE)
+      data = self.read(src, offset, UPLOAD_CHUNK_SIZE.get())
       if offset == 0:
         self.create(dst,
                     overwrite=True,