Browse Source

HUE-2321 [fb] Support HDFS UMASK in octal

Romain Rigaux 11 years ago
parent
commit
c306872

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -586,6 +586,9 @@
       # Change this if your HDFS cluster is Kerberos-secured
       # Change this if your HDFS cluster is Kerberos-secured
       ## security_enabled=false
       ## security_enabled=false
 
 
+      # Default umask for file and directory creation, specified in an octal value.
+      ## umask=022
+
   # Configuration for YARN (MR2)
   # Configuration for YARN (MR2)
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[yarn_clusters]]
   [[yarn_clusters]]

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -593,6 +593,9 @@
       # Change this if your HDFS cluster is Kerberos-secured
       # Change this if your HDFS cluster is Kerberos-secured
       ## security_enabled=false
       ## security_enabled=false
 
 
+      # Default umask for file and directory creation, specified in an octal value.
+      ## umask=022
+
   # Configuration for YARN (MR2)
   # Configuration for YARN (MR2)
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[yarn_clusters]]
   [[yarn_clusters]]

+ 6 - 7
desktop/libs/hadoop/src/hadoop/conf.py

@@ -43,9 +43,8 @@ def find_file_recursive(desired_glob, root):
 def coerce_umask(umask):
 def coerce_umask(umask):
   if len(umask) < 4:
   if len(umask) < 4:
     umask = "1" + umask
     umask = "1" + umask
-    return int(umask)
-  else:
-    return int(umask)
+
+  return int(umask, 8)
 
 
 
 
 UPLOAD_CHUNK_SIZE = Config(
 UPLOAD_CHUNK_SIZE = Config(
@@ -69,16 +68,16 @@ HDFS_CLUSTERS = UnspecifiedConfigSection(
                          help="The URL to WebHDFS/HttpFS service. Defaults to " +
                          help="The URL to WebHDFS/HttpFS service. Defaults to " +
                          "the WebHDFS URL on the NameNode.",
                          "the WebHDFS URL on the NameNode.",
                          type=str, default="http://localhost:50070/webhdfs/v1"),
                          type=str, default="http://localhost:50070/webhdfs/v1"),
-      NN_KERBEROS_PRINCIPAL=Config("nn_kerberos_principal", help="Kerberos principal for NameNode",
+      NN_KERBEROS_PRINCIPAL=Config("nn_kerberos_principal", help="Kerberos principal for NameNode", # Unused
                                    default="hdfs", type=str),
                                    default="hdfs", type=str),
-      DN_KERBEROS_PRINCIPAL=Config("dn_kerberos_principal", help="Kerberos principal for DataNode",
+      DN_KERBEROS_PRINCIPAL=Config("dn_kerberos_principal", help="Kerberos principal for DataNode", # Unused
                                    default="hdfs", type=str),
                                    default="hdfs", type=str),
       SECURITY_ENABLED=Config("security_enabled", help="Is running with Kerberos authentication",
       SECURITY_ENABLED=Config("security_enabled", help="Is running with Kerberos authentication",
                               default=False, type=coerce_bool),
                               default=False, type=coerce_bool),
       TEMP_DIR=Config("temp_dir", help="HDFS directory for temporary files",
       TEMP_DIR=Config("temp_dir", help="HDFS directory for temporary files",
                       default='/tmp', type=str),
                       default='/tmp', type=str),
-      UMASK=Config("umask", help="Umask for creating files/directories in hdfs",
-                      default='1022', type=coerce_umask),
+      UMASK=Config("umask", help="Default umask for file and directory creation, specified in an octal value",
+                   default='022', type=coerce_umask),
     )
     )
   )
   )
 )
 )

+ 71 - 9
desktop/libs/hadoop/src/hadoop/fs/test_webhdfs.py

@@ -135,23 +135,85 @@ class WebhdfsTests(unittest.TestCase):
     assert_raises(WebHdfsException, f.read)
     assert_raises(WebHdfsException, f.read)
     assert_raises(IOError, fs.open, "/test/doesnotexist.txt")
     assert_raises(IOError, fs.open, "/test/doesnotexist.txt")
 
 
-  
+
   def test_umask(self):
   def test_umask(self):
     fs = self.cluster.fs
     fs = self.cluster.fs
+
+    prefix = '/tmp/test_umask'
+    fs_umask = fs._umask
+    fs._umask = 01022
+
     try:
     try:
-      clear = UMASK.set_for_testing("0077")
-      test_dir = '/umask_test_dir'
+      test_dir = prefix + '/umask_test_dir'
       fs.mkdir(test_dir)
       fs.mkdir(test_dir)
-      test_file = '/umask_test.txt'
-      f.open(test_file, "w")
+
+      test_file = prefix + '/umask_test.txt'
+      f = fs.open(test_file, "w")
       f.write("foo")
       f.write("foo")
       f.close()
       f.close()
 
 
+      # Check currrent permissions are 777 (666 for file)
+      assert_equals('40755', '%o' % fs.stats(test_dir).mode)
+      assert_equals('100644', '%o' % fs.stats(test_file).mode)
+    finally:
+      fs._umask = fs_umask
+
+    fs_umask = fs._umask
+    fs._umask = 0077
+    prefix += '/2'
+
+    try:
+      test_dir = prefix + '/umask_test_dir'
+      fs.mkdir(test_dir)
+
+      test_file = prefix + '/umask_test.txt'
+      fs.create(test_file)
+
       # Check currrent permissions are not 777 (666 for file)
       # Check currrent permissions are not 777 (666 for file)
-      assert_equals(1700, fs.stats(test_dir).mode)
-      assert_equals(1700, fs.stats(test_file).mode)
+      assert_equals('41700', '%o' % fs.stats(test_dir).mode)
+      assert_equals('100600', '%o' % fs.stats(test_file).mode)
+    finally:
+      fs._umask = fs_umask
+
+
+  def test_umask_overriden(self):
+    fs = self.cluster.fs
+
+    prefix = '/tmp/test_umask_overriden'
+    fs_umask = fs._umask
+    fs._umask = 01022
+
+    try:
+      test_dir = prefix + '/umask_test_dir'
+      fs.mkdir(test_dir, 0333)
+
+      test_file = prefix + '/umask_test.txt'
+      fs.create(test_file, permission=0333)
+
+      assert_equals('40333', '%o' % fs.stats(test_dir).mode)
+      assert_equals('100333', '%o' % fs.stats(test_file).mode)
+    finally:
+      fs._umask = fs_umask
+
+
+  def test_umask_without_sticky(self):
+    fs = self.cluster.fs
+
+    prefix = '/tmp/test_umask_without_sticky'
+    fs_umask = fs._umask
+    fs._umask = 022
+
+    try:
+      test_dir = prefix + '/umask_test_dir'
+      fs.mkdir(test_dir)
+
+      test_file = prefix + '/umask_test.txt'
+      fs.create(test_file)
+
+      assert_equals('41755', '%o' % fs.stats(test_dir).mode)
+      assert_equals('100644', '%o' % fs.stats(test_file).mode)
     finally:
     finally:
-      clear()
+      fs._umask = fs_umask
 
 
 
 
   def test_copy_remote_dir(self):
   def test_copy_remote_dir(self):
@@ -190,7 +252,7 @@ class WebhdfsTests(unittest.TestCase):
     for stat in dest_stat:
     for stat in dest_stat:
       assert_equals('testcopy', stat.user)
       assert_equals('testcopy', stat.user)
       assert_equals('testcopy', stat.group)
       assert_equals('testcopy', stat.group)
-      assert_equals('100755', '%o' % stat.mode)
+      assert_equals('100644', '%o' % stat.mode)
 
 
   def test_two_files_open(self):
   def test_two_files_open(self):
     """
     """

+ 14 - 26
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -59,7 +59,7 @@ class WebHdfs(Hdfs):
                hdfs_superuser=None,
                hdfs_superuser=None,
                security_enabled=False,
                security_enabled=False,
                temp_dir="/tmp",
                temp_dir="/tmp",
-               umask=1022):
+               umask=01022):
     self._url = url
     self._url = url
     self._superuser = hdfs_superuser
     self._superuser = hdfs_superuser
     self._security_enabled = security_enabled
     self._security_enabled = security_enabled
@@ -344,11 +344,8 @@ class WebHdfs(Hdfs):
     params['op'] = 'MKDIRS'
     params['op'] = 'MKDIRS'
 
 
     if mode is None:
     if mode is None:
-      params['permission'] = self.getDefaultFilePerms()
-      LOG.debug("No permissions set, defaulting to umask: %s" % params['permission'])
-    else:
-      params['permission'] = safe_octal(mode)
-      LOG.debug("Permissions set, using: %s" % params['permission'])
+      mode = self.getDefaultDirPerms()
+    params['permission'] = safe_octal(mode)
 
 
     success = self._root.put(path, params)
     success = self._root.put(path, params)
     if not success:
     if not success:
@@ -455,20 +452,14 @@ class WebHdfs(Hdfs):
 
 
 
 
   def getDefaultFilePerms(self):
   def getDefaultFilePerms(self):
-              umask = int(1777) - int(self.umask)
- 
-              # Below we are making sure that we don't lose the 0 at the beginning
-              # of the permissions
-                
-              if len(str(umask)) < 4:
-                umask = "0" + str(umask)
-                return umask
-              else:
-                return umask
+    return 0666 & (01777 ^ self.umask)
+
+
+  def getDefaultDirPerms(self):
+    return 01777 & (01777 ^ self.umask)
 
 
 
 
-  def create(self, path, overwrite=False, blocksize=None,
-             replication=None, permission=None, data=None):
+  def create(self, path, overwrite=False, blocksize=None, replication=None, permission=None, data=None):
     """
     """
     create(path, overwrite=False, blocksize=None, replication=None, permission=None)
     create(path, overwrite=False, blocksize=None, replication=None, permission=None)
 
 
@@ -484,11 +475,8 @@ class WebHdfs(Hdfs):
     if replication is not None:
     if replication is not None:
       params['replication'] = int(replication)
       params['replication'] = int(replication)
     if permission is None:
     if permission is None:
-      params['permission'] = self.getDefaultFilePerms()
-      LOG.debug("No permissions set, using umask: %s" % params['permission'])
-    else:
-      params['permission'] = safe_octal(permission)
-      LOG.warn("Permissions already set, using: %s" % params['permission'])
+      permission = self.getDefaultFilePerms()
+    params['permission'] = safe_octal(permission)
 
 
     self._invoke_with_redirect('PUT', path, params, data)
     self._invoke_with_redirect('PUT', path, params, data)
 
 
@@ -591,8 +579,8 @@ class WebHdfs(Hdfs):
       owner = self.DEFAULT_USER
       owner = self.DEFAULT_USER
 
 
     if dir_mode is None:
     if dir_mode is None:
-      dir_mode = self.getDefaultFilePerms()
-    LOG.debug("Making directory %s with permissions %s" % (destination, dir_mode))
+      dir_mode = self.getDefaultDirPerms()
+
     self.do_as_user(owner, self.mkdir, destination, mode=dir_mode)
     self.do_as_user(owner, self.mkdir, destination, mode=dir_mode)
 
 
     for stat in self.listdir_stats(source):
     for stat in self.listdir_stats(source):
@@ -653,7 +641,7 @@ class WebHdfs(Hdfs):
           dest = self.join(dest, self.basename(src))
           dest = self.join(dest, self.basename(src))
         else:
         else:
           raise IOError(errno.EEXIST, _("Destination file %s exists and is not a directory.") % dest)
           raise IOError(errno.EEXIST, _("Destination file %s exists and is not a directory.") % dest)
-      
+
       self.do_as_user(owner, self.mkdir, dest, mode=dir_mode)
       self.do_as_user(owner, self.mkdir, dest, mode=dir_mode)
 
 
       # Copy files in 'src' directory to 'dest'.
       # Copy files in 'src' directory to 'dest'.