Browse Source

HUE-794 [desktop,filebrowser] Backend for recursive chmod/chown

- Added test cases in two applications: desktop and filebrowser
- Added 'recursive' checkbox at the bottom of modal box
abec 13 years ago
parent
commit
6380eb2e91

+ 4 - 2
apps/filebrowser/src/filebrowser/forms.py

@@ -82,9 +82,10 @@ class ChownForm(forms.Form):
   # These could be "ChoiceFields", listing only users and groups
   # These could be "ChoiceFields", listing only users and groups
   # that the current user has permissions for.
   # that the current user has permissions for.
   user = CharField(label=_("User"), min_length=1)
   user = CharField(label=_("User"), min_length=1)
-  user_other = CharField(label="OtherUser", min_length=1, required=False)
+  user_other = CharField(label=_("OtherUser"), min_length=1, required=False)
   group = CharField(label=_("Group"), min_length=1)
   group = CharField(label=_("Group"), min_length=1)
-  group_other = CharField(label="OtherGroup", min_length=1, required=False)
+  group_other = CharField(label=_("OtherGroup"), min_length=1, required=False)
+  recursive = BooleanField(label=_("Recursive"), required=False)
 
 
   def __init__(self, *args, **kwargs):
   def __init__(self, *args, **kwargs):
     super(ChownForm, self).__init__(*args, **kwargs)
     super(ChownForm, self).__init__(*args, **kwargs)
@@ -108,6 +109,7 @@ class ChmodForm(forms.Form):
   other_write = BooleanField(required=False)
   other_write = BooleanField(required=False)
   other_execute = BooleanField(required=False)
   other_execute = BooleanField(required=False)
   sticky = BooleanField(required=False)
   sticky = BooleanField(required=False)
+  recursive = BooleanField(required=False)
 
 
   names = ("user_read", "user_write", "user_execute",
   names = ("user_read", "user_write", "user_execute",
       "group_read", "group_write", "group_execute",
       "group_read", "group_write", "group_execute",

+ 5 - 0
apps/filebrowser/src/filebrowser/templates/chmod.mako

@@ -72,6 +72,11 @@ from django.utils.translation import ugettext as _
                 <td class="center">${edit.render_field(form["sticky"], tag="checkbox", button_text=" ", nolabel=True)}</td>
                 <td class="center">${edit.render_field(form["sticky"], tag="checkbox", button_text=" ", nolabel=True)}</td>
                 <td>&nbsp;</td>
                 <td>&nbsp;</td>
             </tr>
             </tr>
+            <tr>
+                <td><strong>${_('Recursive')}</strong></td>
+                <td colspan="3">&nbsp;</td>
+                <td class="center">${edit.render_field(form["recursive"], tag="checkbox", button_text=" ", nolabel=True)}</td>
+                <td>&nbsp;</td>
             </tbody>
             </tbody>
         </table>
         </table>
     </div>
     </div>

+ 2 - 0
apps/filebrowser/src/filebrowser/templates/chown.mako

@@ -83,6 +83,8 @@ from django.utils.translation import ugettext as _
             % else:
             % else:
             ${ selection("group", [group for group in form.all_groups if group in extra_params['current_user'].get_groups()], extract_field_data(form["group"])) }
             ${ selection("group", [group for group in form.all_groups if group in extra_params['current_user'].get_groups()], extract_field_data(form["group"])) }
             % endif
             % endif
+            <br />
+            <label style="display: inline;">${_('Recursive')}</label>${edit.render_field(form["recursive"], tag="checkbox", button_text=" ", nolabel=True)}
         </div>
         </div>
 
 
 
 

+ 7 - 3
apps/filebrowser/src/filebrowser/views.py

@@ -39,6 +39,7 @@ from django.core import urlresolvers, serializers
 from django.template.defaultfilters import stringformat, filesizeformat
 from django.template.defaultfilters import stringformat, filesizeformat
 from django.http import Http404, HttpResponse, HttpResponseNotModified
 from django.http import Http404, HttpResponse, HttpResponseNotModified
 from django.views.static import was_modified_since
 from django.views.static import was_modified_since
+from django.utils.functional import curry
 from django.utils.http import http_date, urlquote
 from django.utils.http import http_date, urlquote
 from django.utils.html import escape
 from django.utils.html import escape
 from cStringIO import StringIO
 from cStringIO import StringIO
@@ -845,7 +846,8 @@ def rmtree(request):
 def chmod(request):
 def chmod(request):
     # mode here is abused: on input, it's a string, but when retrieved,
     # mode here is abused: on input, it's a string, but when retrieved,
     # it's an int.
     # it's an int.
-    return generic_op(ChmodForm, request, request.fs.chmod, ["path", "mode"], "path", template="chmod.mako")
+    op = curry(request.fs.chmod, recursive=request.POST.get('recursive', False))
+    return generic_op(ChmodForm, request, op, ["path", "mode"], "path", template="chmod.mako")
 
 
 
 
 def chown(request):
 def chown(request):
@@ -858,8 +860,10 @@ def chown(request):
     if request.POST.get("group") == "__other__":
     if request.POST.get("group") == "__other__":
         args[2] = "group_other"
         args[2] = "group_other"
 
 
-    return generic_op(ChownForm, request, request.fs.chown, args, "path", template="chown.mako",
-                      extra_params=dict(current_user=request.user, superuser=request.fs.superuser))
+    op = curry(request.fs.chown, recursive=request.POST.get('recursive', False))
+    return generic_op(ChownForm, request, op, args, "path", template="chown.mako",
+                      extra_params=dict(current_user=request.user, 
+                      superuser=request.fs.superuser))
 
 
 
 
 def upload_flash(request):
 def upload_flash(request):

+ 53 - 2
apps/filebrowser/src/filebrowser/views_test.py

@@ -30,7 +30,7 @@ import urlparse
 
 
 from django.utils.encoding import smart_str
 from django.utils.encoding import smart_str
 from nose.plugins.attrib import attr
 from nose.plugins.attrib import attr
-from nose.tools import assert_true, assert_false, assert_equal, assert_raises
+from nose.tools import assert_true, assert_false, assert_equal, assert_raises, assert_not_equal
 
 
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access
 from desktop.lib.test_utils import grant_access
@@ -75,6 +75,49 @@ def test_mkdir_singledir():
       pass      # Don't let cleanup errors mask earlier failures
       pass      # Don't let cleanup errors mask earlier failures
 
 
 
 
+@attr('requires_hadoop')
+def test_chmod():
+  cluster = pseudo_hdfs4.shared_cluster()
+
+  try:
+    c = make_logged_in_client(cluster.superuser)
+    cluster.fs.setuser(cluster.superuser)
+
+    PATH = "/chmod_test"
+    SUBPATH = PATH + '/test'
+    cluster.fs.mkdir(SUBPATH)
+
+    permissions = ('user_read', 'user_write', 'user_execute',
+        'group_read', 'group_write', 'group_execute',
+        'other_read', 'other_write', 'other_execute',
+        'sticky') # Order matters!
+
+    # Get current mode, change mode, check mode
+    # Start with checking current mode
+    assert_not_equal(041777, int(cluster.fs.stats(PATH)["mode"]))
+
+    # Setup post data
+    permissions_dict = dict( zip(permissions, [True]*len(permissions)) )
+    kwargs = {'path': PATH}
+    kwargs.update(permissions_dict)
+
+    # Set 1777, then check permissions of dirs
+    response = c.post("/filebrowser/chmod", kwargs)
+    assert_equal(041777, int(cluster.fs.stats(PATH)["mode"]))
+
+    # Now do the above recursively
+    assert_not_equal(041777, int(cluster.fs.stats(SUBPATH)["mode"]))
+    kwargs['recursive'] = True
+    response = c.post("/filebrowser/chmod", kwargs)
+    assert_equal(041777, int(cluster.fs.stats(SUBPATH)["mode"]))
+
+  finally:
+    try:
+      cluster.fs.rmtree(PATH)     # Clean up
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
+
+
 @attr('requires_hadoop')
 @attr('requires_hadoop')
 def test_chmod_sticky():
 def test_chmod_sticky():
   cluster = pseudo_hdfs4.shared_cluster()
   cluster = pseudo_hdfs4.shared_cluster()
@@ -134,6 +177,15 @@ def test_chown():
   c.post("/filebrowser/chown", dict(path=PATH, user="__other__", user_other="z", group="y"))
   c.post("/filebrowser/chown", dict(path=PATH, user="__other__", user_other="z", group="y"))
   assert_equal("z", cluster.fs.stats(PATH)["user"])
   assert_equal("z", cluster.fs.stats(PATH)["user"])
 
 
+  # Now check recursive
+  SUBPATH = PATH + '/test'
+  cluster.fs.mkdir(SUBPATH)
+  c.post("/filebrowser/chown", dict(path=PATH, user="x", group="y", recursive=True))
+  assert_equal("x", cluster.fs.stats(SUBPATH)["user"])
+  assert_equal("y", cluster.fs.stats(SUBPATH)["group"])
+  c.post("/filebrowser/chown", dict(path=PATH, user="__other__", user_other="z", group="y", recursive=True))
+  assert_equal("z", cluster.fs.stats(SUBPATH)["user"])
+
   # Make sure that the regular user chown form doesn't have useless fields,
   # Make sure that the regular user chown form doesn't have useless fields,
   # and that the superuser's form has all the fields it could dream of.
   # and that the superuser's form has all the fields it could dream of.
   PATH = '/filebrowser/chown-regular-user'
   PATH = '/filebrowser/chown-regular-user'
@@ -145,7 +197,6 @@ def test_chown():
   response = c.get('/filebrowser/chown', dict(path=PATH, user='chown_test', group='chown_test'))
   response = c.get('/filebrowser/chown', dict(path=PATH, user='chown_test', group='chown_test'))
   assert_false('<option value="__other__"' in response.content)
   assert_false('<option value="__other__"' in response.content)
 
 
-
 @attr('requires_hadoop')
 @attr('requires_hadoop')
 def test_listdir():
 def test_listdir():
   cluster = pseudo_hdfs4.shared_cluster()
   cluster = pseudo_hdfs4.shared_cluster()

+ 79 - 1
desktop/libs/hadoop/src/hadoop/fs/test_webhdfs.py

@@ -18,7 +18,7 @@
 """
 """
 Tests for Hadoop FS.
 Tests for Hadoop FS.
 """
 """
-from nose.tools import assert_false, assert_true, assert_equals, assert_raises
+from nose.tools import assert_false, assert_true, assert_equals, assert_raises, assert_not_equals
 from nose.plugins.attrib import attr
 from nose.plugins.attrib import attr
 import logging
 import logging
 import posixfile
 import posixfile
@@ -277,3 +277,81 @@ def test_threadedness():
   assert_equals("alpha", fs.user)
   assert_equals("alpha", fs.user)
   fs.setuser("gamma")
   fs.setuser("gamma")
   assert_equals("gamma", fs.user)
   assert_equals("gamma", fs.user)
+
+@attr('requires_hadoop')
+def test_chmod():
+  # Create a test directory with
+  # a subdirectory and a few files.
+  dir1 = '/test'
+  subdir1 = dir1 + '/test1'
+  file1 = subdir1 + '/test1.txt'
+  cluster = pseudo_hdfs4.shared_cluster()
+  fs = cluster.fs
+  fs.setuser(cluster.superuser)
+  try:
+    fs.mkdir(subdir1)
+    f = fs.open(file1, "w")
+    f.write("hello")
+    f.close()
+
+    # Check currrent permissions are not 777 (666 for file)
+    fs.chmod(dir1, 01000, recursive=True)
+    assert_equals(041000, fs.stats(dir1).mode)
+    assert_equals(041000, fs.stats(subdir1).mode)
+    assert_equals(0100000, fs.stats(file1).mode)
+
+    # Chmod non-recursive
+    fs.chmod(dir1, 01222, recursive=False)
+    assert_equals(041222, fs.stats(dir1).mode)
+    assert_equals(041000, fs.stats(subdir1).mode)
+    assert_equals(0100000, fs.stats(file1).mode)
+
+    # Chmod recursive
+    fs.chmod(dir1, 01444, recursive=True)
+    assert_equals(041444, fs.stats(dir1).mode)
+    assert_equals(041444, fs.stats(subdir1).mode)
+    assert_equals(0100444, fs.stats(file1).mode)
+  finally:
+    try:
+      fs.rmtree(dir1)
+    finally:
+      pass
+
+@attr('requires_hadoop')
+def test_chown():
+  # Create a test directory with
+  # a subdirectory and a few files.
+  dir1 = '/test'
+  subdir1 = dir1 + '/test1'
+  file1 = subdir1 + '/test1.txt'
+  cluster = pseudo_hdfs4.shared_cluster()
+  fs = cluster.fs
+  fs.setuser(cluster.superuser)
+  try:
+    fs.mkdir(subdir1)
+    f = fs.open(file1, "w")
+    f.write("hello")
+    f.close()
+
+    # Check currrent owners are not user test
+    LOG.info(str(fs.stats(dir1).__dict__))
+    assert_not_equals('test', fs.stats(dir1).user)
+    assert_not_equals('test', fs.stats(subdir1).user)
+    assert_not_equals('test', fs.stats(file1).user)
+
+    # Chown non-recursive
+    fs.chown(dir1, 'test', recursive=False)
+    assert_equals('test', fs.stats(dir1).user)
+    assert_not_equals('test', fs.stats(subdir1).user)
+    assert_not_equals('test', fs.stats(file1).user)
+
+    # Chown recursive
+    fs.chown(dir1, 'test', recursive=True)
+    assert_equals('test', fs.stats(dir1).user)
+    assert_equals('test', fs.stats(subdir1).user)
+    assert_equals('test', fs.stats(file1).user)
+  finally:
+    try:
+      fs.rmtree(dir1)
+    finally:
+      pass

+ 30 - 7
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -154,7 +154,7 @@ class WebHdfs(Hdfs):
     Get directory entry names without stats.
     Get directory entry names without stats.
     """
     """
     dirents = self.listdir_stats(path, glob)
     dirents = self.listdir_stats(path, glob)
-    return [ Hdfs.basename(x.path) for x in dirents ]
+    return [Hdfs.basename(x.path) for x in dirents]
 
 
   def get_content_summary(self, path):
   def get_content_summary(self, path):
     """
     """
@@ -291,8 +291,22 @@ class WebHdfs(Hdfs):
     for dirent in ls:
     for dirent in ls:
       self.rename(Hdfs.join(old_dir, dirent), Hdfs.join(new_dir, dirent))
       self.rename(Hdfs.join(old_dir, dirent), Hdfs.join(new_dir, dirent))
 
 
-  def chown(self, path, user=None, group=None):
-    """chown(path, user=None, group=None)"""
+  def _listdir_r(self, path, glob=None):
+    """
+    _listdir_r(path, glob=None) -> [ entry names ]
+
+    Get directory entry names without stats, recursively.
+    """
+    paths = [path]
+    while paths:
+      path = paths.pop()
+      if self.isdir(path):
+        hdfs_paths = self.listdir_stats(path, glob)
+        paths[:0] = [x.path for x in hdfs_paths]
+      yield path
+
+  def chown(self, path, user=None, group=None, recursive=False):
+    """chown(path, user=None, group=None, recursive=False)"""
     path = Hdfs.normpath(path)
     path = Hdfs.normpath(path)
     params = self._getparams()
     params = self._getparams()
     params['op'] = 'SETOWNER'
     params['op'] = 'SETOWNER'
@@ -300,11 +314,16 @@ class WebHdfs(Hdfs):
       params['owner'] = user
       params['owner'] = user
     if group is not None:
     if group is not None:
       params['group'] = group
       params['group'] = group
-    self._root.put(path, params)
+    if recursive:
+      for xpath in self._listdir_r(path):
+        self._root.put(xpath, params)
+    else:
+      self._root.put(path, params)
+
 
 
-  def chmod(self, path, mode):
+  def chmod(self, path, mode, recursive=False):
     """
     """
-    chmod(path, mode)
+    chmod(path, mode, recursive=False)
 
 
     `mode' should be an octal integer or string.
     `mode' should be an octal integer or string.
     """
     """
@@ -312,7 +331,11 @@ class WebHdfs(Hdfs):
     params = self._getparams()
     params = self._getparams()
     params['op'] = 'SETPERMISSION'
     params['op'] = 'SETPERMISSION'
     params['permission'] = safe_octal(mode)
     params['permission'] = safe_octal(mode)
-    self._root.put(path, params)
+    if recursive:
+      for xpath in self._listdir_r(path):
+        self._root.put(xpath, params)
+    else:
+      self._root.put(path, params)
 
 
   def get_home_dir(self):
   def get_home_dir(self):
     """get_home_dir() -> Home directory for the current user"""
     """get_home_dir() -> Home directory for the current user"""