Browse Source

[webhdfs] Handle quoting of weird filename characters

bc Wong 13 years ago
parent
commit
eadc33b7b3

+ 0 - 8
apps/filebrowser/src/filebrowser/views.py

@@ -62,10 +62,6 @@ MAX_FILEEDITOR_SIZE = 256 * 1024
 
 logger = logging.getLogger(__name__)
 
-def _unquote_path(path):
-    """Normalizes paths."""
-    return urllib.unquote(path)
-
 
 def _file_reader(fh):
     """Generator that reads a file, chunk-by-chunk."""
@@ -83,7 +79,6 @@ def download(request, path):
 
     This is inspired by django.views.static.serve.
     """
-    path = _unquote_path(path)
     if not request.fs.exists(path):
         raise Http404("File not found: %s" % escape(path))
     if not request.fs.isfile(path):
@@ -292,7 +287,6 @@ def listdir(request, path, chooser):
 
     Intended to be called via view().
     """
-    path = _unquote_path(path)
     if not request.fs.isdir(path):
         raise PopupException("Not a directory: %s" % (path,))
 
@@ -379,7 +373,6 @@ def stat(request, path):
     Intended for use via AJAX (and hence doesn't provide
     an HTML view).
     """
-    path = _unquote_path(path)
     if not request.fs.exists(path):
         raise Http404("File not found: %s" % escape(path))
     stats = request.fs.stats(path)
@@ -401,7 +394,6 @@ def display(request, path):
     sequence files, decompress gzipped text files, etc.).
     There exists a python-magic package to interface with libmagic.
     """
-    path = _unquote_path(path)
     if not request.fs.isfile(path):
         raise PopupException("Not a file: '%s'" % (path,))
 

+ 24 - 12
apps/filebrowser/src/filebrowser/views_test.py

@@ -31,6 +31,7 @@ except ImportError:
   import simplejson as json
 
 import logging
+import urlparse
 
 LOG = logging.getLogger(__name__)
 
@@ -66,7 +67,7 @@ def test_chown():
 def test_listdir():
   cluster = pseudo_hdfs4.shared_cluster()
   try:
-    c = make_logged_in_client()
+    c = make_logged_in_client(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
 
     # These paths contain non-ascii characters. Your editor will need the
@@ -78,22 +79,32 @@ def test_listdir():
     orig_paths = [
       u'greek-Ελληνικά',
       u'chinese-漢語',
-      'listdir',
-      'non-utf-8-(big5)-\xb2\xc4\xa4@\xb6\xa5\xacq',
+      'listdir%20.,<>~`!@#$%^&()_-+="',
     ]
 
     prefix = '/test-filebrowser/'
     for path in orig_paths:
-      cluster.fs.mkdir(prefix + path)
+      c.post('/filebrowser/mkdir', dict(path=prefix, name=path))
+
+    # Read the parent dir
     response = c.get('/filebrowser/view' + prefix)
-    paths = [f['path'] for f in response.context['files']]
-    for path in orig_paths:
-      if isinstance(path, unicode):
-        uni_path = path
-      else:
-        uni_path = unicode(path, 'utf-8', errors='replace')
-      assert_true(prefix + uni_path in paths,
-                  '%s should be in dir listing %s' % (prefix + uni_path, paths))
+    dir_listing = response.context['files']
+    assert_equal(len(orig_paths) + 1, len(dir_listing))
+
+    for dirent in dir_listing:
+      path = dirent['name']
+      if path == '..':
+        continue
+
+      assert_true(path in orig_paths)
+
+      # Drill down into the subdirectory
+      url = urlparse.urlsplit(dirent['url'])[2]
+      resp = c.get(url)
+
+      # We are actually reading a directory
+      assert_equal('..', resp.context['files'][0]['name'],
+                   "'%s' should be a directory" % (path,))
 
     # Delete user's home if there's already something there
     if cluster.fs.isdir("/user/test"):
@@ -102,6 +113,7 @@ def test_listdir():
 
     # test's home directory now exists. Should be returned.
     cluster.fs.mkdir('/user/test')
+    c = make_logged_in_client()
     response = c.get('/filebrowser/view/test-filebrowser/')
     assert_equal(response.context['home_directory'], '/user/test')
   finally:

+ 2 - 1
desktop/core/src/desktop/lib/rest/http_client.py

@@ -139,7 +139,7 @@ class HttpClient(object):
     """
     Submit an HTTP request.
     @param http_method: GET, POST, PUT, DELETE
-    @param path: The path of the resource.
+    @param path: The path of the resource. Unsafe characters will be quoted.
     @param params: Key-value parameter data.
     @param data: The data to attach to the body of the request.
     @param headers: The headers to set for this request.
@@ -147,6 +147,7 @@ class HttpClient(object):
     @return: The result of urllib2.urlopen()
     """
     # Prepare URL and params
+    path = urllib.quote(smart_str(path))
     url = self._make_url(path, params)
     if http_method in ("GET", "DELETE"):
       if data is not None:

+ 1 - 1
desktop/libs/hadoop/src/hadoop/fs/test_webhdfs.py

@@ -165,7 +165,7 @@ def test_i18n_namespace():
     listing = cluster.fs.listdir(parent)
     assertion(name in listing, "%s should be in %s" % (name, listing))
 
-  name = u'pt-Olá_ch-你好_ko-안녕_ru-Здравствуйте'
+  name = u'''pt-Olá_ch-你好_ko-안녕_ru-Здравствуйте%20,.<>~`!@#$%^&()_-+='"'''
   prefix = '/tmp/i18n'
   dir_path = '%s/%s' % (prefix, name)
   file_path = '%s/%s' % (dir_path, name)