Эх сурвалжийг харах

HUE-59. i18n support in filebrowser

- Added default encoding to config.
- Added i18n support for hdfs namespace.
- Added i18n support for filebrowser.
- FileViewer and FileEditor has an extra "encoding" field (not exposed yet),
  which they use to decode the file data.
bc Wong 15 жил өмнө
parent
commit
8df848fff3

+ 1 - 0
apps/filebrowser/src/filebrowser/forms.py

@@ -37,6 +37,7 @@ class PathField(CharField):
 class EditorForm(forms.Form):
   path = PathField(label="File to edit")
   contents = CharField(widget=Textarea, label="Contents", required=False)
+  encoding = CharField(label='Encoding', required=False)
 
 class RenameForm(forms.Form):
   op = "rename"

+ 9 - 8
apps/filebrowser/src/filebrowser/templates/display.mako

@@ -1,3 +1,4 @@
+## -*- coding: utf-8 -*-
 ## Licensed to Cloudera, Inc. under one
 ## or more contributor license agreements.  See the NOTICE file
 ## distributed with this work for additional information
@@ -52,16 +53,16 @@
        <a class="fv-download" data-filters="ArtButton" target="_blank" data-icon-styles="{'width': 16, 'height': 16}" href="${url('filebrowser.views.download', path=path_enc)}">Download</a>
        <a class="fv-viewLocation" data-filters="ArtButton" data-icon-styles="{'width': 16, 'height': 16}" href="${url('filebrowser.views.view', path=dirname_enc)}" target="FileBrowser">View File Location</a>
        <a class="ccs-refresh large" data-filters="ArtButton">Refresh</a>
-    </div> 
+    </div>
   </div>
   <div class="fv-navhead">
     % if not view['compression'] or view['compression'] == "none":
       <div class="fv-navStatus">
         <form data-filters="SubmitOnChange" class="fv-changeBytesForm" action="${url('filebrowser.views.view', path=path_enc)}" method="GET">
           <span class="fv-bold">Viewing Bytes:</span><a class="fv-editBytes ccs-inline" title="Enter Bytes"></a>
-          <input data-filters="OverText" name="begin" value="${view['offset'] + 1}"/>
+          <input name="begin" value="${view['offset'] + 1}"/>
           -
-          <input data-filters="OverText" value="${view['end']}" name="end"/> of 
+          <input value="${view['end']}" name="end"/> of
           <span class="fv-bold totalBytes">${stats['size']}</span>
           <span class="fv-stepInfo">(${view['length']} B block size)</span>
           % if view['mode']:
@@ -76,14 +77,14 @@
               first = "style='visibility:hidden'"
               prev = "style='visibility:hidden'"
           else:
-              first = "href='%s?offset=0&length=%d&compression=none' title='1 - %d'" %(base_url, view['length'], min(view['length'], stats['size'])) 
-              prev =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, max(0, view['offset']-view['length']), view['length'], max(0, view['offset']-view['length']) + 1, min(max(0, view['offset'] - view['length']) + view['length'], stats['size'])) 
+              first = "href='%s?offset=0&length=%d&compression=none' title='1 - %d'" %(base_url, view['length'], min(view['length'], stats['size']))
+              prev =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, max(0, view['offset']-view['length']), view['length'], max(0, view['offset']-view['length']) + 1, min(max(0, view['offset'] - view['length']) + view['length'], stats['size']))
           if view['offset'] + view['length'] >= stats['size']:
               next = "style='visibility:hidden'"
               last = "style='visibility:hidden'"
           else:
-              next = "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, view['offset'] + view['length'], view['length'], view['offset'] + view['length'] + 1, view['offset'] + (2 * view['length'])) 
-              last =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, stats['size']-(stats['size'] % view['length']), view['length'], stats['size']-(stats['size'] % view['length']) + 1, stats['size']) 
+              next = "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, view['offset'] + view['length'], view['length'], view['offset'] + view['length'] + 1, view['offset'] + (2 * view['length']))
+              last =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, stats['size']-(stats['size'] % view['length']), view['length'], stats['size']-(stats['size'] % view['length']) + 1, stats['size'])
         %>
         ###DEFINE REL
         <a class="ccs-inline fv-firstBlock" data-filters="PointyTip" ${first}>First Block</a>
@@ -113,7 +114,7 @@
     <div class="right_col">
     %if 'contents' in view:
       % if view['masked_binary_data']:
-      <div class="fv-binaryWarning">Warning: some binary data has been masked out with '.'.</div>
+      <div class="fv-binaryWarning">Warning: some binary data has been masked out with '&#xfffd'.</div>
       % endif
     % endif
       <div class="jframe_padded">

+ 4 - 3
apps/filebrowser/src/filebrowser/templates/edit.mako

@@ -35,15 +35,16 @@
   <div class="alert_popup">
     % for field in form:
       % if len(field.errors):
-       ${str(field.errors) | n}
+       ${unicode(field.errors) | n}
       % endif
     % endfor
   </div>
 % endif
 <form class="no_overflow fe-editForm" method="post" action="${url('filebrowser.views.save_file')}">
-    ${edit.render_field(form["path"],hidden=True, notitle=True)}
+    ${edit.render_field(form["path"], hidden=True, notitle=True)}
+    ${edit.render_field(form["encoding"], hidden=True, notitle=True)}
     <h2 class="ccs-hidden">${form["contents"].label_tag() | n}</h2>
-    <div class="fe-divResize">${str(form["contents"]) | n}</div>
+    <div class="fe-divResize">${unicode(form["contents"]) | n}</div>
     <input class="ccs-hidden" type="submit" name="save" value="saveAs">
     <input class="ccs-hidden" type="submit" name="save" value="save">
 </form>

+ 3 - 3
apps/filebrowser/src/filebrowser/templates/editor_components.mako

@@ -23,10 +23,10 @@
     titlecls = "ccs-hidden"
 %>
   <dt class="${titlecls}">${field.label_tag() | n}</dt>
-  <dd class="${cls}">${str(field) | n}</dd>
+  <dd class="${cls}">${unicode(field) | n}</dd>
   % if len(field.errors):
     <dd class="beeswax_error">
-       ${str(field.errors) | n}
+       ${unicode(field.errors) | n}
      </dd>
   % endif
-</%def>
+</%def>

+ 2 - 1
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -16,6 +16,7 @@
 <%!
 import datetime
 from django.template.defaultfilters import urlencode, stringformat, filesizeformat, date, time
+from django.utils.encoding import iri_to_uri
 %>
 
 
@@ -55,7 +56,7 @@ from django.template.defaultfilters import urlencode, stringformat, filesizeform
             display_name = file['path']
           endif
         %>
-        <% path_enc = urlencode(file['path']) %>
+        <% path_enc = iri_to_uri(urlencode(file['path'])) %>
         <tr class="ccs-no_select fb-item-row ${cls}"
          data-filters="ContextMenu"
          data-context-menu-actions="[{'events':['contextmenu','click:relay(.fb-item-options)'],'menu':'ul.context-menu'}]"

+ 2 - 1
apps/filebrowser/src/filebrowser/templates/saveas.mako

@@ -21,7 +21,7 @@
       <div class="alert_popup">
         % for field in form:
           % if len(field.errors):
-               ${str(field.errors) | n}
+               ${unicode(field.errors) | n}
           % endif
         % endfor
       </div>
@@ -31,6 +31,7 @@
           Please enter the location where you'd like to save the file.
           ${edit.render_field(form["path"], notitle=True)}
           <div>${edit.render_field(form["contents"], hidden=True)}</div>
+	  <div>${edit.render_field(form["encoding"], hidden=True)}</div>
           <input type="submit" class="ccs-hidden" name="save" value="save"/>
       </form>
     </div>

+ 55 - 57
apps/filebrowser/src/filebrowser/views.py

@@ -35,7 +35,7 @@ from django.utils.html import escape
 from cStringIO import StringIO
 from gzip import GzipFile
 
-
+from desktop.lib import i18n
 from desktop.lib.django_util import make_absolute, render_json
 from desktop.lib.django_util import PopupException, format_preserving_redirect
 from filebrowser.lib.rwx import filetype, rwx
@@ -45,7 +45,6 @@ from filebrowser.forms import RenameForm, UploadForm, MkDirForm, RmDirForm, RmTr
 from hadoop.fs import normpath
 from filebrowser.plugin.views import render_with_toolbars
 
-import filebrowser.plugin.toolbar
 
 DEFAULT_CHUNK_SIZE_BYTES = 1024*4 # 4KB
 MAX_CHUNK_SIZE_BYTES = 1024*1024 # 1MB
@@ -55,12 +54,9 @@ DOWNLOAD_CHUNK_SIZE = 32*1024 # 32KB
 # Sentences refer to groups of bytes printed together, within a line.
 BYTES_PER_LINE = 16
 BYTES_PER_SENTENCE = 2
-# If the percentage of non-printable bytes is greater than this, binary mode is
-# enabled by default.
-BINARY_PERCENTAGE = 0.10
 
 # The maximum size the file editor will allow you to edit
-MAX_FILEEDITOR_SIZE=256*1024
+MAX_FILEEDITOR_SIZE = 256*1024
 
 logger = logging.getLogger(__name__)
 
@@ -140,21 +136,21 @@ def edit(request, path, form=None):
   if stats and stats['size'] > MAX_FILEEDITOR_SIZE:
     raise PopupException("File too big to edit: %s" % (path,))
 
-  if stats:
-    f = request.fs.open(path)
-    try:
-      current_contents = f.read()
+  if not form:
+    encoding = request.REQUEST.get('encoding', i18n.get_site_encoding())
+    if stats:
+      f = request.fs.open(path)
       try:
-        current_contents = unicode(current_contents)
-      except UnicodeDecodeError:
-        raise PopupException("File is not unicode text; cannot be edited: %s" % (path,))
-    finally:
-      f.close()
-  else:
-    current_contents = ""
+        try:
+          current_contents = unicode(f.read(), encoding)
+        except UnicodeDecodeError:
+          raise PopupException("File is not encoded in %s; cannot be edited: %s" % (encoding, path))
+      finally:
+        f.close()
+    else:
+      current_contents = u""
 
-  if not form:
-    form = EditorForm(dict(path=path, contents=current_contents))
+    form = EditorForm(dict(path=path, contents=current_contents, encoding=encoding))
 
   data = dict(
     exists=(stats is not None),
@@ -187,16 +183,20 @@ def save_file(request):
     return edit(request, path, form=form)
 
   if request.fs.exists(path):
-    _do_overwrite_save(request.fs, path, form.cleaned_data['contents'])
+    _do_overwrite_save(request.fs, path,
+                       form.cleaned_data['contents'],
+                       form.cleaned_data['encoding'])
   else:
-    _do_newfile_save(request.fs, path, form.cleaned_data['contents'])
+    _do_newfile_save(request.fs, path,
+                     form.cleaned_data['contents'],
+                     form.cleaned_data['encoding'])
 
   request.flash.put('Saved %s.' % os.path.basename(path))
   """ Changing path to reflect the request path of the JFrame that will actually be returned."""
   request.path = urlresolvers.reverse("filebrowser.views.edit", kwargs=dict(path=path))
   return edit(request, path, form)
 
-def _do_overwrite_save(fs, path, data):
+def _do_overwrite_save(fs, path, data, encoding):
   """
   Atomically (best-effort) save the specified data to the given path
   on the filesystem.
@@ -214,7 +214,7 @@ def _do_overwrite_save(fs, path, data):
   new_file = fs.open(path_dest, "w")
   try:
     try:
-      new_file.write(data)
+      new_file.write(data.encode(encoding))
       logging.info("Wrote to " + path_dest)
     finally:
       new_file.close()
@@ -253,7 +253,7 @@ def _do_overwrite_save(fs, path, data):
   fs.rename(path_dest, path)
 
 
-def _do_newfile_save(fs, path, data):
+def _do_newfile_save(fs, path, data, encoding):
   """
   Save data to the path 'path' on the filesystem 'fs'.
 
@@ -261,7 +261,7 @@ def _do_newfile_save(fs, path, data):
   """
   new_file = fs.open(path, "w")
   try:
-    new_file.write(data)
+    new_file.write(data.encode(encoding))
   finally:
     new_file.close()
 
@@ -291,12 +291,14 @@ def listdir(request, path):
     'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
     'cwd_set': True
   }
+
   stats = request.fs.listdir_stats(path)
+
   # Include parent dir, unless at filesystem root.
   if normpath(path) != posixpath.sep:
     parent_stat = request.fs.stats(posixpath.join(path, ".."))
-    # the 'path' field would be absolute, but we want its basename to be
-    # actually '..' for display purposes
+    # The 'path' field would be absolute, but we want its basename to be
+    # actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
     parent_stat['path'] = posixpath.join(path, "..")
     stats.insert(0, parent_stat)
 
@@ -314,10 +316,11 @@ def _massage_stats(request, stats):
   Massage a stats record as returned by the filesystem implementation
   into the format that the views would like it in.
   """
-  normalized = normpath(stats['path'])
+  path = stats['path']
+  normalized = normpath(path)
   return {
     'path': normalized,
-    'name': posixpath.basename(stats['path']),
+    'name': posixpath.basename(path),
     'stats': stats,
     'type': filetype(stats['mode']),
     'rwx': rwx(stats['mode']),
@@ -341,8 +344,10 @@ def display(request, path):
   """
   Implements displaying part of a file.
 
-  GET arguments are length, offset, mode and compression with reasonable
-  defaults chosen.
+  GET arguments are length, offset, mode, compression and encoding
+  with reasonable defaults chosen.
+
+  Note that display by length and offset are on bytes, not on characters.
 
   TODO(philip): Could easily built-in file type detection
   (perhaps using something similar to file(1)), as well
@@ -355,6 +360,7 @@ def display(request, path):
     raise PopupException("Not a file: '%s'" % (path,))
 
   stats = request.fs.stats(path)
+  encoding = request.GET.get('encoding', i18n.get_site_encoding())
 
   # I'm mixing URL-based parameters and traditional
   # HTTP GET parameters, since URL-based parameters
@@ -364,11 +370,11 @@ def display(request, path):
   # because the offset came in via the toolbar manual byte entry.
   end = request.GET.get("end")
   if end:
-      end = int(end)
-  begin = request.GET.get("begin")
+    end = int(end)
+  begin = request.GET.get("begin", 1)
   if begin:
-      # Subtract one to zero index for file read
-      begin = int(begin) - 1
+    # Subtract one to zero index for file read
+    begin = int(begin) - 1
   if end:
     offset = begin
     length = end - begin
@@ -379,7 +385,6 @@ def display(request, path):
     # Display first block by default.
     offset = int(request.GET.get("offset", 0))
 
-
   mode = request.GET.get("mode")
   compression = request.GET.get("compression")
 
@@ -392,8 +397,8 @@ def display(request, path):
   if length > MAX_CHUNK_SIZE_BYTES:
     raise PopupException("Cannot request chunks greater than %d bytes" % MAX_CHUNK_SIZE_BYTES)
 
-
-  if not compression:
+  # Auto gzip detection, unless we are explicitly told to view binary
+  if not compression and mode != 'binary':
     if path.endswith('.gz') and detect_gzip(request.fs.open(path).read(2)):
       compression = 'gzip'
       offset = 0
@@ -413,7 +418,6 @@ def display(request, path):
         raise PopupException("Failed to decompress file")
     finally:
       f.close()
-
   else:
     try:
       f.seek(offset)
@@ -421,30 +425,24 @@ def display(request, path):
     finally:
       f.close()
 
-  masked = None
-
-
-  if not mode:
-    # Auto-detect mode:
-    (mask_count, masked) = xxd.mask_not_printable(contents)
-    if mask_count and float(mask_count)/len(contents) > BINARY_PERCENTAGE:
-      mode = "binary"
-
-    else:
-      mode = "text"
+  # Get contents as string for text mode, or at least try
+  uni_contents = None
+  if not mode or mode == 'text':
+    uni_contents = unicode(contents, encoding, errors='replace')
+    is_binary = uni_contents.find(i18n.REPLACEMENT_CHAR) != -1
+    # Auto-detect mode
+    if not mode:
+      mode = is_binary and 'binary' or 'text'
 
+  # Get contents as bytes
   if mode == "binary":
     xxd_out = list(xxd.xxd(offset, contents, BYTES_PER_LINE, BYTES_PER_SENTENCE))
-  else:
-    # May have been calculated already as part of detection.
-    if not masked:
-      (mask_count, masked) = xxd.mask_not_printable(contents)
 
   dirname = posixpath.dirname(path)
   # Start with index-like data:
   data = _massage_stats(request, request.fs.stats(path))
   # And add a view structure:
-  data["success"] = True;
+  data["success"] = True
   data["view"] = {
     'offset': offset,
     'length': length,
@@ -464,8 +462,8 @@ def display(request, path):
     data['view']['xxd'] = xxd_out
     data['view']['masked_binary_data'] =  False
   else:
-    data['view']['contents'] = masked
-    data['view']['masked_binary_data'] = (mask_count > 0)
+    data['view']['contents'] = uni_contents
+    data['view']['masked_binary_data'] = is_binary
 
   return render_with_toolbars("display.mako", request, data)
 

+ 158 - 61
apps/filebrowser/src/filebrowser/views_test.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -33,14 +34,13 @@ def test_chown():
     c = make_logged_in_client(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
 
-    PATH = "/test-chown"
+    PATH = u"/test-chown-en-Español"
     cluster.fs.mkdir(PATH)
     c.post("/filebrowser/chown", dict(path=PATH, user="x", group="y"))
     assert_equal("x", cluster.fs.stats(PATH)["user"])
     assert_equal("y", cluster.fs.stats(PATH)["group"])
     c.post("/filebrowser/chown", dict(path=PATH, user="__other__", user_other="z", group="y"))
     assert_equal("z", cluster.fs.stats(PATH)["user"])
-
   finally:
     cluster.shutdown()
 
@@ -51,16 +51,35 @@ def test_listdir():
     c = make_logged_in_client()
     cluster.fs.setuser(cluster.superuser)
 
-    # Delete if there's already something there
+    # These paths contain non-ascii characters. Your editor will need the
+    # corresponding font library to display them correctly.
+    #
+    # We test that mkdir can handle unicode strings as well as byte strings.
+    # And even when the byte string can't be decoded properly (big5), the listdir
+    # still succeeds.
+    orig_paths = [
+      u'greek-Ελληνικά',
+      u'chinese-漢語',
+      'listdir',
+      'non-utf-8-(big5)-\xb2\xc4\xa4@\xb6\xa5\xacq',
+    ]
+
+    prefix = '/test-filebrowser/'
+    for path in orig_paths:
+      cluster.fs.mkdir(prefix + path)
+    response = c.get('/filebrowser/view' + prefix)
+    paths = [f['path'] for f in response.context['files']]
+    for path in orig_paths:
+      if isinstance(path, unicode):
+        uni_path = path
+      else:
+        uni_path = unicode(path, 'utf-8', errors='replace')
+      assert_true(prefix + uni_path in paths,
+                  '%s should be in dir listing %s' % (prefix + uni_path, paths))
+
+    # Delete user's home if there's already something there
     if cluster.fs.isdir("/user/test"):
       cluster.fs.rmtree("/user/test")
-
-    cluster.fs.mkdir('/test-filebrowser/listdir')
-    response = c.get('/filebrowser/view/test-filebrowser/')
-    paths = [f['path'] for f in response.context['files']]
-    assert_true("/test-filebrowser/listdir" in paths)
-
-    # test's home dir doesn't exist yet
     assert_false(response.context['home_directory'])
 
     # test's home directory now exists. Should be returned.
@@ -68,6 +87,11 @@ def test_listdir():
     response = c.get('/filebrowser/view/test-filebrowser/')
     assert_equal(response.context['home_directory'], '/user/test')
   finally:
+    try:
+      cluster.fs.rmtree('/test-filebrowser')
+      cluster.fs.rmtree('/user/test')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
     cluster.shutdown()
 
 
@@ -84,56 +108,64 @@ def test_view_gz():
     cluster.fs.mkdir('/test-gz-filebrowser/')
 
     f = cluster.fs.open('/test-gz-filebrowser/test-view.gz', "w")
-    sdf_string='\x1f\x8b\x08\x082r\xf4K\x00\x03f\x00+NI\xe3\x02\x00\xad\x96b\xc4\x04\x00\x00\x00'
+    sdf_string = '\x1f\x8b\x08\x082r\xf4K\x00\x03f\x00+NI\xe3\x02\x00\xad\x96b\xc4\x04\x00\x00\x00'
     f.write(sdf_string)
     f.close()
 
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip')
     assert_equal(response.context['view']['contents'], "sdf\n")
 
-# autodetect
+    # autodetect
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz')
     assert_equal(response.context['view']['contents'], "sdf\n")
 
-#offset should do nothing
+    # offset should do nothing
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip&offset=1')
     assert_false(response.context.has_key('view'))
 
-
     f = cluster.fs.open('/test-gz-filebrowser/test-view2.gz', "w")
     f.write("hello")
     f.close()
 
-#we shouldn't autodetect  non gzip files
+    # we shouldn't autodetect non gzip files
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz')
     assert_equal(response.context['view']['contents'], "hello")
 
-#we should fail to do a bad thing if they specify compression when it's not set.
+    # we should fail to do a bad thing if they specify compression when it's not set.
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz?compression=gzip')
     assert_false(response.context.has_key('view'))
 
   finally:
+    try:
+      cluster.fs.rmtree('/test-gz-filebrowser/')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
     cluster.shutdown()
 
+
 @attr('requires_hadoop')
-def test_view():
+def test_view_i18n():
   cluster = mini_cluster.shared_cluster(conf=True)
   try:
-    c = make_logged_in_client()
     cluster.fs.setuser(cluster.superuser)
-
     cluster.fs.mkdir('/test-filebrowser/')
 
-    f = cluster.fs.open('/test-filebrowser/test-view', "w")
-    f.write("hello")
-    f.close()
+    # Test viewing files in different encodings
+    content = u'pt-Olá en-hello ch-你好 ko-안녕 ru-Здравствуйте'
+    view_helper(cluster, 'utf-8', content)
+    view_helper(cluster, 'utf-16', content)
 
-    response = c.get('/filebrowser/view/test-filebrowser/test-view')
-    assert_equal(response.context['view']['contents'], "hello")
+    content = u'你好-big5'
+    view_helper(cluster, 'big5', content)
+
+    content = u'こんにちは-shift-jis'
+    view_helper(cluster, 'shift_jis', content)
 
-    response = c.get('/filebrowser/view/test-filebrowser/test-view?end=2&begin=1')
-    assert_equal(response.context['view']['contents'], "he")
+    content = u'안녕하세요-johab'
+    view_helper(cluster, 'johab', content)
 
+    # Test that the default view is home
+    c = make_logged_in_client()
     response = c.get('/filebrowser/view/')
     assert_equal(response.context['path'], '/')
     cluster.fs.mkdir('/user/test')
@@ -141,60 +173,125 @@ def test_view():
     response = c.get('/filebrowser/view/?default_to_home=1')
     assert_equal("http://testserver/filebrowser/view/user/test", response["location"])
   finally:
+    try:
+      cluster.fs.rmtree('/user/test')
+      cluster.fs.rmtree('/test-filebrowser/')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
     cluster.shutdown()
 
 
+def view_helper(cluster, encoding, content):
+  """
+  Write the content in the given encoding directly into the filesystem.
+  Then try to view it and make sure the data is correct.
+  """
+  c = make_logged_in_client()
+  filename = u'/test-filebrowser/test-view-carácter-internacional'
+  bytestring = content.encode(encoding)
+
+  try:
+    f = cluster.fs.open(filename, "w")
+    f.write(bytestring)
+    f.close()
+
+    response = c.get('/filebrowser/view%s?encoding=%s' % (filename, encoding))
+    assert_equal(response.context['view']['contents'], content)
+
+    response = c.get('/filebrowser/view%s?encoding=%s&end=8&begin=1' % (filename, encoding))
+    assert_equal(response.context['view']['contents'],
+                 unicode(bytestring[0:8], encoding, errors='replace'))
+  finally:
+    try:
+      cluster.fs.remove(filename)
+    except:
+      pass
+
+
 @attr('requires_hadoop')
-def test_edit():
+def test_edit_i18n():
   cluster = mini_cluster.shared_cluster(conf=True)
   try:
-    c = make_logged_in_client(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
-
     cluster.fs.mkdir('/test-filebrowser/')
-    # File doesn't exist - should be empty
-    test_path = '//test-filebrowser//test-edit'
-    # (this path is non-normalized to test normalization too)
-    edit_url = '/filebrowser/edit' + test_path
-    response = c.get(edit_url)
-    assert_equal(response.context['form'].data['path'],
-                 test_path)
-    assert_equal(response.context['form'].data['contents'], "")
-
-    # Just going to the edit page and not hitting save should not
-    # create the file
-    assert_false(cluster.fs.exists(test_path))
 
+    # Test utf-8
+    pass_1 = u'en-hello pt-Olá ch-你好 ko-안녕 ru-Здравствуйте'
+    pass_2 = pass_1 + u'yi-העלא'
+    edit_helper(cluster, 'utf-8', pass_1, pass_2)
+
+    # Test utf-16
+    edit_helper(cluster, 'utf-16', pass_1, pass_2)
+
+    # Test cjk
+    pass_1 = u'big5-你好'
+    pass_2 = pass_1 + u'世界'
+    edit_helper(cluster, 'big5', pass_1, pass_2)
+
+    pass_1 = u'shift_jis-こんにちは'
+    pass_2 = pass_1 + u'世界'
+    edit_helper(cluster, 'shift_jis', pass_1, pass_2)
+
+    pass_1 = u'johab-안녕하세요'
+    pass_2 = pass_1 + u'세상'
+    edit_helper(cluster, 'johab', pass_1, pass_2)
+  finally:
+    try:
+      cluster.fs.rmtree('/test-filebrowser/')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
+    cluster.shutdown()
+
+
+def edit_helper(cluster, encoding, contents_pass_1, contents_pass_2):
+  """
+  Put the content into the file with a specific encoding.
+  """
+  c = make_logged_in_client(cluster.superuser)
+
+  # This path is non-normalized to test normalization too
+  filename = u'//test-filebrowser//./test-edit-carácter-internacional'
+
+  # File doesn't exist - should be empty
+  edit_url = '/filebrowser/edit' + filename
+  response = c.get(edit_url)
+  assert_equal(response.context['form'].data['path'], filename)
+  assert_equal(response.context['form'].data['contents'], "")
+
+  # Just going to the edit page and not hitting save should not
+  # create the file
+  assert_false(cluster.fs.exists(filename))
+
+  try:
     # Put some data in there and post
-    new_contents = "hello world from editor"
     response = c.post("/filebrowser/save", dict(
-        path=test_path,
-        contents=new_contents), follow=True)
-    assert_equal(response.context['form'].data['path'],
-                 test_path)
-    assert_equal(response.context['form'].data['contents'],
-                 new_contents)
+        path=filename,
+        contents=contents_pass_1,
+        encoding=encoding), follow=True)
+    assert_equal(response.context['form'].data['path'], filename)
+    assert_equal(response.context['form'].data['contents'], contents_pass_1)
 
     # File should now exist
-    assert_true(cluster.fs.exists(test_path))
+    assert_true(cluster.fs.exists(filename))
     # And its contents should be what we expect
-    f = cluster.fs.open(test_path)
-    assert_equal(f.read(), new_contents)
+    f = cluster.fs.open(filename)
+    assert_equal(f.read(), contents_pass_1.encode(encoding))
     f.close()
 
     # We should be able to overwrite the file with another save
-    new_contents = "hello world again from editor"
     response = c.post("/filebrowser/save", dict(
-        path=test_path,
-        contents=new_contents), follow=True)
-    assert_equal(response.context['form'].data['path'],
-                 test_path)
-    assert_equal(response.context['form'].data['contents'],
-                 new_contents)
-    f = cluster.fs.open(test_path)
-    assert_equal(f.read(), new_contents)
+        path=filename,
+        contents=contents_pass_2,
+        encoding=encoding), follow=True)
+    assert_equal(response.context['form'].data['path'], filename)
+    assert_equal(response.context['form'].data['contents'], contents_pass_2)
+    f = cluster.fs.open(filename)
+    assert_equal(f.read(), contents_pass_2.encode(encoding))
     f.close()
 
     # TODO(todd) add test for maintaining ownership/permissions
   finally:
-    cluster.shutdown()
+    try:
+      cluster.fs.remove(filename)
+    except:
+      pass

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -50,6 +50,9 @@ http_500_debug_mode=0
 # Filename of SSL RSA Private Key
 ## ssl_private_key=
 
+# Default encoding for site data
+## default_site_encoding=utf-8
+
 # Configuration options for user authentication into the web application
 # ------------------------------------------------------------------------
 [[auth]]

+ 7 - 0
desktop/core/src/desktop/conf.py

@@ -179,6 +179,13 @@ TIME_ZONE = Config(
   default=os.environ.get("TZ", "America/Los_Angeles")
 )
 
+DEFAULT_SITE_ENCODING = Config(
+  key='default_site_encoding',
+  help='Default system-wide unicode encoding',
+  type=str,
+  default='utf-8'
+)
+
 SERVER_USER = Config(
   key="server_user",
   help="Username to run servers as",

+ 31 - 0
desktop/core/src/desktop/lib/django_forms.py

@@ -28,6 +28,9 @@ from django.utils.encoding import StrAndUnicode, force_unicode
 import simplejson
 import urllib
 
+import desktop.lib.i18n
+
+
 class MultipleInputWidget(Widget):
   """
   Together with MultipleInputField, represents repeating a form element many times,
@@ -158,6 +161,34 @@ class KeyValueField(CharField):
     except Exception:
       raise ValidationError("Not in key=value format.")
 
+class UnicodeEncodingField(ChoiceOrOtherField):
+  CHOICES = [
+    ('utf-8', 'Unicode UTF8'),
+    ('utf-16', 'Unicode UTF16'),
+    ('latin_1', 'Western ISO-8859-1'),
+    ('cyrillic', 'Cryrillic'),
+    ('arabic', 'Arabic'),
+    ('greek', 'Greek'),
+    ('hebrew', 'Hebrew'),
+    ('shift_jis', 'Japanese (Shift-JIS)'),
+    ('euc-jp', 'Japanese (EUC-JP)'),
+    ('iso2022_jp', 'Japanese (ISO-2022-JP)'),
+    ('euc-kr', 'Korean (EUC-KR)'),
+    ('iso2022-kr', 'Korean (ISO-2022-KR)'),
+    ('gbk', 'Chinese Simplified (GBK)'),
+    ('big5hkscs', 'Chinese Traditional (Big5)'),
+    ('ascii', 'ASCII'),
+  ]
+
+  def __init__(self, initial=None, *args, **kwargs):
+    ChoiceOrOtherField.__init__(self, UnicodeEncodingField.CHOICES, initial, *args, **kwargs)
+
+  def clean(self, value):
+    encoding = value[0]
+    if encoding and not desktop.lib.i18n.validate_encoding(encoding):
+      raise forms.ValidationError("'%s' encoding is not available" % (encoding,))
+    return value
+
 
 class MultiForm(object):
   """

+ 49 - 0
desktop/core/src/desktop/lib/i18n.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Library methods to deal with non-ascii data
+"""
+
+import codecs
+import desktop.conf
+import logging
+
+SITE_ENCODING = None
+REPLACEMENT_CHAR = u'\ufffd'
+
+def get_site_encoding():
+  """Get the default site encoding"""
+  global SITE_ENCODING
+  if SITE_ENCODING is None:
+    encoding = desktop.conf.DEFAULT_SITE_ENCODING.get()
+    if not validate_encoding(encoding):
+      default = desktop.conf.DEFAULT_SITE_ENCODING.config.default_value
+      msg = 'Invalid HUE configuration value for %s: "%s". Using default "%s"' % \
+                  (desktop.conf.DEFAULT_SITE_ENCODING.config.key, encoding, default)
+      logging.error(msg)
+      encoding = default
+    SITE_ENCODING = encoding
+  return SITE_ENCODING
+
+def validate_encoding(encoding):
+  """Return True/False on whether the system understands this encoding"""
+  try:
+    codecs.lookup(encoding)
+    return True
+  except LookupError:
+    return False

+ 2 - 1
desktop/core/src/desktop/middleware.py

@@ -24,6 +24,7 @@ import django.db
 from django.http import HttpResponseRedirect, HttpResponse
 from django.shortcuts import render_to_response
 from django.utils.http import urlquote
+from django.utils.encoding import iri_to_uri
 import django.views.static
 import django.views.generic.simple
 import django.contrib.auth.views
@@ -97,7 +98,7 @@ class JFrameMiddleware(object):
       query_string = get_params.urlencode()
       if query_string:
         path = request.path + "?" + query_string
-    response['X-Hue-JFrame-Path'] = path
+    response['X-Hue-JFrame-Path'] = iri_to_uri(path)
     if response.status_code == 200:
       if is_jframe_request(request):
         if hasattr(request, "flash"):

+ 53 - 13
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -31,6 +31,7 @@ from thrift.transport import TTransport
 from thrift.transport import TSocket
 from thrift.protocol import TBinaryProtocol
 
+from django.utils.encoding import smart_str, force_unicode
 from desktop.lib import thrift_util
 from hadoop.api.hdfs import Namenode, Datanode
 from hadoop.api.hdfs.constants import QUOTA_DONT_SET, QUOTA_RESET
@@ -59,12 +60,24 @@ DEFAULT_READ_SIZE = 1024*1024 # 1MB
 WRITE_BUFFER_SIZE = 128*1024 # 128K
 
 # Class that we translate into PermissionDeniedException
-HADOOP_ACCESSCONTROLEXCEPTION="org.apache.hadoop.security.AccessControlException"
+HADOOP_ACCESSCONTROLEXCEPTION = "org.apache.hadoop.security.AccessControlException"
 
 # Timeout for thrift calls to NameNode
 NN_THRIFT_TIMEOUT = 15
 DN_THRIFT_TIMEOUT = 3
 
+# Encoding used by HDFS namespace
+HDFS_ENCODING = 'utf-8'
+
+def encode_fs_path(path):
+  """encode_fs_path(path) -> byte string in utf8"""
+  return smart_str(path, HDFS_ENCODING, errors='strict')
+
+def decode_fs_path(path):
+  """decode_fs_path(bytestring) -> unicode path"""
+  return force_unicode(path, HDFS_ENCODING, errors='strict')
+
+
 class HadoopFileSystem(object):
   """
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.
@@ -127,7 +140,6 @@ class HadoopFileSystem(object):
 
     raise Exception("Hadoop binary (%s) does not exist." % self.hadoop_bin_path)
 
-
   @property
   def uri(self):
     return self._get_hdfs_base()
@@ -161,6 +173,7 @@ class HadoopFileSystem(object):
 
   @_coerce_exceptions
   def remove(self, path):
+    path = encode_fs_path(path)
     stat = self._hadoop_stat(path)
     if not stat:
       raise IOError("File not found: %s" % path)
@@ -176,11 +189,13 @@ class HadoopFileSystem(object):
   def mkdir(self, path, mode=0755):
     # TODO(todd) there should be a mkdir that isn't mkdirHIER
     # (this is mkdir -p I think)
+    path = encode_fs_path(path)
     success = self.nn_client.mkdirhier(self.request_context, normpath(path), mode)
     if not success:
       raise IOError("mkdir failed")
 
   def _rmdir(self, path, recursive=False):
+    path = encode_fs_path(path)
     stat = self._hadoop_stat(path)
     if not stat:
       raise IOError("Directory not found: %s" % (path,))
@@ -202,20 +217,30 @@ class HadoopFileSystem(object):
 
   @_coerce_exceptions
   def listdir(self, path):
+    path = encode_fs_path(path)
     stats = self.nn_client.ls(self.request_context, normpath(path))
-    return [self.basename(stat.path) for stat in stats]
+    return [self.basename(decode_fs_path(stat.path)) for stat in stats]
 
   @_coerce_exceptions
   def listdir_stats(self, path):
+    path = encode_fs_path(path)
     stats = self.nn_client.ls(self.request_context, normpath(path))
     return [self._unpack_stat(s) for s in stats]
 
   @_coerce_exceptions
   def get_content_summaries(self, paths):
-    return self.nn_client.multiGetContentSummary(self.request_context, [normpath(path) for path in paths])
+    path = encode_fs_path(path)
+    summaries = self.nn_client.multiGetContentSummary(self.request_context,
+                                                      [normpath(path) for path in paths])
+    def _fix_summary(summary):
+      summary.path = decode_fs_path(summary.path)
+      return summary
+    return [_fix_summary(s) for s in summaries]
 
   @_coerce_exceptions
   def rename(self, old, new):
+    old = encode_fs_path(old)
+    new = encode_fs_path(new)
     success = self.nn_client.rename(
       self.request_context, normpath(old), normpath(new))
     if not success: #TODO(todd) these functions should just throw if failed
@@ -267,10 +292,12 @@ class HadoopFileSystem(object):
 
   @_coerce_exceptions
   def chmod(self, path, mode):
+    path = encode_fs_path(path)
     self.nn_client.chmod(self.request_context, normpath(path), mode)
 
   @_coerce_exceptions
   def chown(self, path, user, group):
+    path = encode_fs_path(path)
     self.nn_client.chown(self.request_context, normpath(path), user, group)
 
   @_coerce_exceptions
@@ -281,6 +308,7 @@ class HadoopFileSystem(object):
                  used_bytes=used,
                  available_bytes=available),
       )
+
   @_coerce_exceptions
   def _get_blocks(self, path, offset, length):
     """
@@ -291,13 +319,20 @@ class HadoopFileSystem(object):
         thriftPort=53417, state=1, remaining=18987925504, host='127.0.0.1',
         storageID='DS-1238582576-127.0.1.1-50010-1240968238474', dfsUsed=36864)], numBytes=424)]
     """
-    return self.nn_client.getBlocks(self.request_context, normpath(path), offset, length)
+    path = encode_fs_path(path)
+    blocks = self.nn_client.getBlocks(self.request_context, normpath(path), offset, length)
+    def _fix_block(blk):
+      blk.path = decode_fs_path(blk.path)
+      return blk
+    return [_fix_block(blk) for blk in blocks]
 
 
   def _hadoop_stat(self, path):
     """Returns None if file does not exist."""
+    path = encode_fs_path(path)
     try:
       stat = self.nn_client.stat(self.request_context, normpath(path))
+      stat.path = decode_fs_path(stat.path)
       return stat
     except IOException, ioe:
       if ioe.clazz == 'java.io.FileNotFoundException':
@@ -315,6 +350,7 @@ class HadoopFileSystem(object):
     @param len the number of bytes to read
     """
     errs = []
+    block.path = encode_fs_path(block.path)
     for node in block.nodes:
       dn_conn = self._connect_dn(node)
       try:
@@ -335,7 +371,7 @@ class HadoopFileSystem(object):
     @param path The path to the given hdfs resource
     @param size The amount of bytes that a given subtree of files can grow to.
     """
-
+    path = encode_fs_path(path)
     if normpath(path) == '/':
       raise ValueError('Cannot set quota for "/"')
 
@@ -352,7 +388,7 @@ class HadoopFileSystem(object):
     @param path The path to the given hdfs resource
     @param num_files The amount of files that can exist within that subtree.
     """
-
+    path = encode_fs_path(path)
     if normpath(path) == '/':
       raise ValueError('Cannot set quota for "/"')
 
@@ -366,6 +402,7 @@ class HadoopFileSystem(object):
     """
     Remove the diskspace quota at a given path
     """
+    path = encode_fs_path(path)
     self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, QUOTA_RESET)
 
   @_coerce_exceptions
@@ -373,6 +410,7 @@ class HadoopFileSystem(object):
     """
     Remove the namespace quota at a given path
     """
+    path = encode_fs_path(path)
     self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_RESET, QUOTA_DONT_SET)
 
 
@@ -381,6 +419,7 @@ class HadoopFileSystem(object):
     """
     Get the current space quota in bytes for disk space. None if it is unset
     """
+    path = encode_fs_path(path)
     space_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).spaceQuota
     if space_quota == QUOTA_RESET or space_quota == QUOTA_DONT_SET:
       return None
@@ -393,6 +432,7 @@ class HadoopFileSystem(object):
     """
     Get the current quota in number of files. None if it is unset
     """
+    path = encode_fs_path(path)
     file_count_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).quota
     if file_count_quota == QUOTA_RESET or file_count_quota == QUOTA_DONT_SET:
       return None
@@ -406,6 +446,7 @@ class HadoopFileSystem(object):
     "space_used", and "space_quota".  The quotas
     may be None.
     """
+    path = encode_fs_path(path)
     summary = self.nn_client.getContentSummary(self.request_context, normpath(path))
     ret = dict()
     ret["file_count"] = summary.fileCount
@@ -430,7 +471,6 @@ class HadoopFileSystem(object):
     client.close = lambda: transport.close()
     return client
 
-
   @staticmethod
   def _unpack_stat(stat):
     """Unpack a Thrift "Stat" object into a dictionary that looks like fs.stat"""
@@ -441,7 +481,7 @@ class HadoopFileSystem(object):
       mode |= statconsts.S_IFREG
 
     return {
-      'path': stat.path,
+      'path': decode_fs_path(stat.path),
       'size': stat.length,
       'mtime': stat.mtime / 1000,
       'mode': mode,
@@ -630,7 +670,7 @@ class FileUpload(object):
                            "-Dfs.default.name=" + self.fs._get_hdfs_base(),
                            "-Dhadoop.job.ugi=" + self.fs.ugi] + \
                            extra_confs + \
-                           ["-put", "-", path]
+                           ["-put", "-", encode_fs_path(path)]
     self.path = path
     self.putter = subprocess.Popen(self.subprocess_cmd,
                                    stdin=subprocess.PIPE,
@@ -647,9 +687,9 @@ class FileUpload(object):
     try:
       (stdout, stderr) = self.putter.communicate()
     except IOError, ioe:
-        logging.debug("Saw IOError writing %r" % self.path, exc_info=1)
-        if ioe.errno == 32: # Broken Pipe
-           stdout, stderr = self.putter.communicate()
+      logging.debug("Saw IOError writing %r" % self.path, exc_info=1)
+      if ioe.errno == errno.EPIPE:
+        stdout, stderr = self.putter.communicate()
     self.closed = True
     if stderr:
       LOG.warn("HDFS FileUpload (cmd='%s')outputted stderr:\n%s" %

+ 62 - 14
desktop/libs/hadoop/src/hadoop/fs/hadoopfs_test.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -203,35 +204,35 @@ def test_quota_space():
       fs.rmtree('/tmp/foo2')
     fs.mkdir("/tmp/foo2", 0777) # this also tests more restrictive subdirectories
     ONE_HUNDRED_192_MEGS = 1024 * 1024 * 192
-    
-    fs.set_diskspace_quota("/tmp/foo2", ONE_HUNDRED_192_MEGS) 
+
+    fs.set_diskspace_quota("/tmp/foo2", ONE_HUNDRED_192_MEGS)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), ONE_HUNDRED_192_MEGS)
 
-    f = fs.open('/tmp/foo2/asdf', 'w')	 # we should be able to do this 
-    f.write('a') 
+    f = fs.open('/tmp/foo2/asdf', 'w')	 # we should be able to do this
+    f.write('a')
     f.close()
 
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), ONE_HUNDRED_192_MEGS)
 
-    fs.set_diskspace_quota("/tmp/foo2", 1) 
+    fs.set_diskspace_quota("/tmp/foo2", 1)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), 1)
 
-    f = fs.open('/tmp/foo2/asdfsd', 'w')	 
+    f = fs.open('/tmp/foo2/asdfsd', 'w')
     f.write('a')
     assert_raises(IOError, f.close)
 
     fs.clear_diskspace_quota("/tmp/foo2")
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), None)
 
-    f = fs.open('/tmp/foo2/asdfsda', 'w')	 
-    f.write('a') 
+    f = fs.open('/tmp/foo2/asdfsda', 'w')
+    f.write('a')
     f.close()
 
     fs.mkdir("/tmp/baz/bar", 0777)  # this tests more permissive subdirectories
     fs.set_diskspace_quota("/tmp/baz", 1)
-    fs.set_diskspace_quota("/tmp/baz/bar", ONE_HUNDRED_192_MEGS) 
+    fs.set_diskspace_quota("/tmp/baz/bar", ONE_HUNDRED_192_MEGS)
 
-    f = fs.open('/tmp/baz/bar', 'w')	 
+    f = fs.open('/tmp/baz/bar', 'w')
     f.write('aaaa') #should violate the subquota
     assert_raises(IOError, f.close)
 
@@ -256,17 +257,17 @@ def test_quota_namespace_count():
     fs.mkdir("/tmp/foo2", 0777)
 
     # check the get_namespace_quota function
-    fs.set_namespace_quota("/tmp/foo2", 4) 
+    fs.set_namespace_quota("/tmp/foo2", 4)
     assert_equals(fs.get_namespace_quota("/tmp/foo2"), 4)
 
     # violate the namespace count
     for i in range(3):
       f = fs.open('/tmp/foo2/works' + str(i), 'w')
-      f.write('a') 
+      f.write('a')
       f.close()
 
     f = fs.open('/tmp/foo2/asdfsdc', 'w')
-    f.write('a') 
+    f.write('a')
     assert_raises(IOError, f.close)
 
     # Check summary stats
@@ -281,9 +282,56 @@ def test_quota_namespace_count():
     assert_equals(fs.get_namespace_quota("/tmp/foo2"), None)
 
     f = fs.open('/tmp/foo2/asdfsdd', 'w')
-    f.write('a') 
+    f.write('a')
     f.close()
   finally:
     if fs.exists('/tmp/foo2'):
       fs.rmtree("/tmp/foo2")
     cluster.shutdown()
+
+
+@attr('requires_hadoop')
+def test_i18n_namespace():
+  cluster = mini_cluster.shared_cluster()
+  cluster.fs.setuser(cluster.superuser)
+
+  def check_existence(name, parent, present=True):
+    assertion = present and assert_true or assert_false
+    listing = cluster.fs.listdir(parent)
+    assertion(name in listing, "%s should be in %s" % (name, listing))
+
+  name = u'pt-Olá_ch-你好_ko-안녕_ru-Здравствуйте'
+  prefix = '/tmp/i18n'
+  dir_path = '%s/%s' % (prefix, name)
+  file_path = '%s/%s' % (dir_path, name)
+
+  try:
+    # Create a directory
+    cluster.fs.mkdir(dir_path)
+    # Create a file (same name) in the directory
+    cluster.fs.open(file_path, 'w').close()
+
+    # Directory is there
+    check_existence(name, prefix)
+    # File is there
+    check_existence(name, dir_path)
+
+    # Test rename
+    new_file_path = file_path + '.new'
+    cluster.fs.rename(file_path, new_file_path)
+    # New file is there
+    check_existence(name + '.new', dir_path)
+
+    # Test remove
+    cluster.fs.remove(new_file_path)
+    check_existence(name + '.new', dir_path, present=False)
+
+    # Test rmtree
+    cluster.fs.rmtree(dir_path)
+    check_existence(name, prefix, present=False)
+  finally:
+    try:
+      cluster.fs.rmtree(prefix)
+    except:
+      pass
+    cluster.shutdown()

+ 4 - 4
dist/README

@@ -5,11 +5,11 @@ If you're impatient, these are the key steps.  Please check the full manual
 for more details.
 
 ## Install
-$ HADOOP_HOME=/path/to/hadoop-0.20.1+152 PREFIX=/path/to/install/into \
-  make install
+$ HADOOP_HOME=/usr/lib/hadoop-0.20 PREFIX=/usr/local make install
+
 ## Install plug-ins
 $ cd /usr/lib/hadoop/lib
-$ ln -s /usr/share/hue/desktop/libs/hadoop/java-lib/hue*jar
+$ ln -s /usr/local/hue/desktop/libs/hadoop/java-lib/hue*jar
 
 ## Configure Hadoop
 Edit hdfs-site.xml:
@@ -37,4 +37,4 @@ Edit mapred-site.xml:
 </property>
 
 ## Run!
-$ /usr/share/hue/build/env/bin/supervisor
+$ /usr/local/hue/build/env/bin/supervisor