Browse Source

HUE-59. i18n support in filebrowser

- Added default encoding to config.
- Added i18n support for hdfs namespace.
- Added i18n support for filebrowser.
- FileViewer and FileEditor has an extra "encoding" field (not exposed yet),
  which they use to decode the file data.
bc Wong 15 years ago
parent
commit
8df848fff3

+ 1 - 0
apps/filebrowser/src/filebrowser/forms.py

@@ -37,6 +37,7 @@ class PathField(CharField):
 class EditorForm(forms.Form):
 class EditorForm(forms.Form):
   path = PathField(label="File to edit")
   path = PathField(label="File to edit")
   contents = CharField(widget=Textarea, label="Contents", required=False)
   contents = CharField(widget=Textarea, label="Contents", required=False)
+  encoding = CharField(label='Encoding', required=False)
 
 
 class RenameForm(forms.Form):
 class RenameForm(forms.Form):
   op = "rename"
   op = "rename"

+ 9 - 8
apps/filebrowser/src/filebrowser/templates/display.mako

@@ -1,3 +1,4 @@
+## -*- coding: utf-8 -*-
 ## Licensed to Cloudera, Inc. under one
 ## Licensed to Cloudera, Inc. under one
 ## or more contributor license agreements.  See the NOTICE file
 ## or more contributor license agreements.  See the NOTICE file
 ## distributed with this work for additional information
 ## distributed with this work for additional information
@@ -52,16 +53,16 @@
        <a class="fv-download" data-filters="ArtButton" target="_blank" data-icon-styles="{'width': 16, 'height': 16}" href="${url('filebrowser.views.download', path=path_enc)}">Download</a>
        <a class="fv-download" data-filters="ArtButton" target="_blank" data-icon-styles="{'width': 16, 'height': 16}" href="${url('filebrowser.views.download', path=path_enc)}">Download</a>
        <a class="fv-viewLocation" data-filters="ArtButton" data-icon-styles="{'width': 16, 'height': 16}" href="${url('filebrowser.views.view', path=dirname_enc)}" target="FileBrowser">View File Location</a>
        <a class="fv-viewLocation" data-filters="ArtButton" data-icon-styles="{'width': 16, 'height': 16}" href="${url('filebrowser.views.view', path=dirname_enc)}" target="FileBrowser">View File Location</a>
        <a class="ccs-refresh large" data-filters="ArtButton">Refresh</a>
        <a class="ccs-refresh large" data-filters="ArtButton">Refresh</a>
-    </div> 
+    </div>
   </div>
   </div>
   <div class="fv-navhead">
   <div class="fv-navhead">
     % if not view['compression'] or view['compression'] == "none":
     % if not view['compression'] or view['compression'] == "none":
       <div class="fv-navStatus">
       <div class="fv-navStatus">
         <form data-filters="SubmitOnChange" class="fv-changeBytesForm" action="${url('filebrowser.views.view', path=path_enc)}" method="GET">
         <form data-filters="SubmitOnChange" class="fv-changeBytesForm" action="${url('filebrowser.views.view', path=path_enc)}" method="GET">
           <span class="fv-bold">Viewing Bytes:</span><a class="fv-editBytes ccs-inline" title="Enter Bytes"></a>
           <span class="fv-bold">Viewing Bytes:</span><a class="fv-editBytes ccs-inline" title="Enter Bytes"></a>
-          <input data-filters="OverText" name="begin" value="${view['offset'] + 1}"/>
+          <input name="begin" value="${view['offset'] + 1}"/>
           -
           -
-          <input data-filters="OverText" value="${view['end']}" name="end"/> of 
+          <input value="${view['end']}" name="end"/> of
           <span class="fv-bold totalBytes">${stats['size']}</span>
           <span class="fv-bold totalBytes">${stats['size']}</span>
           <span class="fv-stepInfo">(${view['length']} B block size)</span>
           <span class="fv-stepInfo">(${view['length']} B block size)</span>
           % if view['mode']:
           % if view['mode']:
@@ -76,14 +77,14 @@
               first = "style='visibility:hidden'"
               first = "style='visibility:hidden'"
               prev = "style='visibility:hidden'"
               prev = "style='visibility:hidden'"
           else:
           else:
-              first = "href='%s?offset=0&length=%d&compression=none' title='1 - %d'" %(base_url, view['length'], min(view['length'], stats['size'])) 
-              prev =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, max(0, view['offset']-view['length']), view['length'], max(0, view['offset']-view['length']) + 1, min(max(0, view['offset'] - view['length']) + view['length'], stats['size'])) 
+              first = "href='%s?offset=0&length=%d&compression=none' title='1 - %d'" %(base_url, view['length'], min(view['length'], stats['size']))
+              prev =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, max(0, view['offset']-view['length']), view['length'], max(0, view['offset']-view['length']) + 1, min(max(0, view['offset'] - view['length']) + view['length'], stats['size']))
           if view['offset'] + view['length'] >= stats['size']:
           if view['offset'] + view['length'] >= stats['size']:
               next = "style='visibility:hidden'"
               next = "style='visibility:hidden'"
               last = "style='visibility:hidden'"
               last = "style='visibility:hidden'"
           else:
           else:
-              next = "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, view['offset'] + view['length'], view['length'], view['offset'] + view['length'] + 1, view['offset'] + (2 * view['length'])) 
-              last =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, stats['size']-(stats['size'] % view['length']), view['length'], stats['size']-(stats['size'] % view['length']) + 1, stats['size']) 
+              next = "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, view['offset'] + view['length'], view['length'], view['offset'] + view['length'] + 1, view['offset'] + (2 * view['length']))
+              last =  "href='%s?offset=%d&length=%d&compression=none' title='%d - %d'" %(base_url, stats['size']-(stats['size'] % view['length']), view['length'], stats['size']-(stats['size'] % view['length']) + 1, stats['size'])
         %>
         %>
         ###DEFINE REL
         ###DEFINE REL
         <a class="ccs-inline fv-firstBlock" data-filters="PointyTip" ${first}>First Block</a>
         <a class="ccs-inline fv-firstBlock" data-filters="PointyTip" ${first}>First Block</a>
@@ -113,7 +114,7 @@
     <div class="right_col">
     <div class="right_col">
     %if 'contents' in view:
     %if 'contents' in view:
       % if view['masked_binary_data']:
       % if view['masked_binary_data']:
-      <div class="fv-binaryWarning">Warning: some binary data has been masked out with '.'.</div>
+      <div class="fv-binaryWarning">Warning: some binary data has been masked out with '&#xfffd'.</div>
       % endif
       % endif
     % endif
     % endif
       <div class="jframe_padded">
       <div class="jframe_padded">

+ 4 - 3
apps/filebrowser/src/filebrowser/templates/edit.mako

@@ -35,15 +35,16 @@
   <div class="alert_popup">
   <div class="alert_popup">
     % for field in form:
     % for field in form:
       % if len(field.errors):
       % if len(field.errors):
-       ${str(field.errors) | n}
+       ${unicode(field.errors) | n}
       % endif
       % endif
     % endfor
     % endfor
   </div>
   </div>
 % endif
 % endif
 <form class="no_overflow fe-editForm" method="post" action="${url('filebrowser.views.save_file')}">
 <form class="no_overflow fe-editForm" method="post" action="${url('filebrowser.views.save_file')}">
-    ${edit.render_field(form["path"],hidden=True, notitle=True)}
+    ${edit.render_field(form["path"], hidden=True, notitle=True)}
+    ${edit.render_field(form["encoding"], hidden=True, notitle=True)}
     <h2 class="ccs-hidden">${form["contents"].label_tag() | n}</h2>
     <h2 class="ccs-hidden">${form["contents"].label_tag() | n}</h2>
-    <div class="fe-divResize">${str(form["contents"]) | n}</div>
+    <div class="fe-divResize">${unicode(form["contents"]) | n}</div>
     <input class="ccs-hidden" type="submit" name="save" value="saveAs">
     <input class="ccs-hidden" type="submit" name="save" value="saveAs">
     <input class="ccs-hidden" type="submit" name="save" value="save">
     <input class="ccs-hidden" type="submit" name="save" value="save">
 </form>
 </form>

+ 3 - 3
apps/filebrowser/src/filebrowser/templates/editor_components.mako

@@ -23,10 +23,10 @@
     titlecls = "ccs-hidden"
     titlecls = "ccs-hidden"
 %>
 %>
   <dt class="${titlecls}">${field.label_tag() | n}</dt>
   <dt class="${titlecls}">${field.label_tag() | n}</dt>
-  <dd class="${cls}">${str(field) | n}</dd>
+  <dd class="${cls}">${unicode(field) | n}</dd>
   % if len(field.errors):
   % if len(field.errors):
     <dd class="beeswax_error">
     <dd class="beeswax_error">
-       ${str(field.errors) | n}
+       ${unicode(field.errors) | n}
      </dd>
      </dd>
   % endif
   % endif
-</%def>
+</%def>

+ 2 - 1
apps/filebrowser/src/filebrowser/templates/listdir_components.mako

@@ -16,6 +16,7 @@
 <%!
 <%!
 import datetime
 import datetime
 from django.template.defaultfilters import urlencode, stringformat, filesizeformat, date, time
 from django.template.defaultfilters import urlencode, stringformat, filesizeformat, date, time
+from django.utils.encoding import iri_to_uri
 %>
 %>
 
 
 
 
@@ -55,7 +56,7 @@ from django.template.defaultfilters import urlencode, stringformat, filesizeform
             display_name = file['path']
             display_name = file['path']
           endif
           endif
         %>
         %>
-        <% path_enc = urlencode(file['path']) %>
+        <% path_enc = iri_to_uri(urlencode(file['path'])) %>
         <tr class="ccs-no_select fb-item-row ${cls}"
         <tr class="ccs-no_select fb-item-row ${cls}"
          data-filters="ContextMenu"
          data-filters="ContextMenu"
          data-context-menu-actions="[{'events':['contextmenu','click:relay(.fb-item-options)'],'menu':'ul.context-menu'}]"
          data-context-menu-actions="[{'events':['contextmenu','click:relay(.fb-item-options)'],'menu':'ul.context-menu'}]"

+ 2 - 1
apps/filebrowser/src/filebrowser/templates/saveas.mako

@@ -21,7 +21,7 @@
       <div class="alert_popup">
       <div class="alert_popup">
         % for field in form:
         % for field in form:
           % if len(field.errors):
           % if len(field.errors):
-               ${str(field.errors) | n}
+               ${unicode(field.errors) | n}
           % endif
           % endif
         % endfor
         % endfor
       </div>
       </div>
@@ -31,6 +31,7 @@
           Please enter the location where you'd like to save the file.
           Please enter the location where you'd like to save the file.
           ${edit.render_field(form["path"], notitle=True)}
           ${edit.render_field(form["path"], notitle=True)}
           <div>${edit.render_field(form["contents"], hidden=True)}</div>
           <div>${edit.render_field(form["contents"], hidden=True)}</div>
+	  <div>${edit.render_field(form["encoding"], hidden=True)}</div>
           <input type="submit" class="ccs-hidden" name="save" value="save"/>
           <input type="submit" class="ccs-hidden" name="save" value="save"/>
       </form>
       </form>
     </div>
     </div>

+ 55 - 57
apps/filebrowser/src/filebrowser/views.py

@@ -35,7 +35,7 @@ from django.utils.html import escape
 from cStringIO import StringIO
 from cStringIO import StringIO
 from gzip import GzipFile
 from gzip import GzipFile
 
 
-
+from desktop.lib import i18n
 from desktop.lib.django_util import make_absolute, render_json
 from desktop.lib.django_util import make_absolute, render_json
 from desktop.lib.django_util import PopupException, format_preserving_redirect
 from desktop.lib.django_util import PopupException, format_preserving_redirect
 from filebrowser.lib.rwx import filetype, rwx
 from filebrowser.lib.rwx import filetype, rwx
@@ -45,7 +45,6 @@ from filebrowser.forms import RenameForm, UploadForm, MkDirForm, RmDirForm, RmTr
 from hadoop.fs import normpath
 from hadoop.fs import normpath
 from filebrowser.plugin.views import render_with_toolbars
 from filebrowser.plugin.views import render_with_toolbars
 
 
-import filebrowser.plugin.toolbar
 
 
 DEFAULT_CHUNK_SIZE_BYTES = 1024*4 # 4KB
 DEFAULT_CHUNK_SIZE_BYTES = 1024*4 # 4KB
 MAX_CHUNK_SIZE_BYTES = 1024*1024 # 1MB
 MAX_CHUNK_SIZE_BYTES = 1024*1024 # 1MB
@@ -55,12 +54,9 @@ DOWNLOAD_CHUNK_SIZE = 32*1024 # 32KB
 # Sentences refer to groups of bytes printed together, within a line.
 # Sentences refer to groups of bytes printed together, within a line.
 BYTES_PER_LINE = 16
 BYTES_PER_LINE = 16
 BYTES_PER_SENTENCE = 2
 BYTES_PER_SENTENCE = 2
-# If the percentage of non-printable bytes is greater than this, binary mode is
-# enabled by default.
-BINARY_PERCENTAGE = 0.10
 
 
 # The maximum size the file editor will allow you to edit
 # The maximum size the file editor will allow you to edit
-MAX_FILEEDITOR_SIZE=256*1024
+MAX_FILEEDITOR_SIZE = 256*1024
 
 
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
 
 
@@ -140,21 +136,21 @@ def edit(request, path, form=None):
   if stats and stats['size'] > MAX_FILEEDITOR_SIZE:
   if stats and stats['size'] > MAX_FILEEDITOR_SIZE:
     raise PopupException("File too big to edit: %s" % (path,))
     raise PopupException("File too big to edit: %s" % (path,))
 
 
-  if stats:
-    f = request.fs.open(path)
-    try:
-      current_contents = f.read()
+  if not form:
+    encoding = request.REQUEST.get('encoding', i18n.get_site_encoding())
+    if stats:
+      f = request.fs.open(path)
       try:
       try:
-        current_contents = unicode(current_contents)
-      except UnicodeDecodeError:
-        raise PopupException("File is not unicode text; cannot be edited: %s" % (path,))
-    finally:
-      f.close()
-  else:
-    current_contents = ""
+        try:
+          current_contents = unicode(f.read(), encoding)
+        except UnicodeDecodeError:
+          raise PopupException("File is not encoded in %s; cannot be edited: %s" % (encoding, path))
+      finally:
+        f.close()
+    else:
+      current_contents = u""
 
 
-  if not form:
-    form = EditorForm(dict(path=path, contents=current_contents))
+    form = EditorForm(dict(path=path, contents=current_contents, encoding=encoding))
 
 
   data = dict(
   data = dict(
     exists=(stats is not None),
     exists=(stats is not None),
@@ -187,16 +183,20 @@ def save_file(request):
     return edit(request, path, form=form)
     return edit(request, path, form=form)
 
 
   if request.fs.exists(path):
   if request.fs.exists(path):
-    _do_overwrite_save(request.fs, path, form.cleaned_data['contents'])
+    _do_overwrite_save(request.fs, path,
+                       form.cleaned_data['contents'],
+                       form.cleaned_data['encoding'])
   else:
   else:
-    _do_newfile_save(request.fs, path, form.cleaned_data['contents'])
+    _do_newfile_save(request.fs, path,
+                     form.cleaned_data['contents'],
+                     form.cleaned_data['encoding'])
 
 
   request.flash.put('Saved %s.' % os.path.basename(path))
   request.flash.put('Saved %s.' % os.path.basename(path))
   """ Changing path to reflect the request path of the JFrame that will actually be returned."""
   """ Changing path to reflect the request path of the JFrame that will actually be returned."""
   request.path = urlresolvers.reverse("filebrowser.views.edit", kwargs=dict(path=path))
   request.path = urlresolvers.reverse("filebrowser.views.edit", kwargs=dict(path=path))
   return edit(request, path, form)
   return edit(request, path, form)
 
 
-def _do_overwrite_save(fs, path, data):
+def _do_overwrite_save(fs, path, data, encoding):
   """
   """
   Atomically (best-effort) save the specified data to the given path
   Atomically (best-effort) save the specified data to the given path
   on the filesystem.
   on the filesystem.
@@ -214,7 +214,7 @@ def _do_overwrite_save(fs, path, data):
   new_file = fs.open(path_dest, "w")
   new_file = fs.open(path_dest, "w")
   try:
   try:
     try:
     try:
-      new_file.write(data)
+      new_file.write(data.encode(encoding))
       logging.info("Wrote to " + path_dest)
       logging.info("Wrote to " + path_dest)
     finally:
     finally:
       new_file.close()
       new_file.close()
@@ -253,7 +253,7 @@ def _do_overwrite_save(fs, path, data):
   fs.rename(path_dest, path)
   fs.rename(path_dest, path)
 
 
 
 
-def _do_newfile_save(fs, path, data):
+def _do_newfile_save(fs, path, data, encoding):
   """
   """
   Save data to the path 'path' on the filesystem 'fs'.
   Save data to the path 'path' on the filesystem 'fs'.
 
 
@@ -261,7 +261,7 @@ def _do_newfile_save(fs, path, data):
   """
   """
   new_file = fs.open(path, "w")
   new_file = fs.open(path, "w")
   try:
   try:
-    new_file.write(data)
+    new_file.write(data.encode(encoding))
   finally:
   finally:
     new_file.close()
     new_file.close()
 
 
@@ -291,12 +291,14 @@ def listdir(request, path):
     'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
     'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
     'cwd_set': True
     'cwd_set': True
   }
   }
+
   stats = request.fs.listdir_stats(path)
   stats = request.fs.listdir_stats(path)
+
   # Include parent dir, unless at filesystem root.
   # Include parent dir, unless at filesystem root.
   if normpath(path) != posixpath.sep:
   if normpath(path) != posixpath.sep:
     parent_stat = request.fs.stats(posixpath.join(path, ".."))
     parent_stat = request.fs.stats(posixpath.join(path, ".."))
-    # the 'path' field would be absolute, but we want its basename to be
-    # actually '..' for display purposes
+    # The 'path' field would be absolute, but we want its basename to be
+    # actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
     parent_stat['path'] = posixpath.join(path, "..")
     parent_stat['path'] = posixpath.join(path, "..")
     stats.insert(0, parent_stat)
     stats.insert(0, parent_stat)
 
 
@@ -314,10 +316,11 @@ def _massage_stats(request, stats):
   Massage a stats record as returned by the filesystem implementation
   Massage a stats record as returned by the filesystem implementation
   into the format that the views would like it in.
   into the format that the views would like it in.
   """
   """
-  normalized = normpath(stats['path'])
+  path = stats['path']
+  normalized = normpath(path)
   return {
   return {
     'path': normalized,
     'path': normalized,
-    'name': posixpath.basename(stats['path']),
+    'name': posixpath.basename(path),
     'stats': stats,
     'stats': stats,
     'type': filetype(stats['mode']),
     'type': filetype(stats['mode']),
     'rwx': rwx(stats['mode']),
     'rwx': rwx(stats['mode']),
@@ -341,8 +344,10 @@ def display(request, path):
   """
   """
   Implements displaying part of a file.
   Implements displaying part of a file.
 
 
-  GET arguments are length, offset, mode and compression with reasonable
-  defaults chosen.
+  GET arguments are length, offset, mode, compression and encoding
+  with reasonable defaults chosen.
+
+  Note that display by length and offset are on bytes, not on characters.
 
 
   TODO(philip): Could easily built-in file type detection
   TODO(philip): Could easily built-in file type detection
   (perhaps using something similar to file(1)), as well
   (perhaps using something similar to file(1)), as well
@@ -355,6 +360,7 @@ def display(request, path):
     raise PopupException("Not a file: '%s'" % (path,))
     raise PopupException("Not a file: '%s'" % (path,))
 
 
   stats = request.fs.stats(path)
   stats = request.fs.stats(path)
+  encoding = request.GET.get('encoding', i18n.get_site_encoding())
 
 
   # I'm mixing URL-based parameters and traditional
   # I'm mixing URL-based parameters and traditional
   # HTTP GET parameters, since URL-based parameters
   # HTTP GET parameters, since URL-based parameters
@@ -364,11 +370,11 @@ def display(request, path):
   # because the offset came in via the toolbar manual byte entry.
   # because the offset came in via the toolbar manual byte entry.
   end = request.GET.get("end")
   end = request.GET.get("end")
   if end:
   if end:
-      end = int(end)
-  begin = request.GET.get("begin")
+    end = int(end)
+  begin = request.GET.get("begin", 1)
   if begin:
   if begin:
-      # Subtract one to zero index for file read
-      begin = int(begin) - 1
+    # Subtract one to zero index for file read
+    begin = int(begin) - 1
   if end:
   if end:
     offset = begin
     offset = begin
     length = end - begin
     length = end - begin
@@ -379,7 +385,6 @@ def display(request, path):
     # Display first block by default.
     # Display first block by default.
     offset = int(request.GET.get("offset", 0))
     offset = int(request.GET.get("offset", 0))
 
 
-
   mode = request.GET.get("mode")
   mode = request.GET.get("mode")
   compression = request.GET.get("compression")
   compression = request.GET.get("compression")
 
 
@@ -392,8 +397,8 @@ def display(request, path):
   if length > MAX_CHUNK_SIZE_BYTES:
   if length > MAX_CHUNK_SIZE_BYTES:
     raise PopupException("Cannot request chunks greater than %d bytes" % MAX_CHUNK_SIZE_BYTES)
     raise PopupException("Cannot request chunks greater than %d bytes" % MAX_CHUNK_SIZE_BYTES)
 
 
-
-  if not compression:
+  # Auto gzip detection, unless we are explicitly told to view binary
+  if not compression and mode != 'binary':
     if path.endswith('.gz') and detect_gzip(request.fs.open(path).read(2)):
     if path.endswith('.gz') and detect_gzip(request.fs.open(path).read(2)):
       compression = 'gzip'
       compression = 'gzip'
       offset = 0
       offset = 0
@@ -413,7 +418,6 @@ def display(request, path):
         raise PopupException("Failed to decompress file")
         raise PopupException("Failed to decompress file")
     finally:
     finally:
       f.close()
       f.close()
-
   else:
   else:
     try:
     try:
       f.seek(offset)
       f.seek(offset)
@@ -421,30 +425,24 @@ def display(request, path):
     finally:
     finally:
       f.close()
       f.close()
 
 
-  masked = None
-
-
-  if not mode:
-    # Auto-detect mode:
-    (mask_count, masked) = xxd.mask_not_printable(contents)
-    if mask_count and float(mask_count)/len(contents) > BINARY_PERCENTAGE:
-      mode = "binary"
-
-    else:
-      mode = "text"
+  # Get contents as string for text mode, or at least try
+  uni_contents = None
+  if not mode or mode == 'text':
+    uni_contents = unicode(contents, encoding, errors='replace')
+    is_binary = uni_contents.find(i18n.REPLACEMENT_CHAR) != -1
+    # Auto-detect mode
+    if not mode:
+      mode = is_binary and 'binary' or 'text'
 
 
+  # Get contents as bytes
   if mode == "binary":
   if mode == "binary":
     xxd_out = list(xxd.xxd(offset, contents, BYTES_PER_LINE, BYTES_PER_SENTENCE))
     xxd_out = list(xxd.xxd(offset, contents, BYTES_PER_LINE, BYTES_PER_SENTENCE))
-  else:
-    # May have been calculated already as part of detection.
-    if not masked:
-      (mask_count, masked) = xxd.mask_not_printable(contents)
 
 
   dirname = posixpath.dirname(path)
   dirname = posixpath.dirname(path)
   # Start with index-like data:
   # Start with index-like data:
   data = _massage_stats(request, request.fs.stats(path))
   data = _massage_stats(request, request.fs.stats(path))
   # And add a view structure:
   # And add a view structure:
-  data["success"] = True;
+  data["success"] = True
   data["view"] = {
   data["view"] = {
     'offset': offset,
     'offset': offset,
     'length': length,
     'length': length,
@@ -464,8 +462,8 @@ def display(request, path):
     data['view']['xxd'] = xxd_out
     data['view']['xxd'] = xxd_out
     data['view']['masked_binary_data'] =  False
     data['view']['masked_binary_data'] =  False
   else:
   else:
-    data['view']['contents'] = masked
-    data['view']['masked_binary_data'] = (mask_count > 0)
+    data['view']['contents'] = uni_contents
+    data['view']['masked_binary_data'] = is_binary
 
 
   return render_with_toolbars("display.mako", request, data)
   return render_with_toolbars("display.mako", request, data)
 
 

+ 158 - 61
apps/filebrowser/src/filebrowser/views_test.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Licensed to Cloudera, Inc. under one
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # distributed with this work for additional information
@@ -33,14 +34,13 @@ def test_chown():
     c = make_logged_in_client(cluster.superuser)
     c = make_logged_in_client(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
 
 
-    PATH = "/test-chown"
+    PATH = u"/test-chown-en-Español"
     cluster.fs.mkdir(PATH)
     cluster.fs.mkdir(PATH)
     c.post("/filebrowser/chown", dict(path=PATH, user="x", group="y"))
     c.post("/filebrowser/chown", dict(path=PATH, user="x", group="y"))
     assert_equal("x", cluster.fs.stats(PATH)["user"])
     assert_equal("x", cluster.fs.stats(PATH)["user"])
     assert_equal("y", cluster.fs.stats(PATH)["group"])
     assert_equal("y", cluster.fs.stats(PATH)["group"])
     c.post("/filebrowser/chown", dict(path=PATH, user="__other__", user_other="z", group="y"))
     c.post("/filebrowser/chown", dict(path=PATH, user="__other__", user_other="z", group="y"))
     assert_equal("z", cluster.fs.stats(PATH)["user"])
     assert_equal("z", cluster.fs.stats(PATH)["user"])
-
   finally:
   finally:
     cluster.shutdown()
     cluster.shutdown()
 
 
@@ -51,16 +51,35 @@ def test_listdir():
     c = make_logged_in_client()
     c = make_logged_in_client()
     cluster.fs.setuser(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
 
 
-    # Delete if there's already something there
+    # These paths contain non-ascii characters. Your editor will need the
+    # corresponding font library to display them correctly.
+    #
+    # We test that mkdir can handle unicode strings as well as byte strings.
+    # And even when the byte string can't be decoded properly (big5), the listdir
+    # still succeeds.
+    orig_paths = [
+      u'greek-Ελληνικά',
+      u'chinese-漢語',
+      'listdir',
+      'non-utf-8-(big5)-\xb2\xc4\xa4@\xb6\xa5\xacq',
+    ]
+
+    prefix = '/test-filebrowser/'
+    for path in orig_paths:
+      cluster.fs.mkdir(prefix + path)
+    response = c.get('/filebrowser/view' + prefix)
+    paths = [f['path'] for f in response.context['files']]
+    for path in orig_paths:
+      if isinstance(path, unicode):
+        uni_path = path
+      else:
+        uni_path = unicode(path, 'utf-8', errors='replace')
+      assert_true(prefix + uni_path in paths,
+                  '%s should be in dir listing %s' % (prefix + uni_path, paths))
+
+    # Delete user's home if there's already something there
     if cluster.fs.isdir("/user/test"):
     if cluster.fs.isdir("/user/test"):
       cluster.fs.rmtree("/user/test")
       cluster.fs.rmtree("/user/test")
-
-    cluster.fs.mkdir('/test-filebrowser/listdir')
-    response = c.get('/filebrowser/view/test-filebrowser/')
-    paths = [f['path'] for f in response.context['files']]
-    assert_true("/test-filebrowser/listdir" in paths)
-
-    # test's home dir doesn't exist yet
     assert_false(response.context['home_directory'])
     assert_false(response.context['home_directory'])
 
 
     # test's home directory now exists. Should be returned.
     # test's home directory now exists. Should be returned.
@@ -68,6 +87,11 @@ def test_listdir():
     response = c.get('/filebrowser/view/test-filebrowser/')
     response = c.get('/filebrowser/view/test-filebrowser/')
     assert_equal(response.context['home_directory'], '/user/test')
     assert_equal(response.context['home_directory'], '/user/test')
   finally:
   finally:
+    try:
+      cluster.fs.rmtree('/test-filebrowser')
+      cluster.fs.rmtree('/user/test')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
     cluster.shutdown()
     cluster.shutdown()
 
 
 
 
@@ -84,56 +108,64 @@ def test_view_gz():
     cluster.fs.mkdir('/test-gz-filebrowser/')
     cluster.fs.mkdir('/test-gz-filebrowser/')
 
 
     f = cluster.fs.open('/test-gz-filebrowser/test-view.gz', "w")
     f = cluster.fs.open('/test-gz-filebrowser/test-view.gz', "w")
-    sdf_string='\x1f\x8b\x08\x082r\xf4K\x00\x03f\x00+NI\xe3\x02\x00\xad\x96b\xc4\x04\x00\x00\x00'
+    sdf_string = '\x1f\x8b\x08\x082r\xf4K\x00\x03f\x00+NI\xe3\x02\x00\xad\x96b\xc4\x04\x00\x00\x00'
     f.write(sdf_string)
     f.write(sdf_string)
     f.close()
     f.close()
 
 
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip')
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip')
     assert_equal(response.context['view']['contents'], "sdf\n")
     assert_equal(response.context['view']['contents'], "sdf\n")
 
 
-# autodetect
+    # autodetect
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz')
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz')
     assert_equal(response.context['view']['contents'], "sdf\n")
     assert_equal(response.context['view']['contents'], "sdf\n")
 
 
-#offset should do nothing
+    # offset should do nothing
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip&offset=1')
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view.gz?compression=gzip&offset=1')
     assert_false(response.context.has_key('view'))
     assert_false(response.context.has_key('view'))
 
 
-
     f = cluster.fs.open('/test-gz-filebrowser/test-view2.gz', "w")
     f = cluster.fs.open('/test-gz-filebrowser/test-view2.gz', "w")
     f.write("hello")
     f.write("hello")
     f.close()
     f.close()
 
 
-#we shouldn't autodetect  non gzip files
+    # we shouldn't autodetect non gzip files
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz')
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz')
     assert_equal(response.context['view']['contents'], "hello")
     assert_equal(response.context['view']['contents'], "hello")
 
 
-#we should fail to do a bad thing if they specify compression when it's not set.
+    # we should fail to do a bad thing if they specify compression when it's not set.
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz?compression=gzip')
     response = c.get('/filebrowser/view/test-gz-filebrowser/test-view2.gz?compression=gzip')
     assert_false(response.context.has_key('view'))
     assert_false(response.context.has_key('view'))
 
 
   finally:
   finally:
+    try:
+      cluster.fs.rmtree('/test-gz-filebrowser/')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
     cluster.shutdown()
     cluster.shutdown()
 
 
+
 @attr('requires_hadoop')
 @attr('requires_hadoop')
-def test_view():
+def test_view_i18n():
   cluster = mini_cluster.shared_cluster(conf=True)
   cluster = mini_cluster.shared_cluster(conf=True)
   try:
   try:
-    c = make_logged_in_client()
     cluster.fs.setuser(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
-
     cluster.fs.mkdir('/test-filebrowser/')
     cluster.fs.mkdir('/test-filebrowser/')
 
 
-    f = cluster.fs.open('/test-filebrowser/test-view', "w")
-    f.write("hello")
-    f.close()
+    # Test viewing files in different encodings
+    content = u'pt-Olá en-hello ch-你好 ko-안녕 ru-Здравствуйте'
+    view_helper(cluster, 'utf-8', content)
+    view_helper(cluster, 'utf-16', content)
 
 
-    response = c.get('/filebrowser/view/test-filebrowser/test-view')
-    assert_equal(response.context['view']['contents'], "hello")
+    content = u'你好-big5'
+    view_helper(cluster, 'big5', content)
+
+    content = u'こんにちは-shift-jis'
+    view_helper(cluster, 'shift_jis', content)
 
 
-    response = c.get('/filebrowser/view/test-filebrowser/test-view?end=2&begin=1')
-    assert_equal(response.context['view']['contents'], "he")
+    content = u'안녕하세요-johab'
+    view_helper(cluster, 'johab', content)
 
 
+    # Test that the default view is home
+    c = make_logged_in_client()
     response = c.get('/filebrowser/view/')
     response = c.get('/filebrowser/view/')
     assert_equal(response.context['path'], '/')
     assert_equal(response.context['path'], '/')
     cluster.fs.mkdir('/user/test')
     cluster.fs.mkdir('/user/test')
@@ -141,60 +173,125 @@ def test_view():
     response = c.get('/filebrowser/view/?default_to_home=1')
     response = c.get('/filebrowser/view/?default_to_home=1')
     assert_equal("http://testserver/filebrowser/view/user/test", response["location"])
     assert_equal("http://testserver/filebrowser/view/user/test", response["location"])
   finally:
   finally:
+    try:
+      cluster.fs.rmtree('/user/test')
+      cluster.fs.rmtree('/test-filebrowser/')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
     cluster.shutdown()
     cluster.shutdown()
 
 
 
 
+def view_helper(cluster, encoding, content):
+  """
+  Write the content in the given encoding directly into the filesystem.
+  Then try to view it and make sure the data is correct.
+  """
+  c = make_logged_in_client()
+  filename = u'/test-filebrowser/test-view-carácter-internacional'
+  bytestring = content.encode(encoding)
+
+  try:
+    f = cluster.fs.open(filename, "w")
+    f.write(bytestring)
+    f.close()
+
+    response = c.get('/filebrowser/view%s?encoding=%s' % (filename, encoding))
+    assert_equal(response.context['view']['contents'], content)
+
+    response = c.get('/filebrowser/view%s?encoding=%s&end=8&begin=1' % (filename, encoding))
+    assert_equal(response.context['view']['contents'],
+                 unicode(bytestring[0:8], encoding, errors='replace'))
+  finally:
+    try:
+      cluster.fs.remove(filename)
+    except:
+      pass
+
+
 @attr('requires_hadoop')
 @attr('requires_hadoop')
-def test_edit():
+def test_edit_i18n():
   cluster = mini_cluster.shared_cluster(conf=True)
   cluster = mini_cluster.shared_cluster(conf=True)
   try:
   try:
-    c = make_logged_in_client(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
     cluster.fs.setuser(cluster.superuser)
-
     cluster.fs.mkdir('/test-filebrowser/')
     cluster.fs.mkdir('/test-filebrowser/')
-    # File doesn't exist - should be empty
-    test_path = '//test-filebrowser//test-edit'
-    # (this path is non-normalized to test normalization too)
-    edit_url = '/filebrowser/edit' + test_path
-    response = c.get(edit_url)
-    assert_equal(response.context['form'].data['path'],
-                 test_path)
-    assert_equal(response.context['form'].data['contents'], "")
-
-    # Just going to the edit page and not hitting save should not
-    # create the file
-    assert_false(cluster.fs.exists(test_path))
 
 
+    # Test utf-8
+    pass_1 = u'en-hello pt-Olá ch-你好 ko-안녕 ru-Здравствуйте'
+    pass_2 = pass_1 + u'yi-העלא'
+    edit_helper(cluster, 'utf-8', pass_1, pass_2)
+
+    # Test utf-16
+    edit_helper(cluster, 'utf-16', pass_1, pass_2)
+
+    # Test cjk
+    pass_1 = u'big5-你好'
+    pass_2 = pass_1 + u'世界'
+    edit_helper(cluster, 'big5', pass_1, pass_2)
+
+    pass_1 = u'shift_jis-こんにちは'
+    pass_2 = pass_1 + u'世界'
+    edit_helper(cluster, 'shift_jis', pass_1, pass_2)
+
+    pass_1 = u'johab-안녕하세요'
+    pass_2 = pass_1 + u'세상'
+    edit_helper(cluster, 'johab', pass_1, pass_2)
+  finally:
+    try:
+      cluster.fs.rmtree('/test-filebrowser/')
+    except:
+      pass      # Don't let cleanup errors mask earlier failures
+    cluster.shutdown()
+
+
+def edit_helper(cluster, encoding, contents_pass_1, contents_pass_2):
+  """
+  Put the content into the file with a specific encoding.
+  """
+  c = make_logged_in_client(cluster.superuser)
+
+  # This path is non-normalized to test normalization too
+  filename = u'//test-filebrowser//./test-edit-carácter-internacional'
+
+  # File doesn't exist - should be empty
+  edit_url = '/filebrowser/edit' + filename
+  response = c.get(edit_url)
+  assert_equal(response.context['form'].data['path'], filename)
+  assert_equal(response.context['form'].data['contents'], "")
+
+  # Just going to the edit page and not hitting save should not
+  # create the file
+  assert_false(cluster.fs.exists(filename))
+
+  try:
     # Put some data in there and post
     # Put some data in there and post
-    new_contents = "hello world from editor"
     response = c.post("/filebrowser/save", dict(
     response = c.post("/filebrowser/save", dict(
-        path=test_path,
-        contents=new_contents), follow=True)
-    assert_equal(response.context['form'].data['path'],
-                 test_path)
-    assert_equal(response.context['form'].data['contents'],
-                 new_contents)
+        path=filename,
+        contents=contents_pass_1,
+        encoding=encoding), follow=True)
+    assert_equal(response.context['form'].data['path'], filename)
+    assert_equal(response.context['form'].data['contents'], contents_pass_1)
 
 
     # File should now exist
     # File should now exist
-    assert_true(cluster.fs.exists(test_path))
+    assert_true(cluster.fs.exists(filename))
     # And its contents should be what we expect
     # And its contents should be what we expect
-    f = cluster.fs.open(test_path)
-    assert_equal(f.read(), new_contents)
+    f = cluster.fs.open(filename)
+    assert_equal(f.read(), contents_pass_1.encode(encoding))
     f.close()
     f.close()
 
 
     # We should be able to overwrite the file with another save
     # We should be able to overwrite the file with another save
-    new_contents = "hello world again from editor"
     response = c.post("/filebrowser/save", dict(
     response = c.post("/filebrowser/save", dict(
-        path=test_path,
-        contents=new_contents), follow=True)
-    assert_equal(response.context['form'].data['path'],
-                 test_path)
-    assert_equal(response.context['form'].data['contents'],
-                 new_contents)
-    f = cluster.fs.open(test_path)
-    assert_equal(f.read(), new_contents)
+        path=filename,
+        contents=contents_pass_2,
+        encoding=encoding), follow=True)
+    assert_equal(response.context['form'].data['path'], filename)
+    assert_equal(response.context['form'].data['contents'], contents_pass_2)
+    f = cluster.fs.open(filename)
+    assert_equal(f.read(), contents_pass_2.encode(encoding))
     f.close()
     f.close()
 
 
     # TODO(todd) add test for maintaining ownership/permissions
     # TODO(todd) add test for maintaining ownership/permissions
   finally:
   finally:
-    cluster.shutdown()
+    try:
+      cluster.fs.remove(filename)
+    except:
+      pass

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -50,6 +50,9 @@ http_500_debug_mode=0
 # Filename of SSL RSA Private Key
 # Filename of SSL RSA Private Key
 ## ssl_private_key=
 ## ssl_private_key=
 
 
+# Default encoding for site data
+## default_site_encoding=utf-8
+
 # Configuration options for user authentication into the web application
 # Configuration options for user authentication into the web application
 # ------------------------------------------------------------------------
 # ------------------------------------------------------------------------
 [[auth]]
 [[auth]]

+ 7 - 0
desktop/core/src/desktop/conf.py

@@ -179,6 +179,13 @@ TIME_ZONE = Config(
   default=os.environ.get("TZ", "America/Los_Angeles")
   default=os.environ.get("TZ", "America/Los_Angeles")
 )
 )
 
 
+DEFAULT_SITE_ENCODING = Config(
+  key='default_site_encoding',
+  help='Default system-wide unicode encoding',
+  type=str,
+  default='utf-8'
+)
+
 SERVER_USER = Config(
 SERVER_USER = Config(
   key="server_user",
   key="server_user",
   help="Username to run servers as",
   help="Username to run servers as",

+ 31 - 0
desktop/core/src/desktop/lib/django_forms.py

@@ -28,6 +28,9 @@ from django.utils.encoding import StrAndUnicode, force_unicode
 import simplejson
 import simplejson
 import urllib
 import urllib
 
 
+import desktop.lib.i18n
+
+
 class MultipleInputWidget(Widget):
 class MultipleInputWidget(Widget):
   """
   """
   Together with MultipleInputField, represents repeating a form element many times,
   Together with MultipleInputField, represents repeating a form element many times,
@@ -158,6 +161,34 @@ class KeyValueField(CharField):
     except Exception:
     except Exception:
       raise ValidationError("Not in key=value format.")
       raise ValidationError("Not in key=value format.")
 
 
+class UnicodeEncodingField(ChoiceOrOtherField):
+  CHOICES = [
+    ('utf-8', 'Unicode UTF8'),
+    ('utf-16', 'Unicode UTF16'),
+    ('latin_1', 'Western ISO-8859-1'),
+    ('cyrillic', 'Cryrillic'),
+    ('arabic', 'Arabic'),
+    ('greek', 'Greek'),
+    ('hebrew', 'Hebrew'),
+    ('shift_jis', 'Japanese (Shift-JIS)'),
+    ('euc-jp', 'Japanese (EUC-JP)'),
+    ('iso2022_jp', 'Japanese (ISO-2022-JP)'),
+    ('euc-kr', 'Korean (EUC-KR)'),
+    ('iso2022-kr', 'Korean (ISO-2022-KR)'),
+    ('gbk', 'Chinese Simplified (GBK)'),
+    ('big5hkscs', 'Chinese Traditional (Big5)'),
+    ('ascii', 'ASCII'),
+  ]
+
+  def __init__(self, initial=None, *args, **kwargs):
+    ChoiceOrOtherField.__init__(self, UnicodeEncodingField.CHOICES, initial, *args, **kwargs)
+
+  def clean(self, value):
+    encoding = value[0]
+    if encoding and not desktop.lib.i18n.validate_encoding(encoding):
+      raise forms.ValidationError("'%s' encoding is not available" % (encoding,))
+    return value
+
 
 
 class MultiForm(object):
 class MultiForm(object):
   """
   """

+ 49 - 0
desktop/core/src/desktop/lib/i18n.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Library methods to deal with non-ascii data
+"""
+
+import codecs
+import desktop.conf
+import logging
+
+SITE_ENCODING = None
+REPLACEMENT_CHAR = u'\ufffd'
+
+def get_site_encoding():
+  """Get the default site encoding"""
+  global SITE_ENCODING
+  if SITE_ENCODING is None:
+    encoding = desktop.conf.DEFAULT_SITE_ENCODING.get()
+    if not validate_encoding(encoding):
+      default = desktop.conf.DEFAULT_SITE_ENCODING.config.default_value
+      msg = 'Invalid HUE configuration value for %s: "%s". Using default "%s"' % \
+                  (desktop.conf.DEFAULT_SITE_ENCODING.config.key, encoding, default)
+      logging.error(msg)
+      encoding = default
+    SITE_ENCODING = encoding
+  return SITE_ENCODING
+
+def validate_encoding(encoding):
+  """Return True/False on whether the system understands this encoding"""
+  try:
+    codecs.lookup(encoding)
+    return True
+  except LookupError:
+    return False

+ 2 - 1
desktop/core/src/desktop/middleware.py

@@ -24,6 +24,7 @@ import django.db
 from django.http import HttpResponseRedirect, HttpResponse
 from django.http import HttpResponseRedirect, HttpResponse
 from django.shortcuts import render_to_response
 from django.shortcuts import render_to_response
 from django.utils.http import urlquote
 from django.utils.http import urlquote
+from django.utils.encoding import iri_to_uri
 import django.views.static
 import django.views.static
 import django.views.generic.simple
 import django.views.generic.simple
 import django.contrib.auth.views
 import django.contrib.auth.views
@@ -97,7 +98,7 @@ class JFrameMiddleware(object):
       query_string = get_params.urlencode()
       query_string = get_params.urlencode()
       if query_string:
       if query_string:
         path = request.path + "?" + query_string
         path = request.path + "?" + query_string
-    response['X-Hue-JFrame-Path'] = path
+    response['X-Hue-JFrame-Path'] = iri_to_uri(path)
     if response.status_code == 200:
     if response.status_code == 200:
       if is_jframe_request(request):
       if is_jframe_request(request):
         if hasattr(request, "flash"):
         if hasattr(request, "flash"):

+ 53 - 13
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -31,6 +31,7 @@ from thrift.transport import TTransport
 from thrift.transport import TSocket
 from thrift.transport import TSocket
 from thrift.protocol import TBinaryProtocol
 from thrift.protocol import TBinaryProtocol
 
 
+from django.utils.encoding import smart_str, force_unicode
 from desktop.lib import thrift_util
 from desktop.lib import thrift_util
 from hadoop.api.hdfs import Namenode, Datanode
 from hadoop.api.hdfs import Namenode, Datanode
 from hadoop.api.hdfs.constants import QUOTA_DONT_SET, QUOTA_RESET
 from hadoop.api.hdfs.constants import QUOTA_DONT_SET, QUOTA_RESET
@@ -59,12 +60,24 @@ DEFAULT_READ_SIZE = 1024*1024 # 1MB
 WRITE_BUFFER_SIZE = 128*1024 # 128K
 WRITE_BUFFER_SIZE = 128*1024 # 128K
 
 
 # Class that we translate into PermissionDeniedException
 # Class that we translate into PermissionDeniedException
-HADOOP_ACCESSCONTROLEXCEPTION="org.apache.hadoop.security.AccessControlException"
+HADOOP_ACCESSCONTROLEXCEPTION = "org.apache.hadoop.security.AccessControlException"
 
 
 # Timeout for thrift calls to NameNode
 # Timeout for thrift calls to NameNode
 NN_THRIFT_TIMEOUT = 15
 NN_THRIFT_TIMEOUT = 15
 DN_THRIFT_TIMEOUT = 3
 DN_THRIFT_TIMEOUT = 3
 
 
+# Encoding used by HDFS namespace
+HDFS_ENCODING = 'utf-8'
+
+def encode_fs_path(path):
+  """encode_fs_path(path) -> byte string in utf8"""
+  return smart_str(path, HDFS_ENCODING, errors='strict')
+
+def decode_fs_path(path):
+  """decode_fs_path(bytestring) -> unicode path"""
+  return force_unicode(path, HDFS_ENCODING, errors='strict')
+
+
 class HadoopFileSystem(object):
 class HadoopFileSystem(object):
   """
   """
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.
@@ -127,7 +140,6 @@ class HadoopFileSystem(object):
 
 
     raise Exception("Hadoop binary (%s) does not exist." % self.hadoop_bin_path)
     raise Exception("Hadoop binary (%s) does not exist." % self.hadoop_bin_path)
 
 
-
   @property
   @property
   def uri(self):
   def uri(self):
     return self._get_hdfs_base()
     return self._get_hdfs_base()
@@ -161,6 +173,7 @@ class HadoopFileSystem(object):
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def remove(self, path):
   def remove(self, path):
+    path = encode_fs_path(path)
     stat = self._hadoop_stat(path)
     stat = self._hadoop_stat(path)
     if not stat:
     if not stat:
       raise IOError("File not found: %s" % path)
       raise IOError("File not found: %s" % path)
@@ -176,11 +189,13 @@ class HadoopFileSystem(object):
   def mkdir(self, path, mode=0755):
   def mkdir(self, path, mode=0755):
     # TODO(todd) there should be a mkdir that isn't mkdirHIER
     # TODO(todd) there should be a mkdir that isn't mkdirHIER
     # (this is mkdir -p I think)
     # (this is mkdir -p I think)
+    path = encode_fs_path(path)
     success = self.nn_client.mkdirhier(self.request_context, normpath(path), mode)
     success = self.nn_client.mkdirhier(self.request_context, normpath(path), mode)
     if not success:
     if not success:
       raise IOError("mkdir failed")
       raise IOError("mkdir failed")
 
 
   def _rmdir(self, path, recursive=False):
   def _rmdir(self, path, recursive=False):
+    path = encode_fs_path(path)
     stat = self._hadoop_stat(path)
     stat = self._hadoop_stat(path)
     if not stat:
     if not stat:
       raise IOError("Directory not found: %s" % (path,))
       raise IOError("Directory not found: %s" % (path,))
@@ -202,20 +217,30 @@ class HadoopFileSystem(object):
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def listdir(self, path):
   def listdir(self, path):
+    path = encode_fs_path(path)
     stats = self.nn_client.ls(self.request_context, normpath(path))
     stats = self.nn_client.ls(self.request_context, normpath(path))
-    return [self.basename(stat.path) for stat in stats]
+    return [self.basename(decode_fs_path(stat.path)) for stat in stats]
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def listdir_stats(self, path):
   def listdir_stats(self, path):
+    path = encode_fs_path(path)
     stats = self.nn_client.ls(self.request_context, normpath(path))
     stats = self.nn_client.ls(self.request_context, normpath(path))
     return [self._unpack_stat(s) for s in stats]
     return [self._unpack_stat(s) for s in stats]
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def get_content_summaries(self, paths):
   def get_content_summaries(self, paths):
-    return self.nn_client.multiGetContentSummary(self.request_context, [normpath(path) for path in paths])
+    path = encode_fs_path(path)
+    summaries = self.nn_client.multiGetContentSummary(self.request_context,
+                                                      [normpath(path) for path in paths])
+    def _fix_summary(summary):
+      summary.path = decode_fs_path(summary.path)
+      return summary
+    return [_fix_summary(s) for s in summaries]
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def rename(self, old, new):
   def rename(self, old, new):
+    old = encode_fs_path(old)
+    new = encode_fs_path(new)
     success = self.nn_client.rename(
     success = self.nn_client.rename(
       self.request_context, normpath(old), normpath(new))
       self.request_context, normpath(old), normpath(new))
     if not success: #TODO(todd) these functions should just throw if failed
     if not success: #TODO(todd) these functions should just throw if failed
@@ -267,10 +292,12 @@ class HadoopFileSystem(object):
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def chmod(self, path, mode):
   def chmod(self, path, mode):
+    path = encode_fs_path(path)
     self.nn_client.chmod(self.request_context, normpath(path), mode)
     self.nn_client.chmod(self.request_context, normpath(path), mode)
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def chown(self, path, user, group):
   def chown(self, path, user, group):
+    path = encode_fs_path(path)
     self.nn_client.chown(self.request_context, normpath(path), user, group)
     self.nn_client.chown(self.request_context, normpath(path), user, group)
 
 
   @_coerce_exceptions
   @_coerce_exceptions
@@ -281,6 +308,7 @@ class HadoopFileSystem(object):
                  used_bytes=used,
                  used_bytes=used,
                  available_bytes=available),
                  available_bytes=available),
       )
       )
+
   @_coerce_exceptions
   @_coerce_exceptions
   def _get_blocks(self, path, offset, length):
   def _get_blocks(self, path, offset, length):
     """
     """
@@ -291,13 +319,20 @@ class HadoopFileSystem(object):
         thriftPort=53417, state=1, remaining=18987925504, host='127.0.0.1',
         thriftPort=53417, state=1, remaining=18987925504, host='127.0.0.1',
         storageID='DS-1238582576-127.0.1.1-50010-1240968238474', dfsUsed=36864)], numBytes=424)]
         storageID='DS-1238582576-127.0.1.1-50010-1240968238474', dfsUsed=36864)], numBytes=424)]
     """
     """
-    return self.nn_client.getBlocks(self.request_context, normpath(path), offset, length)
+    path = encode_fs_path(path)
+    blocks = self.nn_client.getBlocks(self.request_context, normpath(path), offset, length)
+    def _fix_block(blk):
+      blk.path = decode_fs_path(blk.path)
+      return blk
+    return [_fix_block(blk) for blk in blocks]
 
 
 
 
   def _hadoop_stat(self, path):
   def _hadoop_stat(self, path):
     """Returns None if file does not exist."""
     """Returns None if file does not exist."""
+    path = encode_fs_path(path)
     try:
     try:
       stat = self.nn_client.stat(self.request_context, normpath(path))
       stat = self.nn_client.stat(self.request_context, normpath(path))
+      stat.path = decode_fs_path(stat.path)
       return stat
       return stat
     except IOException, ioe:
     except IOException, ioe:
       if ioe.clazz == 'java.io.FileNotFoundException':
       if ioe.clazz == 'java.io.FileNotFoundException':
@@ -315,6 +350,7 @@ class HadoopFileSystem(object):
     @param len the number of bytes to read
     @param len the number of bytes to read
     """
     """
     errs = []
     errs = []
+    block.path = encode_fs_path(block.path)
     for node in block.nodes:
     for node in block.nodes:
       dn_conn = self._connect_dn(node)
       dn_conn = self._connect_dn(node)
       try:
       try:
@@ -335,7 +371,7 @@ class HadoopFileSystem(object):
     @param path The path to the given hdfs resource
     @param path The path to the given hdfs resource
     @param size The amount of bytes that a given subtree of files can grow to.
     @param size The amount of bytes that a given subtree of files can grow to.
     """
     """
-
+    path = encode_fs_path(path)
     if normpath(path) == '/':
     if normpath(path) == '/':
       raise ValueError('Cannot set quota for "/"')
       raise ValueError('Cannot set quota for "/"')
 
 
@@ -352,7 +388,7 @@ class HadoopFileSystem(object):
     @param path The path to the given hdfs resource
     @param path The path to the given hdfs resource
     @param num_files The amount of files that can exist within that subtree.
     @param num_files The amount of files that can exist within that subtree.
     """
     """
-
+    path = encode_fs_path(path)
     if normpath(path) == '/':
     if normpath(path) == '/':
       raise ValueError('Cannot set quota for "/"')
       raise ValueError('Cannot set quota for "/"')
 
 
@@ -366,6 +402,7 @@ class HadoopFileSystem(object):
     """
     """
     Remove the diskspace quota at a given path
     Remove the diskspace quota at a given path
     """
     """
+    path = encode_fs_path(path)
     self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, QUOTA_RESET)
     self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, QUOTA_RESET)
 
 
   @_coerce_exceptions
   @_coerce_exceptions
@@ -373,6 +410,7 @@ class HadoopFileSystem(object):
     """
     """
     Remove the namespace quota at a given path
     Remove the namespace quota at a given path
     """
     """
+    path = encode_fs_path(path)
     self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_RESET, QUOTA_DONT_SET)
     self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_RESET, QUOTA_DONT_SET)
 
 
 
 
@@ -381,6 +419,7 @@ class HadoopFileSystem(object):
     """
     """
     Get the current space quota in bytes for disk space. None if it is unset
     Get the current space quota in bytes for disk space. None if it is unset
     """
     """
+    path = encode_fs_path(path)
     space_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).spaceQuota
     space_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).spaceQuota
     if space_quota == QUOTA_RESET or space_quota == QUOTA_DONT_SET:
     if space_quota == QUOTA_RESET or space_quota == QUOTA_DONT_SET:
       return None
       return None
@@ -393,6 +432,7 @@ class HadoopFileSystem(object):
     """
     """
     Get the current quota in number of files. None if it is unset
     Get the current quota in number of files. None if it is unset
     """
     """
+    path = encode_fs_path(path)
     file_count_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).quota
     file_count_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).quota
     if file_count_quota == QUOTA_RESET or file_count_quota == QUOTA_DONT_SET:
     if file_count_quota == QUOTA_RESET or file_count_quota == QUOTA_DONT_SET:
       return None
       return None
@@ -406,6 +446,7 @@ class HadoopFileSystem(object):
     "space_used", and "space_quota".  The quotas
     "space_used", and "space_quota".  The quotas
     may be None.
     may be None.
     """
     """
+    path = encode_fs_path(path)
     summary = self.nn_client.getContentSummary(self.request_context, normpath(path))
     summary = self.nn_client.getContentSummary(self.request_context, normpath(path))
     ret = dict()
     ret = dict()
     ret["file_count"] = summary.fileCount
     ret["file_count"] = summary.fileCount
@@ -430,7 +471,6 @@ class HadoopFileSystem(object):
     client.close = lambda: transport.close()
     client.close = lambda: transport.close()
     return client
     return client
 
 
-
   @staticmethod
   @staticmethod
   def _unpack_stat(stat):
   def _unpack_stat(stat):
     """Unpack a Thrift "Stat" object into a dictionary that looks like fs.stat"""
     """Unpack a Thrift "Stat" object into a dictionary that looks like fs.stat"""
@@ -441,7 +481,7 @@ class HadoopFileSystem(object):
       mode |= statconsts.S_IFREG
       mode |= statconsts.S_IFREG
 
 
     return {
     return {
-      'path': stat.path,
+      'path': decode_fs_path(stat.path),
       'size': stat.length,
       'size': stat.length,
       'mtime': stat.mtime / 1000,
       'mtime': stat.mtime / 1000,
       'mode': mode,
       'mode': mode,
@@ -630,7 +670,7 @@ class FileUpload(object):
                            "-Dfs.default.name=" + self.fs._get_hdfs_base(),
                            "-Dfs.default.name=" + self.fs._get_hdfs_base(),
                            "-Dhadoop.job.ugi=" + self.fs.ugi] + \
                            "-Dhadoop.job.ugi=" + self.fs.ugi] + \
                            extra_confs + \
                            extra_confs + \
-                           ["-put", "-", path]
+                           ["-put", "-", encode_fs_path(path)]
     self.path = path
     self.path = path
     self.putter = subprocess.Popen(self.subprocess_cmd,
     self.putter = subprocess.Popen(self.subprocess_cmd,
                                    stdin=subprocess.PIPE,
                                    stdin=subprocess.PIPE,
@@ -647,9 +687,9 @@ class FileUpload(object):
     try:
     try:
       (stdout, stderr) = self.putter.communicate()
       (stdout, stderr) = self.putter.communicate()
     except IOError, ioe:
     except IOError, ioe:
-        logging.debug("Saw IOError writing %r" % self.path, exc_info=1)
-        if ioe.errno == 32: # Broken Pipe
-           stdout, stderr = self.putter.communicate()
+      logging.debug("Saw IOError writing %r" % self.path, exc_info=1)
+      if ioe.errno == errno.EPIPE:
+        stdout, stderr = self.putter.communicate()
     self.closed = True
     self.closed = True
     if stderr:
     if stderr:
       LOG.warn("HDFS FileUpload (cmd='%s')outputted stderr:\n%s" %
       LOG.warn("HDFS FileUpload (cmd='%s')outputted stderr:\n%s" %

+ 62 - 14
desktop/libs/hadoop/src/hadoop/fs/hadoopfs_test.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Licensed to Cloudera, Inc. under one
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # distributed with this work for additional information
@@ -203,35 +204,35 @@ def test_quota_space():
       fs.rmtree('/tmp/foo2')
       fs.rmtree('/tmp/foo2')
     fs.mkdir("/tmp/foo2", 0777) # this also tests more restrictive subdirectories
     fs.mkdir("/tmp/foo2", 0777) # this also tests more restrictive subdirectories
     ONE_HUNDRED_192_MEGS = 1024 * 1024 * 192
     ONE_HUNDRED_192_MEGS = 1024 * 1024 * 192
-    
-    fs.set_diskspace_quota("/tmp/foo2", ONE_HUNDRED_192_MEGS) 
+
+    fs.set_diskspace_quota("/tmp/foo2", ONE_HUNDRED_192_MEGS)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), ONE_HUNDRED_192_MEGS)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), ONE_HUNDRED_192_MEGS)
 
 
-    f = fs.open('/tmp/foo2/asdf', 'w')	 # we should be able to do this 
-    f.write('a') 
+    f = fs.open('/tmp/foo2/asdf', 'w')	 # we should be able to do this
+    f.write('a')
     f.close()
     f.close()
 
 
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), ONE_HUNDRED_192_MEGS)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), ONE_HUNDRED_192_MEGS)
 
 
-    fs.set_diskspace_quota("/tmp/foo2", 1) 
+    fs.set_diskspace_quota("/tmp/foo2", 1)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), 1)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), 1)
 
 
-    f = fs.open('/tmp/foo2/asdfsd', 'w')	 
+    f = fs.open('/tmp/foo2/asdfsd', 'w')
     f.write('a')
     f.write('a')
     assert_raises(IOError, f.close)
     assert_raises(IOError, f.close)
 
 
     fs.clear_diskspace_quota("/tmp/foo2")
     fs.clear_diskspace_quota("/tmp/foo2")
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), None)
     assert_equals(fs.get_diskspace_quota("/tmp/foo2"), None)
 
 
-    f = fs.open('/tmp/foo2/asdfsda', 'w')	 
-    f.write('a') 
+    f = fs.open('/tmp/foo2/asdfsda', 'w')
+    f.write('a')
     f.close()
     f.close()
 
 
     fs.mkdir("/tmp/baz/bar", 0777)  # this tests more permissive subdirectories
     fs.mkdir("/tmp/baz/bar", 0777)  # this tests more permissive subdirectories
     fs.set_diskspace_quota("/tmp/baz", 1)
     fs.set_diskspace_quota("/tmp/baz", 1)
-    fs.set_diskspace_quota("/tmp/baz/bar", ONE_HUNDRED_192_MEGS) 
+    fs.set_diskspace_quota("/tmp/baz/bar", ONE_HUNDRED_192_MEGS)
 
 
-    f = fs.open('/tmp/baz/bar', 'w')	 
+    f = fs.open('/tmp/baz/bar', 'w')
     f.write('aaaa') #should violate the subquota
     f.write('aaaa') #should violate the subquota
     assert_raises(IOError, f.close)
     assert_raises(IOError, f.close)
 
 
@@ -256,17 +257,17 @@ def test_quota_namespace_count():
     fs.mkdir("/tmp/foo2", 0777)
     fs.mkdir("/tmp/foo2", 0777)
 
 
     # check the get_namespace_quota function
     # check the get_namespace_quota function
-    fs.set_namespace_quota("/tmp/foo2", 4) 
+    fs.set_namespace_quota("/tmp/foo2", 4)
     assert_equals(fs.get_namespace_quota("/tmp/foo2"), 4)
     assert_equals(fs.get_namespace_quota("/tmp/foo2"), 4)
 
 
     # violate the namespace count
     # violate the namespace count
     for i in range(3):
     for i in range(3):
       f = fs.open('/tmp/foo2/works' + str(i), 'w')
       f = fs.open('/tmp/foo2/works' + str(i), 'w')
-      f.write('a') 
+      f.write('a')
       f.close()
       f.close()
 
 
     f = fs.open('/tmp/foo2/asdfsdc', 'w')
     f = fs.open('/tmp/foo2/asdfsdc', 'w')
-    f.write('a') 
+    f.write('a')
     assert_raises(IOError, f.close)
     assert_raises(IOError, f.close)
 
 
     # Check summary stats
     # Check summary stats
@@ -281,9 +282,56 @@ def test_quota_namespace_count():
     assert_equals(fs.get_namespace_quota("/tmp/foo2"), None)
     assert_equals(fs.get_namespace_quota("/tmp/foo2"), None)
 
 
     f = fs.open('/tmp/foo2/asdfsdd', 'w')
     f = fs.open('/tmp/foo2/asdfsdd', 'w')
-    f.write('a') 
+    f.write('a')
     f.close()
     f.close()
   finally:
   finally:
     if fs.exists('/tmp/foo2'):
     if fs.exists('/tmp/foo2'):
       fs.rmtree("/tmp/foo2")
       fs.rmtree("/tmp/foo2")
     cluster.shutdown()
     cluster.shutdown()
+
+
+@attr('requires_hadoop')
+def test_i18n_namespace():
+  cluster = mini_cluster.shared_cluster()
+  cluster.fs.setuser(cluster.superuser)
+
+  def check_existence(name, parent, present=True):
+    assertion = present and assert_true or assert_false
+    listing = cluster.fs.listdir(parent)
+    assertion(name in listing, "%s should be in %s" % (name, listing))
+
+  name = u'pt-Olá_ch-你好_ko-안녕_ru-Здравствуйте'
+  prefix = '/tmp/i18n'
+  dir_path = '%s/%s' % (prefix, name)
+  file_path = '%s/%s' % (dir_path, name)
+
+  try:
+    # Create a directory
+    cluster.fs.mkdir(dir_path)
+    # Create a file (same name) in the directory
+    cluster.fs.open(file_path, 'w').close()
+
+    # Directory is there
+    check_existence(name, prefix)
+    # File is there
+    check_existence(name, dir_path)
+
+    # Test rename
+    new_file_path = file_path + '.new'
+    cluster.fs.rename(file_path, new_file_path)
+    # New file is there
+    check_existence(name + '.new', dir_path)
+
+    # Test remove
+    cluster.fs.remove(new_file_path)
+    check_existence(name + '.new', dir_path, present=False)
+
+    # Test rmtree
+    cluster.fs.rmtree(dir_path)
+    check_existence(name, prefix, present=False)
+  finally:
+    try:
+      cluster.fs.rmtree(prefix)
+    except:
+      pass
+    cluster.shutdown()

+ 4 - 4
dist/README

@@ -5,11 +5,11 @@ If you're impatient, these are the key steps.  Please check the full manual
 for more details.
 for more details.
 
 
 ## Install
 ## Install
-$ HADOOP_HOME=/path/to/hadoop-0.20.1+152 PREFIX=/path/to/install/into \
-  make install
+$ HADOOP_HOME=/usr/lib/hadoop-0.20 PREFIX=/usr/local make install
+
 ## Install plug-ins
 ## Install plug-ins
 $ cd /usr/lib/hadoop/lib
 $ cd /usr/lib/hadoop/lib
-$ ln -s /usr/share/hue/desktop/libs/hadoop/java-lib/hue*jar
+$ ln -s /usr/local/hue/desktop/libs/hadoop/java-lib/hue*jar
 
 
 ## Configure Hadoop
 ## Configure Hadoop
 Edit hdfs-site.xml:
 Edit hdfs-site.xml:
@@ -37,4 +37,4 @@ Edit mapred-site.xml:
 </property>
 </property>
 
 
 ## Run!
 ## Run!
-$ /usr/share/hue/build/env/bin/supervisor
+$ /usr/local/hue/build/env/bin/supervisor