Эх сурвалжийг харах

HUE-54. beeswax can't handle unicode data

Fixed encoding checks for create table wizard.
Improved error messages in app_reg.
Log error if we fail to parse HUE config.
bc Wong 15 жил өмнө
parent
commit
3f0a6a4a74

+ 4 - 5
apps/beeswax/src/beeswax/create_table.py

@@ -24,7 +24,7 @@ import gzip
 
 
 from django.core import urlresolvers
 from django.core import urlresolvers
 
 
-from desktop.lib import django_mako
+from desktop.lib import django_mako, i18n
 from desktop.lib.django_util import render, PopupException
 from desktop.lib.django_util import render, PopupException
 from desktop.lib.django_forms import MultiForm
 from desktop.lib.django_forms import MultiForm
 from hadoop.fs import hadoopfs
 from hadoop.fs import hadoopfs
@@ -76,7 +76,6 @@ def create_table(request):
   ))
   ))
 
 
 
 
-IMPORT_DEFAULT_ENCODING = 'utf8'
 IMPORT_PEEK_SIZE = 8192
 IMPORT_PEEK_SIZE = 8192
 IMPORT_PEEK_NLINES = 10
 IMPORT_PEEK_NLINES = 10
 DELIMITERS = [ hive_val for hive_val, _, _ in beeswax.common.TERMINATORS ]
 DELIMITERS = [ hive_val for hive_val, _, _ in beeswax.common.TERMINATORS ]
@@ -98,7 +97,7 @@ def import_wizard(request):
     - No partition table.
     - No partition table.
     - Does not work with binary data.
     - Does not work with binary data.
   """
   """
-  encoding = IMPORT_DEFAULT_ENCODING
+  encoding = i18n.get_site_encoding()
 
 
   if request.method == 'POST':
   if request.method == 'POST':
     # Have a while loop to allow an easy way to break
     # Have a while loop to allow an easy way to break
@@ -389,7 +388,7 @@ class GzipFileReader(object):
     except IOError:
     except IOError:
       return None
       return None
     try:
     try:
-      return unicode(data, encoding).split('\n')[:IMPORT_PEEK_NLINES]
+      return unicode(data, encoding, errors='replace').split('\n')[:IMPORT_PEEK_NLINES]
     except UnicodeError:
     except UnicodeError:
       return None
       return None
 
 
@@ -405,7 +404,7 @@ class TextFileReader(object):
     """readlines(fileobj, encoding) -> list of lines"""
     """readlines(fileobj, encoding) -> list of lines"""
     try:
     try:
       data = fileobj.read(IMPORT_PEEK_SIZE)
       data = fileobj.read(IMPORT_PEEK_SIZE)
-      return unicode(data, encoding).split('\n')[:IMPORT_PEEK_NLINES]
+      return unicode(data, encoding, errors='replace').split('\n')[:IMPORT_PEEK_NLINES]
     except UnicodeError:
     except UnicodeError:
       return None
       return None
 
 

+ 3 - 3
apps/beeswax/src/beeswax/templates/beeswax_components.mako

@@ -45,7 +45,7 @@
     for key, value in attributes.iteritems():
     for key, value in attributes.iteritems():
       if key == "klass":
       if key == "klass":
         key = "class"
         key = "class"
-      ret_str += "%s='%s'" % (key.replace("_", "-"), str(value))
+      ret_str += "%s='%s'" % (key.replace("_", "-"), unicode(value))
     return ret_str
     return ret_str
 
 
   if not attrs:
   if not attrs:
@@ -71,12 +71,12 @@
   titlecls = ' '.join(title_classes)
   titlecls = ' '.join(title_classes)
 %>
 %>
   % if field.is_hidden:
   % if field.is_hidden:
-    ${str(field) | n}
+    ${unicode(field) | n}
   % else:
   % else:
     <dt class="${titlecls}" ${make_attr_str(dt_attrs) | n}>${field.label_tag() | n}</dt>
     <dt class="${titlecls}" ${make_attr_str(dt_attrs) | n}>${field.label_tag() | n}</dt>
     <dd class="${cls}" ${make_attr_str(dd_attrs) | n}>
     <dd class="${cls}" ${make_attr_str(dd_attrs) | n}>
       % if render_default:
       % if render_default:
-        ${str(field) | n}
+        ${unicode(field) | n}
       % else:
       % else:
         % if tag == 'textarea':
         % if tag == 'textarea':
           <textarea name="${field.html_name | n}" ${make_attr_str(attrs) | n} />${extract_field_data(field) or ''}</textarea>
           <textarea name="${field.html_name | n}" ${make_attr_str(attrs) | n} />${extract_field_data(field) or ''}</textarea>

+ 2 - 2
apps/beeswax/src/beeswax/templates/util.mako

@@ -21,10 +21,10 @@
 
 
 <%def name="render_field(field)">
 <%def name="render_field(field)">
   % if field.is_hidden:
   % if field.is_hidden:
-    ${str(field) | n}
+    ${unicode(field) | n}
   % else:
   % else:
     <dt>${field.label_tag() | n}</dt>
     <dt>${field.label_tag() | n}</dt>
-    <dd>${str(field) | n}</dd>
+    <dd>${unicode(field) | n}</dd>
     % if len(field.errors):
     % if len(field.errors):
       <dd class="ccs-error">
       <dd class="ccs-error">
         ${render_error(field.errors)}
         ${render_error(field.errors)}

+ 2 - 2
apps/beeswax/src/beeswax/tests.py

@@ -757,7 +757,7 @@ for x in sys.stdin:
       f.write(data)
       f.write(data)
       f.close()
       f.close()
 
 
-    write_file('/tmp/space.dat', RAW_FIELDS, ' ')
+    write_file('/tmp/spacé.dat', RAW_FIELDS, ' ')
     write_file('/tmp/tab.dat', RAW_FIELDS, '\t')
     write_file('/tmp/tab.dat', RAW_FIELDS, '\t')
     write_file('/tmp/comma.dat', RAW_FIELDS, ',')
     write_file('/tmp/comma.dat', RAW_FIELDS, ',')
     write_file('/tmp/comma.dat.gz', RAW_FIELDS, ',', do_gzip=True)
     write_file('/tmp/comma.dat.gz', RAW_FIELDS, ',', do_gzip=True)
@@ -781,7 +781,7 @@ for x in sys.stdin:
     # Make sure space works
     # Make sure space works
     resp = self.client.post('/beeswax/create/import_wizard', {
     resp = self.client.post('/beeswax/create/import_wizard', {
       'submit_preview': 'on',
       'submit_preview': 'on',
-      'path': '/tmp/space.dat',
+      'path': '/tmp/spacé.dat',
       'name': 'test_create_import',
       'name': 'test_create_import',
       'delimiter_0': ' ',
       'delimiter_0': ' ',
       'delimiter_1': '',
       'delimiter_1': '',

+ 6 - 1
desktop/core/src/desktop/lib/conf.py

@@ -461,7 +461,12 @@ def _configs_from_dir(conf_dir):
     if filename.startswith(".") or not filename.endswith('.ini'):
     if filename.startswith(".") or not filename.endswith('.ini'):
       continue
       continue
     logging.debug("Loading configuration from: %s" % filename)
     logging.debug("Loading configuration from: %s" % filename)
-    conf = configobj.ConfigObj(os.path.join(conf_dir, filename))
+    try:
+      conf = configobj.ConfigObj(os.path.join(conf_dir, filename))
+    except configobj.ConfigObjError, ex:
+      logging.error("Error in configuration file '%s': %s" %
+                    (os.path.join(conf_dir, filename), ex))
+      raise
     conf['DEFAULT'] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir())
     conf['DEFAULT'] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir())
     yield conf
     yield conf
 
 

+ 13 - 9
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -444,17 +444,21 @@ class HadoopFileSystem(object):
     @param len the number of bytes to read
     @param len the number of bytes to read
     """
     """
     errs = []
     errs = []
+    unipath = block.path
     block.path = encode_fs_path(block.path)
     block.path = encode_fs_path(block.path)
-    for node in block.nodes:
-      dn_conn = self._connect_dn(node)
-      try:
+    try:
+      for node in block.nodes:
+        dn_conn = self._connect_dn(node)
         try:
         try:
-          data = dn_conn.readBlock(self.request_context, block, offset, len)
-          return data.data
-        except Exception, e:
-          errs.append(e)
-      finally:
-        dn_conn.close()
+          try:
+            data = dn_conn.readBlock(self.request_context, block, offset, len)
+            return data.data
+          except Exception, e:
+            errs.append(e)
+        finally:
+          dn_conn.close()
+    finally:
+      block.path = unipath
 
 
     raise IOError("Could not read block %s from any replicas: %s" % (block, repr(errs)))
     raise IOError("Could not read block %s from any replicas: %s" % (block, repr(errs)))
 
 

+ 2 - 2
tools/app_reg/registry.py

@@ -84,7 +84,7 @@ class AppRegistry(object):
       elif version_diff < 0:
       elif version_diff < 0:
         LOG.info('Upgrading %s from version %s' % (app, existing.version))
         LOG.info('Upgrading %s from version %s' % (app, existing.version))
       elif version_diff > 0:
       elif version_diff > 0:
-        LOG.error('A newer version of %s is already installed' % (app,))
+        LOG.error('A newer version (%s) of %s is already installed' % (existing.version, app))
         return False
         return False
     except KeyError:
     except KeyError:
       pass
       pass
@@ -134,7 +134,7 @@ class HueApp(object):
     self.author = author
     self.author = author
 
 
   def __str__(self):
   def __str__(self):
-    return "%s (version %s)" % (self.name, self.version)
+    return "%s v.%s" % (self.name, self.version)
 
 
   def __cmp__(self, other):
   def __cmp__(self, other):
     if not isinstance(other, HueApp):
     if not isinstance(other, HueApp):