瀏覽代碼

[sqoop] Not unicode strings in argument expansion

Python 2.6.4 > do not support unicode strings in argument expansion.
Abraham Elmahrek 11 年之前
父節點
當前提交
e11b285549

+ 4 - 2
apps/sqoop/src/sqoop/client/connection.py

@@ -16,6 +16,8 @@
 
 import logging
 
+from desktop.lib.python_util import force_dict_to_strings
+
 from exception import SqoopException
 from form import Form
 
@@ -139,7 +141,7 @@ class Connection(object):
     if not 'update_date' in connection_dict:
       connection_dict['update_date'] = connection_dict.setdefault('update-date', 0)
 
-    return Connection(**connection_dict)
+    return Connection(**force_dict_to_strings(connection_dict))
 
   def to_dict(self):
     d = {
@@ -195,7 +197,7 @@ class SqoopConnectionException(SqoopException):
 
   @classmethod
   def from_dict(cls, error_dict):
-    return SqoopConnectionException(**error_dict)
+    return SqoopConnectionException(**force_dict_to_strings(error_dict))
 
   def to_dict(self):
     return {

+ 3 - 1
apps/sqoop/src/sqoop/client/connector.py

@@ -16,6 +16,8 @@
 
 import logging
 
+from desktop.lib.python_util import force_dict_to_strings
+
 from form import Form
 
 
@@ -238,7 +240,7 @@ class Connector(object):
 
     connector_dict['resources'] = resources_dict.setdefault(unicode(connector_dict['id']), {})
 
-    return Connector(**connector_dict)
+    return Connector(**force_dict_to_strings(connector_dict))
 
   def to_dict(self):
     d = {

+ 4 - 2
apps/sqoop/src/sqoop/client/form.py

@@ -16,6 +16,8 @@
 
 import logging
 
+from desktop.lib.python_util import force_dict_to_strings
+
 
 class Form(object):
   """
@@ -86,7 +88,7 @@ class Form(object):
   @staticmethod
   def from_dict(form_dict):
     form_dict['inputs'] = [Input.from_dict(input_dict) for input_dict in form_dict.setdefault('inputs', [])]
-    return Form(**form_dict)
+    return Form(**force_dict_to_strings(form_dict))
 
   def to_dict(self):
     return {
@@ -127,7 +129,7 @@ class Input(object):
   def from_dict(input_dict):
     if 'values' in input_dict and isinstance(input_dict['values'], basestring):
       input_dict['values'] = input_dict['values'].split(',')
-    return Input(**input_dict)
+    return Input(**force_dict_to_strings(input_dict))
 
   def to_dict(self):
     d = {

+ 3 - 1
apps/sqoop/src/sqoop/client/framework.py

@@ -16,6 +16,8 @@
 
 import logging
 
+from desktop.lib.python_util import force_dict_to_strings
+
 from form import Form
 
 
@@ -181,7 +183,7 @@ class Framework(object):
     framework_dict.setdefault('con-forms', [])
     framework_dict['con_forms'] = [ Form.from_dict(con_form_dict) for con_form_dict in framework_dict['con-forms'] ]
 
-    return Framework(**framework_dict)
+    return Framework(**force_dict_to_strings(framework_dict))
 
   def to_dict(self):
     d = {

+ 4 - 2
apps/sqoop/src/sqoop/client/job.py

@@ -16,6 +16,8 @@
 
 import logging
 
+from desktop.lib.python_util import force_dict_to_strings
+
 from exception import SqoopException
 from form import Form
 
@@ -193,7 +195,7 @@ class Job(object):
     if not 'update_date' in job_dict:
       job_dict['update_date'] = job_dict.setdefault('update-date', 0)
 
-    return Job(**job_dict)
+    return Job(**force_dict_to_strings(job_dict))
 
   def to_dict(self):
     d = {
@@ -250,7 +252,7 @@ class SqoopJobException(SqoopException):
 
   @classmethod
   def from_dict(cls, error_dict):
-    return SqoopJobException(**error_dict)
+    return SqoopJobException(**force_dict_to_strings(error_dict))
 
   def to_dict(self):
     return {

+ 1 - 0
apps/sqoop/src/sqoop/client/resource.py

@@ -21,6 +21,7 @@ except ImportError:
 
 from django.utils.translation import ugettext as _
 
+from desktop.lib.python_util import force_dict_to_strings
 from desktop.lib.rest.resource import Resource
 
 

+ 4 - 2
apps/sqoop/src/sqoop/client/submission.py

@@ -16,6 +16,8 @@
 
 import logging
 
+from desktop.lib.python_util import force_dict_to_strings
+
 from exception import SqoopException
 from form import Form
 
@@ -51,7 +53,7 @@ class Submission(object):
     submission_dict['updated'] = submission_dict['last-update-date']
     submission_dict['external_id'] = submission_dict.get('external-id', None)
     submission_dict['external_link'] = submission_dict.get('external-link', None)
-    return Submission(**submission_dict)
+    return Submission(**force_dict_to_strings(submission_dict))
 
   def to_dict(self):
     d = {
@@ -107,7 +109,7 @@ class SqoopSubmissionException(SqoopException):
     submission_dict['updated'] = submission_dict['last-update-date']
     submission_dict['exception'] = submission_dict.get('exception', None)
     submission_dict['exception_trace'] = submission_dict.get('exception-trace', None)
-    submission = SqoopSubmissionException(**submission_dict)
+    submission = SqoopSubmissionException(**force_dict_to_strings(submission_dict))
     return submission
 
   def to_dict(self):

+ 23 - 0
desktop/core/src/desktop/lib/python_util.py

@@ -98,6 +98,27 @@ def find_unused_port():
   return port
 
 
+def force_list_to_strings(lst):
+  if not lst:
+    return lst
+
+  new_list = []
+  for item in lst:
+    if isinstance(item, basestring):
+      # Strings should not be unicode.
+      new_list.append(smart_str(item))
+    elif isinstance(item, dict):
+      # Recursively force dicts to strings.
+      new_list.append(force_dict_to_strings(item))
+    elif isinstance(item, list):
+      new_list.append(force_list_to_strings(item))
+    else:
+      # Normal objects, or other literals, should not be converted.
+      new_list.append(item)
+
+  return new_list
+
+
 def force_dict_to_strings(dictionary):
   if not dictionary:
     return dictionary
@@ -111,6 +132,8 @@ def force_dict_to_strings(dictionary):
     elif isinstance(dictionary[k], dict):
       # Recursively force dicts to strings.
       new_dict[new_key] = force_dict_to_strings(dictionary[k])
+    elif isinstance(dictionary[k], list):
+      new_dict[new_key] = force_list_to_strings(dictionary[k])
     else:
       # Normal objects, or other literals, should not be converted.
       new_dict[new_key] = dictionary[k]

+ 14 - 1
desktop/core/src/desktop/lib/python_util_test.py

@@ -20,7 +20,7 @@ import datetime
 
 from nose.tools import assert_true, assert_equal, assert_not_equal
 
-from desktop.lib.python_util import CaseInsensitiveDict, force_dict_to_strings
+from desktop.lib.python_util import CaseInsensitiveDict, force_dict_to_strings, force_list_to_strings
 
 
 class TestPythonUtil(object):
@@ -40,7 +40,20 @@ class TestPythonUtil(object):
     transformed_dict = force_dict_to_strings(unicode_dict)
     assert_equal(string_dict, transformed_dict)
 
+    # Embedded
     unicode_dict = {u'test': {u'test': u'test'}}
     string_dict = {'test': {'test': 'test'}}
     transformed_dict = force_dict_to_strings(unicode_dict)
     assert_equal(string_dict, transformed_dict)
+
+    # Embedded list
+    unicode_dict = {u'test': [{u'test': u'test'}]}
+    string_dict = {'test': [{'test': 'test'}]}
+    transformed_dict = force_dict_to_strings(unicode_dict)
+    assert_equal(string_dict, transformed_dict)
+
+  def test_force_list_to_strings(self):
+    unicode_list = [u'test', {u'test': u'test'}]
+    string_list = ['test', {'test': 'test'}]
+    transformed_list = force_list_to_strings(unicode_list)
+    assert_equal(string_list, transformed_list)