瀏覽代碼

HUE-8737 [core] Fix beeswax, hdfs, oozie and desktop libs py3 compatible code

Ying Chen 6 年之前
父節點
當前提交
292db6d788

+ 7 - 2
apps/beeswax/src/beeswax/hive_site.py

@@ -25,6 +25,7 @@ import logging
 import os.path
 import os.path
 import re
 import re
 import socket
 import socket
+import sys
 
 
 from desktop.lib import security_util
 from desktop.lib import security_util
 from hadoop import confparse
 from hadoop import confparse
@@ -32,6 +33,10 @@ from hadoop.ssl_client_site import get_trustore_location, get_trustore_password
 
 
 import beeswax.conf
 import beeswax.conf
 
 
+if sys.version_info[0] > 2:
+  open_file = open
+else:
+  open_file = file
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
@@ -196,7 +201,7 @@ def _parse_hive_site():
 
 
   _HIVE_SITE_PATH = os.path.join(beeswax.conf.HIVE_CONF_DIR.get(), 'hive-site.xml')
   _HIVE_SITE_PATH = os.path.join(beeswax.conf.HIVE_CONF_DIR.get(), 'hive-site.xml')
   try:
   try:
-    data = file(_HIVE_SITE_PATH, 'r').read()
+    data = open_file(_HIVE_SITE_PATH, 'r').read()
   except IOError as err:
   except IOError as err:
     if err.errno != errno.ENOENT:
     if err.errno != errno.ENOENT:
       LOG.error('Cannot read from "%s": %s' % (_HIVE_SITE_PATH, err))
       LOG.error('Cannot read from "%s": %s' % (_HIVE_SITE_PATH, err))
@@ -211,4 +216,4 @@ def get_hive_site_content():
   if not os.path.exists(hive_site_path):
   if not os.path.exists(hive_site_path):
     return ''
     return ''
   else:
   else:
-    return file(hive_site_path, 'r').read()
+    return open_file(hive_site_path, 'r').read()

+ 7 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -15,17 +15,16 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 
 
-from builtins import str
 from builtins import object
 from builtins import object
 import logging
 import logging
 import re
 import re
+import sys
 import threading
 import threading
 import time
 import time
 import json
 import json
 
 
 from django.core.cache import caches
 from django.core.cache import caches
 from django.urls import reverse
 from django.urls import reverse
-from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 from kazoo.client import KazooClient
 from kazoo.client import KazooClient
 
 
@@ -50,6 +49,12 @@ from beeswax.hive_site import hiveserver2_use_ssl
 from beeswax.models import QueryHistory, QUERY_TYPES
 from beeswax.models import QueryHistory, QUERY_TYPES
 
 
 
 
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
+
+
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 DBMS_CACHE = {}
 DBMS_CACHE = {}

+ 3 - 1
apps/oozie/src/oozie/models.py

@@ -39,7 +39,7 @@ from django.core.validators import RegexValidator
 from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes.models import ContentType
 from django.forms.models import inlineformset_factory
 from django.forms.models import inlineformset_factory
-from django.utils.encoding import force_unicode, smart_str
+from django.utils.encoding import smart_str
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 import django.utils.timezone as dtz
 import django.utils.timezone as dtz
 
 
@@ -61,8 +61,10 @@ from oozie.timezones import TIMEZONES
 
 
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
   from io import StringIO as string_io
   from io import StringIO as string_io
+  from django.utils.encoding import force_text as force_unicode
 else:
 else:
   from cStringIO import StringIO as string_io
   from cStringIO import StringIO as string_io
+  from django.utils.encoding import force_unicode
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)

+ 6 - 1
apps/oozie/src/oozie/models2.py

@@ -24,6 +24,7 @@ import json
 import logging
 import logging
 import os
 import os
 import re
 import re
+import sys
 import time
 import time
 import uuid
 import uuid
 
 
@@ -34,7 +35,6 @@ from xml.sax.saxutils import escape
 
 
 from django.urls import reverse
 from django.urls import reverse
 from django.db.models import Q
 from django.db.models import Q
-from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
 from desktop.conf import USE_DEFAULT_CONFIGURATION
 from desktop.conf import USE_DEFAULT_CONFIGURATION
@@ -57,6 +57,11 @@ from oozie.conf import REMOTE_SAMPLE_DIR
 from oozie.utils import utc_datetime_format, UTC_TIME_FORMAT, convert_to_server_timezone
 from oozie.utils import utc_datetime_format, UTC_TIME_FORMAT, convert_to_server_timezone
 from oozie.importlib.workflows import generate_v2_graph_nodes, MalformedWfDefException, InvalidTagWithNamespaceException
 from oozie.importlib.workflows import generate_v2_graph_nodes, MalformedWfDefException, InvalidTagWithNamespaceException
 
 
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
+
 WORKFLOW_DEPTH_LIMIT = 24
 WORKFLOW_DEPTH_LIMIT = 24
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 

+ 3 - 1
desktop/core/src/desktop/lib/django_forms.py

@@ -33,7 +33,7 @@ from django.forms.fields import MultiValueField, CharField, ChoiceField, Boolean
 from django.forms.widgets import MultiWidget, Select, TextInput, Textarea, HiddenInput, Input
 from django.forms.widgets import MultiWidget, Select, TextInput, Textarea, HiddenInput, Input
 from django.utils import formats
 from django.utils import formats
 from django.utils.safestring import mark_safe
 from django.utils.safestring import mark_safe
-from django.utils.encoding import python_2_unicode_compatible, force_unicode
+from django.utils.encoding import python_2_unicode_compatible
 
 
 import desktop.lib.i18n
 import desktop.lib.i18n
 from desktop.lib.i18n import smart_str
 from desktop.lib.i18n import smart_str
@@ -41,8 +41,10 @@ from desktop.lib.i18n import smart_str
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
   import urllib.request, urllib.error
   import urllib.request, urllib.error
   from urllib.parse import quote_plus as urllib_quote_plus
   from urllib.parse import quote_plus as urllib_quote_plus
+  from django.utils.encoding import force_text as force_unicode
 else:
 else:
   from urllib import quote_plus as urllib_quote_plus
   from urllib import quote_plus as urllib_quote_plus
+  from django.utils.encoding import force_unicode
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 

+ 4 - 1
desktop/core/src/desktop/lib/exceptions_renderable.py

@@ -23,7 +23,10 @@ import logging
 import sys
 import sys
 import traceback
 import traceback
 
 
-from django.utils.encoding import force_unicode
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
 
 
 import desktop.lib.django_util
 import desktop.lib.django_util
 
 

+ 1 - 1
desktop/core/src/desktop/lib/i18n.py

@@ -59,7 +59,7 @@ def smart_unicode(s, strings_only=False, errors='strict', encoding=None):
   Wrapper around Django's version, while supplying our configured encoding.
   Wrapper around Django's version, while supplying our configured encoding.
   Decode char array to unicode.
   Decode char array to unicode.
   """
   """
-  return django.utils.encoding.smart_unicode(
+  return django.utils.encoding.smart_text(
         s, encoding if encoding is not None else get_site_encoding(), strings_only, errors)
         s, encoding if encoding is not None else get_site_encoding(), strings_only, errors)
 
 
 def force_unicode(s, strings_only=False, errors='strict'):
 def force_unicode(s, strings_only=False, errors='strict'):

+ 2 - 1
desktop/libs/aws/src/aws/s3/__init__.py

@@ -16,6 +16,7 @@
 from __future__ import absolute_import
 from __future__ import absolute_import
 
 
 from builtins import map
 from builtins import map
+from future.utils import raise_
 import calendar
 import calendar
 import errno
 import errno
 import logging
 import logging
@@ -55,7 +56,7 @@ def translate_s3_error(fn):
       _, exc, tb = sys.exc_info()
       _, exc, tb = sys.exc_info()
       logging.error('S3 error: %s' % exc)
       logging.error('S3 error: %s' % exc)
       lookup = lookup_s3error(exc)
       lookup = lookup_s3error(exc)
-      raise lookup.__class__, lookup, tb
+      raise_(lookup.__class__, lookup, tb)
   return wrapped
   return wrapped
 
 
 
 

+ 8 - 2
desktop/libs/aws/src/aws/s3/s3fs.py

@@ -23,7 +23,7 @@ import logging
 import os
 import os
 import posixpath
 import posixpath
 import re
 import re
-from urlparse import urlparse
+import sys
 import time
 import time
 
 
 from boto.exception import BotoClientError, S3ResponseError
 from boto.exception import BotoClientError, S3ResponseError
@@ -38,6 +38,12 @@ from aws.conf import get_default_region, get_locations, PERMISSION_ACTION_S3
 from aws.s3 import normpath, s3file, translate_s3_error, S3A_ROOT
 from aws.s3 import normpath, s3file, translate_s3_error, S3A_ROOT
 from aws.s3.s3stat import S3Stat
 from aws.s3.s3stat import S3Stat
 
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote as urllib_quote, urlparse as lib_urlparse
+else:
+  from urllib import quote as urllib_quote
+  from urlparse import urlparse as lib_urlparse
 
 
 DEFAULT_READ_SIZE = 1024 * 1024  # 1MB
 DEFAULT_READ_SIZE = 1024 * 1024  # 1MB
 BUCKET_NAME_PATTERN = re.compile("^((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_\-]*[A-Za-z0-9]))$")
 BUCKET_NAME_PATTERN = re.compile("^((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_\-]*[A-Za-z0-9]))$")
@@ -205,7 +211,7 @@ class S3FileSystem(object):
 
 
   @staticmethod
   @staticmethod
   def isroot(path):
   def isroot(path):
-    parsed = urlparse(path)
+    parsed = lib_urlparse(path)
     return (parsed.path == '/' or parsed.path == '') and parsed.netloc == ''
     return (parsed.path == '/' or parsed.path == '') and parsed.netloc == ''
 
 
   @staticmethod
   @staticmethod

+ 10 - 4
desktop/libs/azure/src/azure/abfs/abfs.py

@@ -23,13 +23,12 @@ standard_library.install_aliases()
 from builtins import object
 from builtins import object
 import logging
 import logging
 import os
 import os
+import sys
 import threading
 import threading
 import re
 import re
 
 
 from math import ceil
 from math import ceil
 from posixpath import join
 from posixpath import join
-from urllib.parse import urlparse
-from urllib import quote
 
 
 from hadoop.hdfs_site import get_umask_mode
 from hadoop.hdfs_site import get_umask_mode
 from hadoop.fs.exceptions import WebHdfsException
 from hadoop.fs.exceptions import WebHdfsException
@@ -41,6 +40,13 @@ from azure.abfs.abfsfile import ABFSFile
 from azure.abfs.abfsstats import ABFSStat
 from azure.abfs.abfsstats import ABFSStat
 from azure.conf import PERMISSION_ACTION_ABFS
 from azure.conf import PERMISSION_ACTION_ABFS
 
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote as urllib_quote
+  from urllib.parse import urlparse as lib_urlparse
+else:
+  from urlparse import urlparse as lib_urlparse
+  from urllib import quote as urllib_quote
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
@@ -75,7 +81,7 @@ class ABFS(object):
     self._logical_name = logical_name
     self._logical_name = logical_name
     self._supergroup = hdfs_supergroup
     self._supergroup = hdfs_supergroup
     self._auth_provider = auth_provider
     self._auth_provider = auth_provider
-    split = urlparse(fs_defaultfs)
+    split = lib_urlparse(fs_defaultfs)
     self._scheme = split.scheme
     self._scheme = split.scheme
     self._netloc = split.netloc
     self._netloc = split.netloc
     self._is_remote = True
     self._is_remote = True
@@ -509,7 +515,7 @@ class ABFS(object):
     Renames a file
     Renames a file
     """ 
     """ 
     LOG.debug("%s\n%s" % (old, new))
     LOG.debug("%s\n%s" % (old, new))
-    headers = {'x-ms-rename-source' : '/' + quote(Init_ABFS.strip_scheme(old)) }
+    headers = {'x-ms-rename-source' : '/' + urllib_quote(Init_ABFS.strip_scheme(old)) }
     try:
     try:
       self._create_path(new, headers=headers, overwrite=True)
       self._create_path(new, headers=headers, overwrite=True)
     except WebHdfsException as e:
     except WebHdfsException as e:

+ 5 - 1
desktop/libs/dashboard/src/dashboard/api.py

@@ -19,9 +19,9 @@ from builtins import filter
 import hashlib
 import hashlib
 import json
 import json
 import logging
 import logging
+import sys
 import uuid
 import uuid
 
 
-from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
 from desktop.conf import ENABLE_DOWNLOAD
 from desktop.conf import ENABLE_DOWNLOAD
@@ -46,6 +46,10 @@ from dashboard.facet_builder import _guess_gap, _zoom_range_facet, _new_range_fa
 from dashboard.models import Collection2, augment_solr_response, pairwise2, augment_solr_exception,\
 from dashboard.models import Collection2, augment_solr_response, pairwise2, augment_solr_exception,\
   NESTED_FACET_FORM, COMPARE_FACET, QUERY_FACET, extract_solr_exception_message
   NESTED_FACET_FORM, COMPARE_FACET, QUERY_FACET, extract_solr_exception_message
 
 
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 

+ 3 - 1
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -36,7 +36,7 @@ import random
 import subprocess
 import subprocess
 import sys
 import sys
 
 
-from django.utils.encoding import smart_str, force_unicode
+from django.utils.encoding import smart_str
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
 from desktop.lib import i18n
 from desktop.lib import i18n
@@ -46,8 +46,10 @@ from hadoop.fs import normpath, SEEK_SET, SEEK_CUR, SEEK_END
 from hadoop.fs.exceptions import PermissionDeniedException
 from hadoop.fs.exceptions import PermissionDeniedException
 
 
 if sys.version_info[0] > 2:
 if sys.version_info[0] > 2:
+  from django.utils.encoding import force_text as force_unicode
   from urllib.parse import urlsplit as lib_urlsplit
   from urllib.parse import urlsplit as lib_urlsplit
 else:
 else:
+  from django.utils.encoding import force_unicode
   from urlparse import urlsplit as lib_urlsplit
   from urlparse import urlsplit as lib_urlsplit
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)

+ 1 - 1
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -78,7 +78,7 @@ class WebHdfs(Hdfs):
       security_enabled=False,
       security_enabled=False,
       ssl_cert_ca_verify=True,
       ssl_cert_ca_verify=True,
       temp_dir="/tmp",
       temp_dir="/tmp",
-      umask=01022,
+      umask=0o1022,
       hdfs_supergroup=None):
       hdfs_supergroup=None):
     self._url = url
     self._url = url
     self._superuser = hdfs_superuser
     self._superuser = hdfs_superuser