Преглед изворни кода

HUE-9045 [core] Remove chardet import from requests-2.18.4

Ying Chen пре 6 година
родитељ
комит
b3ccb6f35a

+ 3 - 12
desktop/core/ext-py/requests-2.18.4/requests/__init__.py

@@ -41,12 +41,11 @@ is at <http://python-requests.org>.
 """
 
 import urllib3
-import chardet
 import warnings
 from .exceptions import RequestsDependencyWarning
 
 
-def check_compatibility(urllib3_version, chardet_version):
+def check_compatibility(urllib3_version):
     urllib3_version = urllib3_version.split('.')
     assert urllib3_version != ['dev']  # Verify urllib3 isn't installed from git.
 
@@ -62,21 +61,13 @@ def check_compatibility(urllib3_version, chardet_version):
     assert minor >= 21
     assert minor <= 22
 
-    # Check chardet for compatibility.
-    major, minor, patch = chardet_version.split('.')[:3]
-    major, minor, patch = int(major), int(minor), int(patch)
-    # chardet >= 3.0.2, < 3.1.0
-    assert major == 3
-    assert minor < 1
-    assert patch >= 2
-
 
 # Check imported dependencies for compatibility.
 try:
-    check_compatibility(urllib3.__version__, chardet.__version__)
+    check_compatibility(urllib3.__version__)
 except (AssertionError, ValueError):
     warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported "
-                  "version!".format(urllib3.__version__, chardet.__version__),
+                  "version!".format(urllib3.__version__),
                   RequestsDependencyWarning)
 
 # Attempt to enable urllib3's SNI support, if possible

+ 0 - 2
desktop/core/ext-py/requests-2.18.4/requests/compat.py

@@ -8,8 +8,6 @@ This module handles import compatibility issues between Python 2 and
 Python 3.
 """
 
-import chardet
-
 import sys
 
 # -------

+ 0 - 3
desktop/core/ext-py/requests-2.18.4/requests/help.py

@@ -8,7 +8,6 @@ import ssl
 
 import idna
 import urllib3
-import chardet
 
 from . import __version__ as requests_version
 
@@ -71,7 +70,6 @@ def info():
 
     implementation_info = _implementation()
     urllib3_info = {'version': urllib3.__version__}
-    chardet_info = {'version': chardet.__version__}
 
     pyopenssl_info = {
         'version': None,
@@ -102,7 +100,6 @@ def info():
         'using_pyopenssl': pyopenssl is not None,
         'pyOpenSSL': pyopenssl_info,
         'urllib3': urllib3_info,
-        'chardet': chardet_info,
         'cryptography': cryptography_info,
         'idna': idna_info,
         'requests': {

+ 16 - 3
desktop/core/ext-py/requests-2.18.4/requests/models.py

@@ -38,7 +38,7 @@ from .utils import (
     iter_slices, guess_json_utf, super_len, check_header_validity)
 from .compat import (
     cookielib, urlunparse, urlsplit, urlencode, str, bytes,
-    is_py2, chardet, builtin_str, basestring)
+    is_py2, builtin_str, basestring)
 from .compat import json as complexjson
 from .status_codes import codes
 
@@ -56,6 +56,8 @@ DEFAULT_REDIRECT_LIMIT = 30
 CONTENT_CHUNK_SIZE = 10 * 1024
 ITER_CHUNK_SIZE = 512
 
+from desktop.lib.python_util import check_encoding
+
 
 class RequestEncodingMixin(object):
     @property
@@ -718,8 +720,19 @@ class Response(object):
 
     @property
     def apparent_encoding(self):
-        """The apparent encoding, provided by the chardet library."""
-        return chardet.detect(self.content)['encoding']
+        try:
+            import chardet
+            chardet_version = chardet.__version__
+            major, minor, patch = chardet_version.split('.')[:3]
+            major, minor, patch = int(major), int(minor), int(patch)
+            # chardet >= 3.0.2, < 3.1.0
+            assert major == 3
+            assert minor < 1
+            assert patch >= 2
+        except (ImportError, AssertionError) as e:
+            return check_encoding(self.content)
+        else:
+            return chardet.detect(self.content)['encoding']
 
     def iter_content(self, chunk_size=1, decode_unicode=False):
         """Iterates over the response data.  When stream=True is set on the

+ 1 - 1
desktop/core/ext-py/requests-2.18.4/requests/packages.py

@@ -3,7 +3,7 @@ import sys
 # This code exists for backwards compatibility reasons.
 # I don't like it either. Just look the other way. :)
 
-for package in ('urllib3', 'idna', 'chardet'):
+for package in ('urllib3', 'idna'):
     locals()[package] = __import__(package)
     # This traversal is apparently necessary such that the identities are
     # preserved (requests.packages.urllib3.* is urllib3.*)

+ 73 - 0
desktop/core/src/desktop/lib/python_util.py

@@ -21,10 +21,20 @@ from past.builtins import basestring
 from builtins import object
 import select
 import socket
+import sys
 from django.utils.translation import ugettext as _
 from desktop import conf
 from desktop.lib.i18n import smart_str
 
+from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE
+
+BOMS = (
+    (BOM_UTF8, "UTF-8"),
+    (BOM_UTF32_BE, "UTF-32-BE"),
+    (BOM_UTF32_LE, "UTF-32-LE"),
+    (BOM_UTF16_BE, "UTF-16-BE"),
+    (BOM_UTF16_LE, "UTF-16-LE"),
+)
 
 __all__ = ['CaseInsensitiveDict', 'create_synchronous_io_multiplexer']
 
@@ -144,3 +154,66 @@ def force_dict_to_strings(dictionary):
       new_dict[new_key] = dictionary[k]
 
   return new_dict
+
+
+def isASCII(data):
+  try:
+    data.decode('ASCII')
+  except UnicodeDecodeError:
+    return False
+  else:
+    return True
+
+
+def isUTF8(data):
+  try:
+    data.decode('UTF-8')
+  except UnicodeDecodeError:
+    return False
+  else:
+    return True
+
+
+def isGB2312(data):
+  try:
+    data.decode('gb2312')
+  except UnicodeDecodeError:
+    return False
+  else:
+    return True
+
+
+def isUTF8Strict(data):
+  try:
+    decoded = data.decode('UTF-8')
+  except UnicodeDecodeError:
+    return False
+  else:
+    for ch in decoded:
+      if 0xD800 <= ord(ch) <= 0xDFFF:
+        return False
+    return True
+
+
+def check_bom(data):
+  return [encoding for bom, encoding in BOMS if data.startswith(bom)]
+
+
+def check_encoding(data):
+  """
+  this is a simplified alternative to GPL chardet
+  """
+  if isASCII(data):
+    return 'ASCII'
+  elif sys.version_info[0] == 2 and isUTF8(data):
+    return 'utf-8'
+  elif sys.version_info[0] > 2 and isUTF8Strict(data):
+    return 'utf-8'
+  else:
+    encoding = check_bom(data)
+    if encoding:
+      return encoding[0]
+    elif isGB2312(data):
+      return 'gb2312'
+    else:
+      return 'cp1252'

+ 2 - 2
desktop/libs/indexer/src/indexer/api3.py

@@ -21,7 +21,6 @@ standard_library.install_aliases()
 from builtins import oct
 from builtins import zip
 from past.builtins import basestring
-import chardet
 import json
 import logging
 import urllib.request, urllib.error
@@ -37,6 +36,7 @@ from desktop.lib import django_mako
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_unicode
+from desktop.lib.python_util import check_encoding
 from desktop.models import Document2
 from kafka.kafka_api import get_topics
 from metadata.manager_client import ManagerApi
@@ -177,7 +177,7 @@ def guess_field_types(request):
     indexer = MorphlineIndexer(request.user, request.fs)
     path = urllib_unquote(file_format["path"])
     stream = request.fs.open(path)
-    encoding = chardet.detect(stream.read(10000)).get('encoding')
+    encoding = check_encoding(stream.read(10000))
     stream.seek(0)
     _convert_format(file_format["format"], inverse=True)