Эх сурвалжийг харах

[django] upgraded django to 4.1

Several other libraries have also been upgraded along with it.
Four code changes had to be made.

1. Django 4 wants the middlewares to be explicitly sync and async.
Rather than rewriting all of the middleware classes, I took the
shortcut of creating a new mixin class that supports only sync calls.

2. CSRF_TRUSTED_ORIGINS in django 4+ need the scheme part as well. We
now check and add `http` and `https` to the origins.

3. django-babel also needs a change every time we do a django upgrade.
This is mostly just changes around django version support.
Amit Srivastava 9 сар өмнө
parent
commit
9ea4732b7c

+ 8 - 8
desktop/core/base_requirements.txt

@@ -5,28 +5,28 @@ avro-python3==1.8.2
 Babel==2.9.1
 Babel==2.9.1
 celery[redis]==5.4.0
 celery[redis]==5.4.0
 cffi==1.15.0
 cffi==1.15.0
-channels==3.0.3
-channels-redis==3.2.0
+channels==4.0.0
+channels-redis==4.0.0
 configobj==5.0.9
 configobj==5.0.9
 cx-Oracle==8.3.0
 cx-Oracle==8.3.0
 django-auth-ldap==4.3.0
 django-auth-ldap==4.3.0
-Django==3.2.25
+Django==4.1.13
 daphne==3.0.2
 daphne==3.0.2
 django-redis==5.4.0
 django-redis==5.4.0
 django-celery-beat==2.6.0
 django-celery-beat==2.6.0
 django-celery-results==2.5.1
 django-celery-results==2.5.1
-django-cors-headers==3.7.0
+django-cors-headers==3.13.0
 django-crequest==2018.5.11
 django-crequest==2018.5.11
 django-debug-panel==0.8.3
 django-debug-panel==0.8.3
-django-debug-toolbar==1.11.1
-django-extensions==3.1.3
+django-debug-toolbar==3.6.0
+django-extensions==3.2.1
 django-ipware==3.0.2
 django-ipware==3.0.2
 django_opentracing==1.1.0
 django_opentracing==1.1.0
 django_prometheus==1.0.15
 django_prometheus==1.0.15
 django-webpack-loader==1.0.0
 django-webpack-loader==1.0.0
 djangomako==1.3.2
 djangomako==1.3.2
-djangorestframework-simplejwt==5.2.0
-djangorestframework==3.12.2
+djangorestframework-simplejwt==5.2.1
+djangorestframework==3.14.0
 future==0.18.3
 future==0.18.3
 gcs-oauth2-boto-plugin==3.0
 gcs-oauth2-boto-plugin==3.0
 greenlet==3.1.1
 greenlet==3.1.1

+ 3 - 0
desktop/core/ext-py3/django-babel/.travis.yml

@@ -17,6 +17,7 @@ env:
   - TOX_ENV=py35-django30
   - TOX_ENV=py35-django30
   - TOX_ENV=py35-django31
   - TOX_ENV=py35-django31
   - TOX_ENV=py35-django32
   - TOX_ENV=py35-django32
+  - TOX_ENV=py38-django41
 matrix:
 matrix:
   include:
   include:
     - env: TOX_ENV=py36-django18
     - env: TOX_ENV=py36-django18
@@ -35,6 +36,8 @@ matrix:
       python: "3.6"
       python: "3.6"
     - env: TOX_ENV=py36-django32
     - env: TOX_ENV=py36-django32
       python: "3.6"
       python: "3.6"
+    - env: TOX_ENV=py38-django41
+      python: "3.8"
     - env: TOX_ENV=py36-djangomaster
     - env: TOX_ENV=py36-djangomaster
       python: "3.6"
       python: "3.6"
 install: pip install tox
 install: pip install tox

+ 1 - 1
desktop/core/ext-py3/django-babel/setup.py

@@ -24,7 +24,7 @@ setup(
     url='https://github.com/python-babel/django-babel/',
     url='https://github.com/python-babel/django-babel/',
     packages=find_packages(exclude=('tests',)),
     packages=find_packages(exclude=('tests',)),
     install_requires=[
     install_requires=[
-        'django>=1.8,<3.3',
+        'django>=1.8,<4.2',
         'babel>=1.3',
         'babel>=1.3',
     ],
     ],
     classifiers=[
     classifiers=[

+ 2 - 1
desktop/core/ext-py3/django-babel/tox.ini

@@ -1,6 +1,6 @@
 [tox]
 [tox]
 envlist = py{27,34,35,36}-django{18,111},
 envlist = py{27,34,35,36}-django{18,111},
-          py{35,36,37}-django{20,21,22,30,31,32},
+          py{35,36,37,38}-django{20,21,22,30,31,32,41},
           py{36,37}-django{master},
           py{36,37}-django{master},
           lint, docs
           lint, docs
 
 
@@ -19,6 +19,7 @@ deps =
     django30: Django>=3.0,<3.1
     django30: Django>=3.0,<3.1
     django31: Django>=3.1,<3.2
     django31: Django>=3.1,<3.2
     django32: Django>=3.2,<3.3
     django32: Django>=3.2,<3.3
+    django41: Django>=4.1,<4.2
     djangomaster: https://github.com/django/django/archive/master.tar.gz#egg=Django
     djangomaster: https://github.com/django/django/archive/master.tar.gz#egg=Django
 commands = py.test {posargs}
 commands = py.test {posargs}
 
 

+ 39 - 14
desktop/core/src/desktop/middleware.py

@@ -161,7 +161,32 @@ if ENABLE_PROMETHEUS.get():
 HUE_LB_HOSTS = [urlparse(hue_lb).netloc for hue_lb in HUE_LOAD_BALANCER.get()] if HUE_LOAD_BALANCER.get() else []
 HUE_LB_HOSTS = [urlparse(hue_lb).netloc for hue_lb in HUE_LOAD_BALANCER.get()] if HUE_LOAD_BALANCER.get() else []
 
 
 
 
-class AjaxMiddleware(MiddlewareMixin):
+class Django4MiddlewareAdapterMixin(MiddlewareMixin):
+  """
+  A mixin to adapt Django 3-style middleware to be compatible with Django 4.
+  This is intended as a transitional solution and should not be used for
+  new middleware implementations.
+  """
+  def __call__(self, request):
+    """Sync version of the middleware"""
+    if hasattr(self, 'process_request'):
+      self.process_request(request)
+
+    response = self.get_response(request)
+
+    if hasattr(self, 'process_response'):
+      response = self.process_response(request, response)
+
+    return response
+
+  async def __aenter__(self):
+    pass  # Provide async behavior if needed
+
+  async def __aexit__(self, exc_type, exc_value, traceback):
+    pass  # Handle async cleanup
+
+
+class AjaxMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Middleware that augments request to set request.ajax
   Middleware that augments request to set request.ajax
   for either is_ajax() (looks at HTTP headers) or ?format=json
   for either is_ajax() (looks at HTTP headers) or ?format=json
@@ -172,7 +197,7 @@ class AjaxMiddleware(MiddlewareMixin):
     return None
     return None
 
 
 
 
-class ExceptionMiddleware(MiddlewareMixin):
+class ExceptionMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   If exceptions know how to render themselves, use that.
   If exceptions know how to render themselves, use that.
   """
   """
@@ -202,7 +227,7 @@ class ExceptionMiddleware(MiddlewareMixin):
     return None
     return None
 
 
 
 
-class ClusterMiddleware(MiddlewareMixin):
+class ClusterMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Manages setting request.fs and request.jt
   Manages setting request.fs and request.jt
   """
   """
@@ -236,7 +261,7 @@ class ClusterMiddleware(MiddlewareMixin):
     request.jt = None
     request.jt = None
 
 
 
 
-class NotificationMiddleware(MiddlewareMixin):
+class NotificationMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Manages setting request.info and request.error
   Manages setting request.info and request.error
   """
   """
@@ -365,7 +390,7 @@ class AppSpecificMiddleware(object):
     return result
     return result
 
 
 
 
-class LoginAndPermissionMiddleware(MiddlewareMixin):
+class LoginAndPermissionMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Middleware that forces all views (except those that opt out) through authentication.
   Middleware that forces all views (except those that opt out) through authentication.
   """
   """
@@ -503,7 +528,7 @@ class JsonMessage(object):
     return json.dumps(self.kwargs)
     return json.dumps(self.kwargs)
 
 
 
 
-class AuditLoggingMiddleware(MiddlewareMixin):
+class AuditLoggingMiddleware(Django4MiddlewareAdapterMixin):
 
 
   def __init__(self, get_response):
   def __init__(self, get_response):
     self.get_response = get_response
     self.get_response = get_response
@@ -563,7 +588,7 @@ class AuditLoggingMiddleware(MiddlewareMixin):
     return allowed
     return allowed
 
 
 
 
-class ProxyMiddleware(MiddlewareMixin):
+class ProxyMiddleware(Django4MiddlewareAdapterMixin):
 
 
   def __init__(self, get_response):
   def __init__(self, get_response):
     self.get_response = get_response
     self.get_response = get_response
@@ -622,7 +647,7 @@ class ProxyMiddleware(MiddlewareMixin):
     return username
     return username
 
 
 
 
-class SpnegoMiddleware(MiddlewareMixin):
+class SpnegoMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Based on the WSGI SPNEGO middlware class posted here:
   Based on the WSGI SPNEGO middlware class posted here:
   http://code.activestate.com/recipes/576992/
   http://code.activestate.com/recipes/576992/
@@ -790,7 +815,7 @@ class SpnegoMiddleware(MiddlewareMixin):
     return username
     return username
 
 
 
 
-class HueRemoteUserMiddleware(RemoteUserMiddleware):
+class HueRemoteUserMiddleware(Django4MiddlewareAdapterMixin, RemoteUserMiddleware):
   """
   """
   Middleware to delegate authentication to a proxy server. The proxy server
   Middleware to delegate authentication to a proxy server. The proxy server
   will set an HTTP header (defaults to Remote-User) with the name of the
   will set an HTTP header (defaults to Remote-User) with the name of the
@@ -816,7 +841,7 @@ class EnsureSafeMethodMiddleware(MiddlewareMixin):
       return HttpResponseNotAllowed(HTTP_ALLOWED_METHODS.get())
       return HttpResponseNotAllowed(HTTP_ALLOWED_METHODS.get())
 
 
 
 
-class EnsureSafeRedirectURLMiddleware(MiddlewareMixin):
+class EnsureSafeRedirectURLMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Middleware to white list configured redirect URLs.
   Middleware to white list configured redirect URLs.
   """
   """
@@ -844,7 +869,7 @@ class EnsureSafeRedirectURLMiddleware(MiddlewareMixin):
       return response
       return response
 
 
 
 
-class MetricsMiddleware(MiddlewareMixin):
+class MetricsMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Middleware to track the number of active requests.
   Middleware to track the number of active requests.
   """
   """
@@ -869,7 +894,7 @@ class MetricsMiddleware(MiddlewareMixin):
     return response
     return response
 
 
 
 
-class ContentSecurityPolicyMiddleware(MiddlewareMixin):
+class ContentSecurityPolicyMiddleware(Django4MiddlewareAdapterMixin):
   def __init__(self, get_response):
   def __init__(self, get_response):
     self.get_response = get_response
     self.get_response = get_response
     self.secure_content_security_policy = SECURE_CONTENT_SECURITY_POLICY.get()
     self.secure_content_security_policy = SECURE_CONTENT_SECURITY_POLICY.get()
@@ -926,7 +951,7 @@ class ContentSecurityPolicyMiddleware(MiddlewareMixin):
     return response
     return response
 
 
 
 
-class MimeTypeJSFileFixStreamingMiddleware(MiddlewareMixin):
+class MimeTypeJSFileFixStreamingMiddleware(Django4MiddlewareAdapterMixin):
   """
   """
   Middleware to detect and fix ".js" mimetype. SLES 11SP4 as example OS which detect js file
   Middleware to detect and fix ".js" mimetype. SLES 11SP4 as example OS which detect js file
   as "text/x-js" and if strict X-Content-Type-Options=nosniff is set then browser fails to
   as "text/x-js" and if strict X-Content-Type-Options=nosniff is set then browser fails to
@@ -985,7 +1010,7 @@ class MultipleProxyMiddleware:
     return self.get_response(request)
     return self.get_response(request)
 
 
 
 
-class CacheControlMiddleware(MiddlewareMixin):
+class CacheControlMiddleware(Django4MiddlewareAdapterMixin):
   def __init__(self, get_response):
   def __init__(self, get_response):
     self.get_response = get_response
     self.get_response = get_response
     self.custom_cache_control = CUSTOM_CACHE_CONTROL.get()
     self.custom_cache_control = CUSTOM_CACHE_CONTROL.get()

+ 7 - 1
desktop/core/src/desktop/settings.py

@@ -498,8 +498,14 @@ if desktop.conf.KNOX.KNOX_PROXYHOSTS.get():  # The hosts provided here don't hav
   else:
   else:
     TRUSTED_ORIGINS += desktop.conf.KNOX.KNOX_PROXYHOSTS.get()
     TRUSTED_ORIGINS += desktop.conf.KNOX.KNOX_PROXYHOSTS.get()
 
 
+CSRF_TRUSTED_ORIGINS = []
 if TRUSTED_ORIGINS:
 if TRUSTED_ORIGINS:
-  CSRF_TRUSTED_ORIGINS = TRUSTED_ORIGINS
+  for origin in TRUSTED_ORIGINS:
+    if 'http://' in origin or 'https://' in origin:
+      CSRF_TRUSTED_ORIGINS.append(origin)
+    else:
+      CSRF_TRUSTED_ORIGINS.append('http://%s' % origin)
+      CSRF_TRUSTED_ORIGINS.append('https://%s' % origin)
 
 
 SECURE_HSTS_SECONDS = desktop.conf.SECURE_HSTS_SECONDS.get()
 SECURE_HSTS_SECONDS = desktop.conf.SECURE_HSTS_SECONDS.get()
 SECURE_HSTS_INCLUDE_SUBDOMAINS = desktop.conf.SECURE_HSTS_INCLUDE_SUBDOMAINS.get()
 SECURE_HSTS_INCLUDE_SUBDOMAINS = desktop.conf.SECURE_HSTS_INCLUDE_SUBDOMAINS.get()

+ 3 - 0
desktop/core/src/desktop/tests.py

@@ -1423,6 +1423,9 @@ def test_db_migrations_sqlite():
       'OPTIONS': {},
       'OPTIONS': {},
       'ATOMIC_REQUESTS': True,
       'ATOMIC_REQUESTS': True,
       'CONN_MAX_AGE': 0,
       'CONN_MAX_AGE': 0,
+      'AUTOCOMMIT': True,
+      'CONN_HEALTH_CHECKS': False,
+      'TIME_ZONE': None,
     }
     }
     try:
     try:
       call_command('migrate', '--fake-initial', '--database=' + name)
       call_command('migrate', '--fake-initial', '--database=' + name)