浏览代码

HUE-3287 [core] Django 1.11 upgrade
- To upgrade Django 1.8, following changes are needed:

- Django models now accept function datetime.now instead of datetime.today()
- ForeignKey with unique=True needs to redefined as OneToOneField
- django.db.backends has been renamed to django.db.backends.base.base
- LaxOptionParser is no more available, use CommandParser instead
- generic moved from django.contrib.contenttypes.generic to django.contrib.contenttypes.fields
- django.forms.util moved to django.forms.utils
- Fixing dtz.now instance as method call
- Adding Django Mako backend in settings.py
- Adding django hack for getting context processors
- Removing manually added Django-1.7/django/middleware/security.py file
- Enabling pylint test to work
- Adding pylint argument processing change

Prakash Ranade 8 年之前
父节点
当前提交
0c58e57dd0

+ 2 - 2
apps/beeswax/src/beeswax/models.py

@@ -22,7 +22,7 @@ import json
 
 from django.db import models
 from django.contrib.auth.models import User
-from django.contrib.contenttypes import generic
+from django.contrib.contenttypes.fields import GenericRelation
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
@@ -275,7 +275,7 @@ class SavedQuery(models.Model):
 
   is_redacted = models.BooleanField(default=False)
 
-  doc = generic.GenericRelation(Document, related_query_name='hql_doc')
+  doc = GenericRelation(Document, related_query_name='hql_doc')
 
   class Meta:
     ordering = ['-mtime']

+ 1 - 1
apps/filebrowser/src/filebrowser/views.py

@@ -284,7 +284,7 @@ def edit(request, path, form=None):
         breadcrumbs=parse_breadcrumbs(path),
         is_embeddable=request.GET.get('is_embeddable', False),
         show_download_button=SHOW_DOWNLOAD_BUTTON.get())
-    if request.META.get('HTTP_X_REQUESTED_WITH') != 'XMLHttpRequest':
+    if not request.is_ajax():
         data['stats'] = stats;
         data['form'] = form;
     return render("edit.mako", request, data)

+ 7 - 6
apps/oozie/src/oozie/models.py

@@ -32,11 +32,12 @@ from django.db.models import Q
 from django.core.urlresolvers import reverse
 from django.core.validators import RegexValidator
 from django.contrib.auth.models import User
-from django.contrib.contenttypes import generic
+from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.forms.models import inlineformset_factory
 from django.utils.encoding import force_unicode, smart_str
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
+import django.utils.timezone as dtz
 
 from desktop.log.access import access_warn
 from desktop.lib import django_mako
@@ -125,7 +126,7 @@ class Job(models.Model):
                                 help_text=_t('Parameters used at the submission time (e.g. market=US, oozie.use.system.libpath=true).'))
   is_trashed = models.BooleanField(default=False, db_index=True, verbose_name=_t('Is trashed'), blank=True, # Deprecated
                                    help_text=_t('If this job is trashed.'))
-  doc = generic.GenericRelation(Document, related_query_name='oozie_doc')
+  doc = GenericRelation(Document, related_query_name='oozie_doc')
   data = models.TextField(blank=True, default=json.dumps({}))  # e.g. data=json.dumps({'sla': [python data], ...})
 
   objects = JobManager()
@@ -1391,9 +1392,9 @@ class Coordinator(Job):
                                     help_text=_t('The unit of the rate at which data is periodically created.')) # unused
   timezone = models.CharField(max_length=24, choices=TIMEZONES, default='America/Los_Angeles', verbose_name=_t('Timezone'),
                               help_text=_t('The timezone of the coordinator. Only used for managing the daylight saving time changes when combining several coordinators.'))
-  start = models.DateTimeField(default=datetime.today(), verbose_name=_t('Start'),
+  start = models.DateTimeField(default=dtz.now, verbose_name=_t('Start'),
                                help_text=_t('When to start the first workflow.'))
-  end = models.DateTimeField(default=datetime.today() + timedelta(days=3), verbose_name=_t('End'),
+  end = models.DateTimeField(default=dtz.now, verbose_name=_t('End'),
                              help_text=_t('When to start the last workflow.'))
   workflow = models.ForeignKey(Workflow, null=True, verbose_name=_t('Workflow'),
                                help_text=_t('The workflow to schedule repeatedly.'))
@@ -1654,7 +1655,7 @@ class Dataset(models.Model):
                           help_text=_t('The name of the dataset.'))
   description = models.CharField(max_length=1024, blank=True, default='', verbose_name=_t('Description'),
                                  help_text=_t('A description of the dataset.'))
-  start = models.DateTimeField(default=datetime.today(), verbose_name=_t('Start'),
+  start = models.DateTimeField(default=dtz.now, verbose_name=_t('Start'),
                                help_text=_t(' The UTC datetime of the initial instance of the dataset. The initial instance also provides '
                                             'the baseline datetime to compute instances of the dataset using multiples of the frequency.'))
   frequency_number = models.SmallIntegerField(default=1, choices=FREQUENCY_NUMBERS, verbose_name=_t('Frequency number'),
@@ -1758,7 +1759,7 @@ class BundledCoordinator(models.Model):
 
 
 class Bundle(Job):
-  kick_off_time = models.DateTimeField(default=datetime.today(), verbose_name=_t('Start'),
+  kick_off_time = models.DateTimeField(default=dtz.now, verbose_name=_t('Start'),
                                        help_text=_t('When to start the first coordinators.'))
   coordinators = models.ManyToManyField(Coordinator, through='BundledCoordinator')
 

+ 2 - 2
apps/pig/src/pig/models.py

@@ -20,7 +20,7 @@ import posixpath
 
 from django.db import models
 from django.contrib.auth.models import User
-from django.contrib.contenttypes import generic
+from django.contrib.contenttypes.fields import GenericRelation
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
@@ -57,7 +57,7 @@ class PigScript(Document):
       'hadoopProperties': []
   }))
 
-  doc = generic.GenericRelation(Doc, related_query_name='pig_doc')
+  doc = GenericRelation(Doc, related_query_name='pig_doc')
 
   isV2 = False
 

+ 1 - 1
apps/useradmin/src/useradmin/forms.py

@@ -22,7 +22,7 @@ import django.contrib.auth.forms
 from django import forms
 from django.contrib.auth.models import User, Group
 from django.forms import ValidationError
-from django.forms.util import ErrorList
+from django.forms.utils import ErrorList
 from django.utils.translation import get_language, ugettext as _, ugettext_lazy as _t
 
 from desktop import conf as desktop_conf

+ 4 - 3
apps/useradmin/src/useradmin/models.py

@@ -57,6 +57,7 @@ from django.db import connection, models, transaction
 from django.contrib.auth import models as auth_models
 from django.core.cache import cache
 from django.utils.translation import ugettext_lazy as _t
+import django.utils.timezone as dtz
 
 from desktop import appmanager
 from desktop.lib.exceptions_renderable import PopupException
@@ -91,12 +92,12 @@ class UserProfile(models.Model):
   # Enum for describing the creation method of a user.
   CreationMethod = Enum('HUE', 'EXTERNAL')
 
-  user = models.ForeignKey(auth_models.User, unique=True)
+  user = models.OneToOneField(auth_models.User, unique=True)
   home_directory = models.CharField(editable=True, max_length=1024, null=True)
   creation_method = models.CharField(editable=True, null=False, max_length=64, default=str(CreationMethod.HUE))
   first_login = models.BooleanField(default=True, verbose_name=_t('First Login'),
                                    help_text=_t('If this is users first login.'))
-  last_activity = models.DateTimeField(default=datetime.fromtimestamp(0), db_index=True)
+  last_activity = models.DateTimeField(default=dtz.now, db_index=True)
 
   def get_groups(self):
     return self.user.groups.all()
@@ -162,7 +163,7 @@ def group_permissions(group):
 def create_profile_for_user(user):
   p = UserProfile()
   p.user = user
-  p.last_activity = datetime.now()
+  p.last_activity = dtz.now()
   p.home_directory = "/user/%s" % p.user.username
   try:
     p.save()

+ 1 - 1
apps/useradmin/src/useradmin/views.py

@@ -33,7 +33,7 @@ from ldap_access import LdapBindException, LdapSearchException
 from django.contrib.auth.models import User, Group
 from django.core.urlresolvers import reverse
 from django.forms import ValidationError
-from django.forms.util import ErrorList
+from django.forms.utils import ErrorList
 from django.http import HttpResponse
 from django.shortcuts import redirect
 from django.utils.encoding import smart_str

+ 0 - 45
desktop/core/ext-py/Django-1.7/django/middleware/security.py

@@ -1,45 +0,0 @@
-# This code is picked from Django 1.8 truck from https://github.com/django/django/blob/master/django/middleware/security.py
-import re
-
-from django.conf import settings
-from django.http import HttpResponsePermanentRedirect
-
-
-class SecurityMiddleware(object):
-    def __init__(self, get_response=None):
-        self.sts_seconds = settings.SECURE_HSTS_SECONDS
-        self.sts_include_subdomains = settings.SECURE_HSTS_INCLUDE_SUBDOMAINS
-        self.content_type_nosniff = settings.SECURE_CONTENT_TYPE_NOSNIFF
-        self.xss_filter = settings.SECURE_BROWSER_XSS_FILTER
-        self.redirect = settings.SECURE_SSL_REDIRECT
-        self.redirect_host = settings.SECURE_SSL_HOST
-        self.redirect_exempt = [re.compile(r) for r in settings.SECURE_REDIRECT_EXEMPT]
-        self.get_response = get_response
-
-    def process_request(self, request):
-        path = request.path.lstrip("/")
-        if (self.redirect and not request.is_secure() and
-                not any(pattern.search(path)
-                        for pattern in self.redirect_exempt)):
-            host = self.redirect_host or request.get_host()
-            return HttpResponsePermanentRedirect(
-                "https://%s%s" % (host, request.get_full_path())
-            )
-
-    def process_response(self, request, response):
-        if (self.sts_seconds and request.is_secure() and
-                'strict-transport-security' not in response):
-            sts_header = "max-age=%s" % self.sts_seconds
-
-            if self.sts_include_subdomains:
-                sts_header = sts_header + "; includeSubDomains"
-
-            response["strict-transport-security"] = sts_header
-
-        if self.content_type_nosniff and 'x-content-type-options' not in response:
-            response["x-content-type-options"] = "nosniff"
-
-        if self.xss_filter and 'x-xss-protection' not in response:
-            response["x-xss-protection"] = "1; mode=block"
-
-        return response

+ 22 - 1
desktop/core/ext-py/Django-1.8/django/template/context.py

@@ -195,6 +195,24 @@ class RenderContext(BaseContext):
     def __getitem__(self, key):
         return self.dicts[-1][key]
 
+from django.utils.module_loading import import_string
+_standard_context_processors = None
+
+# This is a function rather than module-level procedural code because we only
+# want it to execute if somebody uses RequestContext.
+def get_standard_processors():
+    from django.conf import settings
+    global _standard_context_processors
+    if _standard_context_processors is None:
+        processors = []
+        collect = []
+        collect.extend(_builtin_context_processors)
+        collect.extend(settings.TEMPLATE_CONTEXT_PROCESSORS)
+        for path in collect:
+            func = import_string(path)
+            processors.append(func)
+        _standard_context_processors = tuple(processors)
+    return _standard_context_processors
 
 class RequestContext(Context):
     """
@@ -219,7 +237,10 @@ class RequestContext(Context):
         self.request = request
         self._processors = () if processors is None else tuple(processors)
         self._processors_index = len(self.dicts)
-        self.update({})         # placeholder for context processors output
+        updates = dict()
+        for processor in get_standard_processors():
+            updates.update(processor(request))
+        self.update(updates)
 
     @contextmanager
     def bind_template(self, template):

+ 2 - 2
desktop/core/ext-py/djangomako-1.0.1/djangomako/backends.py

@@ -12,7 +12,7 @@ import tempfile
 
 from django.core.urlresolvers import reverse
 from django.contrib.staticfiles.templatetags.staticfiles import static
-from django.template import TemplateDoesNotExist, TemplateSyntaxError
+from django.template.base import TemplateDoesNotExist, TemplateSyntaxError
 from django.template.backends.base import BaseEngine
 from django.template.backends.utils import csrf_input_lazy, \
     csrf_token_lazy
@@ -159,4 +159,4 @@ class Template(object):
             context['static'] = static
             context['url'] = reverse
 
-        return self.template.render(**context)
+        return self.template.render(**context)

+ 1 - 1
desktop/core/src/desktop/lib/django_forms.py

@@ -23,7 +23,7 @@ import urllib
 
 from django.forms import Widget, Field
 from django import forms
-from django.forms.util import ErrorList, ValidationError, flatatt
+from django.forms.utils import ErrorList, ValidationError, flatatt
 from django.forms.fields import MultiValueField, CharField, ChoiceField, BooleanField
 from django.forms.widgets import MultiWidget, Select, TextInput, Textarea, HiddenInput, Input
 from django.utils import formats

+ 5 - 4
desktop/core/src/desktop/lib/django_mako.py

@@ -103,9 +103,11 @@ def render_to_string_test(template_name, django_context):
 
 def render_to_string_normal(template_name, django_context):
   data_dict = dict()
-  if isinstance(django_context, django.template.Context):
+  if isinstance(django_context, django.template.context.Context):
     for d in reversed(django_context.dicts):
-      data_dict.update(d)
+      if d:
+        data_dict.update(d)
+    data_dict.update(django_context.request)
   else:
     data_dict = django_context
 
@@ -130,8 +132,7 @@ def url(view_name, *args, **view_args):
   from django.core.urlresolvers import reverse
   return reverse(view_name, args=args, kwargs=view_args)
 
-
-from django.core.context_processors import csrf
+from django.template.context_processors import csrf
 
 def csrf_token(request):
   """

+ 4 - 4
desktop/core/src/desktop/lib/django_util.py

@@ -25,16 +25,16 @@ import datetime
 
 from django.conf import settings
 from django.core import urlresolvers, serializers
-from django.core.context_processors import csrf
+from django.template.context_processors import csrf
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.http import QueryDict, HttpResponse, HttpResponseRedirect
 from django.shortcuts import render_to_response as django_render_to_response
-from django.template import RequestContext
+from django.template.context import RequestContext
 from django.template.loader import render_to_string as django_render_to_string
 from django.utils.http import urlencode # this version is unicode-friendly
 from django.utils.translation import ungettext, ugettext
-from django.utils.tzinfo import LocalTimezone
+from django.utils.timezone import LocalTimezone
 
 import desktop.conf
 import desktop.lib.thrift_util
@@ -224,7 +224,7 @@ def render(template, request, data, json=None, template_lib=None, force_template
   else:
     return _render_to_response(template,
                                request,
-                               RequestContext(request=request, dict_=data),
+                               RequestContext(request, data),
                                template_lib=template_lib,
                                status=status,
                                **kwargs)

+ 9 - 7
desktop/core/src/desktop/manage_entry.py

@@ -37,19 +37,21 @@ def entry():
 
   from django.core.exceptions import ImproperlyConfigured
   from django.core.management import execute_from_command_line, find_commands
-  from django.core.management import LaxOptionParser
+  from django.core.management import CommandParser
   from django.core.management.base import BaseCommand
 
   os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
 
   # What's the subcommand being run?
   # This code uses the same logic from django.core.management to handle command args
-  argv = sys.argv[:]
-  parser = LaxOptionParser(option_list=BaseCommand.option_list)
-  parser.parse_args(argv)
-
-  if len(argv) > 1:
-    prof_id = subcommand = argv[1]
+  subcommand = None
+  if len(sys.argv) > 1:
+    subcommand = sys.argv[1]
+  parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]", add_help=False)
+  parser.parse_known_args(sys.argv[2:])
+
+  if len(sys.argv) > 1:
+    prof_id = subcommand = sys.argv[1]
     commands_req_db = [ "changepassword", "createsuperuser",
                         "clean_history_docs", "convert_documents", "sync_documents",
                         "dbshell", "dumpdata", "loaddata", "shell",

+ 25 - 12
desktop/core/src/desktop/management/commands/runpylint.py

@@ -39,23 +39,36 @@ class Command(BaseCommand):
     python core/manage.py runpylint
   """)
 
-  def handle(self, *args, **options):
-    """Check the source code using PyLint."""
+  def valid_app(self):
+    from desktop import appmanager
+    apps = ["desktop"]
+    for app in appmanager.DESKTOP_APPS:
+      apps.append(app.name)
+    return apps
 
-    pylint_args = list(args)
+  def add_arguments(self, parser):
+    parser.add_argument('-f', '--force', dest='force', default='true', action="store_true")
+    parser.add_argument('--output-format',
+      action='store', dest='outputformat', default='parseable')
+    parser.add_argument('-a', '--app', dest='app', action='store', default='all', choices=self.valid_app())
 
-    if "all" in pylint_args or len(pylint_args) == 0:
-      if "all" in pylint_args:
-        pylint_args.remove("all")
-      from desktop import appmanager
-      apps = ["desktop"]
-      for app in appmanager.DESKTOP_APPS:
-        apps.append(app.name)
-      pylint_args = apps + pylint_args
+  def handle(self, *args, **options):
+    """Check the source code using PyLint."""
 
     # Note that get_build_dir() is suitable for testing use only.
     pylint_prog = paths.get_build_dir('env', 'bin', 'pylint')
-    pylint_args = [pylint_prog, "--rcfile=" + settings.PYLINTRC] + pylint_args
+    pylint_args = [pylint_prog, "--rcfile=" + settings.PYLINTRC]
+
+    if options['app']=='all':
+      pylint_args.extend(self.valid_app())
+    else:
+      pylint_args.append(options['app'])
+
+    if options['force']:
+      pylint_args.append('-f')
+
+    if options['outputformat']:
+      pylint_args.append(options['outputformat'])
 
     if not os.path.exists(pylint_prog):
       msg = _("Cannot find pylint at '%(path)s'. Please install pylint first.") % {'path': pylint_prog}

+ 3 - 3
desktop/core/src/desktop/models.py

@@ -30,7 +30,7 @@ except ImportError:
 from itertools import chain
 
 from django.contrib.auth import models as auth_models
-from django.contrib.contenttypes import generic
+from django.contrib.contenttypes.fields import GenericRelation, GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.staticfiles.storage import staticfiles_storage
 from django.core.urlresolvers import reverse, NoReverseMatch
@@ -611,7 +611,7 @@ class Document(models.Model):
 
   content_type = models.ForeignKey(ContentType)
   object_id = models.PositiveIntegerField()
-  content_object = generic.GenericForeignKey('content_type', 'object_id')
+  content_object = GenericForeignKey('content_type', 'object_id')
 
   objects = DocumentManager()
 
@@ -1077,7 +1077,7 @@ class Document2(models.Model):
 
   parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)
 
-  doc = generic.GenericRelation(Document, related_query_name='doc_doc') # Compatibility with Hue 3
+  doc = GenericRelation(Document, related_query_name='doc_doc') # Compatibility with Hue 3
 
   objects = Document2Manager()
 

+ 15 - 5
desktop/core/src/desktop/settings.py

@@ -210,15 +210,25 @@ LOCALE_PATHS = [
 # Keep default values up to date
 TEMPLATE_CONTEXT_PROCESSORS = (
   'django.contrib.auth.context_processors.auth',
-  'django.core.context_processors.debug',
-  'django.core.context_processors.i18n',
-  'django.core.context_processors.media',
-  'django.core.context_processors.request',
+  'django.template.context_processors.debug',
+  'django.template.context_processors.i18n',
+  'django.template.context_processors.media',
+  'django.template.context_processors.request',
   'django.contrib.messages.context_processors.messages',
    # Not default
   'desktop.context_processors.app_name',
 )
 
+TEMPLATES = [
+  {
+    'BACKEND': 'djangomako.backends.MakoBackend',
+    'DIRS': TEMPLATE_DIRS,
+    'NAME': 'mako',
+    'OPTIONS': {
+      'context_processors': TEMPLATE_CONTEXT_PROCESSORS,
+    },
+  },
+]
 
 # Desktop doesn't use an auth profile module, because
 # because it doesn't mesh very well with the notion
@@ -515,7 +525,7 @@ if desktop.conf.INSTRUMENTATION.get():
 
 if not desktop.conf.DATABASE_LOGGING.get():
   def disable_database_logging():
-    from django.db.backends import BaseDatabaseWrapper
+    from django.db.backends.base.base import BaseDatabaseWrapper
     from django.db.backends.util import CursorWrapper
 
     BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(cursor, self)

+ 1 - 1
desktop/core/src/desktop/views.py

@@ -511,7 +511,7 @@ def commonheader(title, section, user, request=None, padding="90px", skip_topbar
 
 def get_banner_message(request):
   banner_message = None
-  forwarded_host = request.META.get('HTTP_X_FORWARDED_HOST')
+  forwarded_host = request.get_host()
 
   message = None;
   path_info = request.environ.get("PATH_INFO")

+ 1 - 1
tools/jenkins/jenkins.sh

@@ -45,7 +45,7 @@ build_sqoop
 
 make apps
 
-build/env/bin/hue runpylint all -- -f parseable > PYLINT.txt
+build/env/bin/hue runpylint > PYLINT.txt
 
 rm -f JAVASCRIPTLINT.txt
 for FILE in $(find . -name *.js);