Ver Fonte

Merge remote-tracking branch 'origin/master'

Romain há 6 anos atrás
pai
commit
55ee255cd4

+ 3 - 1
apps/beeswax/src/beeswax/hive_site.py

@@ -28,7 +28,7 @@ import socket
 
 from desktop.lib import security_util
 from hadoop import confparse
-from hadoop.ssl_client_site import get_trustore_location
+from hadoop.ssl_client_site import get_trustore_location, get_trustore_password
 
 import beeswax.conf
 
@@ -155,6 +155,8 @@ def hiveserver2_jdbc_url():
 
     if get_conf().get(_CNF_HIVESERVER2_TRUSTSTORE_PASSWORD):
       urlbase += ';trustStorePassword=%s' % get_conf().get(_CNF_HIVESERVER2_TRUSTSTORE_PASSWORD)
+    elif get_trustore_password():
+      urlbase += ';trustStorePassword=%s' % get_trustore_password()
 
   if is_transport_mode_http:
     urlbase += ';transportMode=http'

+ 48 - 0
apps/useradmin/src/useradmin/forms.py

@@ -244,6 +244,54 @@ if ENABLE_ORGANIZATIONS.get():
           self.initial['groups'] = []
 
 
+if ENABLE_ORGANIZATIONS.get():
+  class OrganizationUserChangeForm(UserChangeForm):
+    username = None
+    email = forms.CharField(
+        label=_t("Email"),
+        widget=forms.TextInput(attrs={'maxlength': 150, 'placeholder': _t("Email"), 'autocomplete': 'off', 'autofocus': 'autofocus'})
+    )
+
+    class Meta(django.contrib.auth.forms.UserChangeForm.Meta):
+      model =  User
+      fields = ["first_name", "last_name", "email", "ensure_home_directory"]
+      if ENABLE_ORGANIZATIONS.get():
+        fields.append('organization') # Because of import logic
+
+    def __init__(self, *args, **kwargs):
+      super(OrganizationUserChangeForm, self).__init__(*args, **kwargs)
+
+      if self.instance.id:
+        self.fields['email'].widget.attrs['readonly'] = True
+
+      self.fields['organization'] = forms.ChoiceField(choices=((default_organization().id, default_organization()),), initial=default_organization())
+
+    def clean_organization(self):
+      try:
+        return Organization.objects.get(id=int(self.cleaned_data.get('organization')))
+      except:
+        LOG.exception('The organization does not exist.')
+        return None
+
+  # Mixin __init__ method?
+  class OrganizationSuperUserChangeForm(OrganizationUserChangeForm):
+    class Meta(UserChangeForm.Meta):
+      fields = ["email", "is_active"] + OrganizationUserChangeForm.Meta.fields + ["is_superuser", "unlock_account", "groups"]
+
+    def __init__(self, *args, **kwargs):
+      super(OrganizationSuperUserChangeForm, self).__init__(*args, **kwargs)
+      if self.instance.id:
+        # If the user exists already, we'll use its current group memberships
+        self.initial['groups'] = set(self.instance.groups.all())
+      else:
+        # If this is a new user, suggest the default group
+        default_group = get_default_user_group()
+        if default_group is not None:
+          self.initial['groups'] = set([default_group])
+        else:
+          self.initial['groups'] = []
+
+
 class PasswordChangeForm(UserChangeForm):
   """
   This inherits from UserChangeForm to allow for forced password change on first login

+ 3 - 0
apps/useradmin/src/useradmin/models.py

@@ -66,6 +66,9 @@ else:
   def default_organization(): pass
   def get_organization(): pass
 
+from desktop.monkey_patches import monkey_patch_username_validator
+monkey_patch_username_validator()
+
 
 LOG = logging.getLogger(__name__)
 

+ 15 - 8
apps/useradmin/src/useradmin/tests.py

@@ -696,17 +696,24 @@ class TestUserAdmin(BaseUserAdminTests):
 
       # Create a new regular user (duplicate name)
       response = c.post('/useradmin/users/new', dict(username="test", password1="test", password2="test"))
-      assert_equal({ 'username': ['Username already exists.']}, response.context[0]["form"].errors)
+      assert_equal({'username': ['Username already exists.']}, response.context[0]["form"].errors)
 
       # Create a new regular user (for real)
-      response = c.post('/useradmin/users/new', dict(username=FUNNY_NAME,
-                                               password1="test",
-                                               password2="test",
-                                               is_superuser=True,
-                                               is_active=True))
-      response = c.get('/useradmin/')
+      response = c.post('/useradmin/users/new', dict(
+          username=FUNNY_NAME,
+          password1="test",
+          password2="test",
+          is_superuser=True,
+          is_active=True
+        ),
+        follow=True
+      )
+      if response.status_code != 200:
+        assert_false(response.context[0]["form"].errors)
+      assert_equal(response.status_code, 200, response.content)
 
-      assert_true(FUNNY_NAME in response.content)
+      response = c.get('/useradmin/')
+      assert_true(FUNNY_NAME in response.content, response.content)
       assert_true(len(response.context[0]["users"]) > 1)
       assert_true("Hue Users" in response.content)
       # Validate profile is created.

+ 1 - 0
apps/useradmin/src/useradmin/urls.py

@@ -22,6 +22,7 @@ from desktop.lib.django_util import get_username_re_rule, get_groupname_re_rule
 from useradmin import views as useradmin_views
 from useradmin import api as useradmin_api
 
+
 username_re = get_username_re_rule()
 groupname_re = get_groupname_re_rule()
 

+ 5 - 1
apps/useradmin/src/useradmin/views.py

@@ -22,6 +22,7 @@ import grp
 import logging
 import threading
 import subprocess
+import sys
 import json
 
 from axes.decorators import FAILURE_LIMIT, LOCK_OUT_AT_FAILURE
@@ -53,6 +54,9 @@ from useradmin.forms import SyncLdapUsersGroupsForm, AddLdapGroupsForm, AddLdapU
 from useradmin.ldap_access import LdapBindException, LdapSearchException
 from useradmin.models import HuePermission, UserProfile, LdapGroup, get_profile, get_default_user_group, User, Group
 
+if sys.version_info[0] > 2:
+  unicode = str
+
 if ENABLE_ORGANIZATIONS.get():
   from useradmin.forms import OrganizationUserChangeForm as UserChangeForm, OrganizationSuperUserChangeForm as SuperUserChangeForm
 else:
@@ -830,7 +834,7 @@ def ensure_home_directory(fs, user):
     home_directory = userprofile.home_directory.split('@')[0]
 
   if userprofile is not None and userprofile.home_directory:
-    if not isinstance(home_directory, str):
+    if not isinstance(home_directory, unicode):
       home_directory = home_directory.decode("utf-8")
     fs.do_as_user(username, fs.create_home_dir, home_directory)
   else:

+ 1 - 1
desktop/core/src/desktop/auth/backend.py

@@ -48,7 +48,7 @@ from mozilla_django_oidc.auth import OIDCAuthenticationBackend, default_username
 from mozilla_django_oidc.utils import absolutify, import_from_settings
 
 from desktop import metrics
-from desktop.conf import AUTH, LDAP, OIDC
+from desktop.conf import AUTH, LDAP, OIDC, ENABLE_ORGANIZATIONS
 from desktop.settings import LOAD_BALANCER_COOKIE
 
 from useradmin import ldap_access

+ 3 - 2
desktop/core/src/desktop/lib/fs/proxyfs.py

@@ -27,7 +27,7 @@ from urlparse import urlparse
 from useradmin.models import User
 
 from desktop.auth.backend import is_admin
-from desktop.conf import DEFAULT_USER
+from desktop.conf import DEFAULT_USER, ENABLE_ORGANIZATIONS
 
 if sys.version_info[0] > 2:
   from urllib.parse import urlparse as lib_urlparse
@@ -44,7 +44,8 @@ class ProxyFS(object):
   def __init__(self, filesystems_dict, default_scheme, name='default'):
     if default_scheme not in filesystems_dict:
       raise ValueError(
-        'Default scheme "%s" is not a member of provided schemes: %s' % (default_scheme, list(filesystems_dict.keys())))
+        'Default scheme "%s" is not a member of provided schemes: %s' % (default_scheme, list(filesystems_dict.keys()))
+      )
 
     self._name = name
     self._fs_dict = filesystems_dict

+ 3 - 2
desktop/core/src/desktop/lib/fsmanager.py

@@ -27,7 +27,7 @@ from azure.conf import is_adls_enabled, is_abfs_enabled, has_adls_access, has_ab
 
 from desktop.lib.fs.proxyfs import ProxyFS
 from desktop.conf import is_gs_enabled, has_gs_access
-from desktop.lib.fs.gc.client import get_client as get_client_gs 
+from desktop.lib.fs.gc.client import get_client as get_client_gs
 
 from hadoop.cluster import get_hdfs
 from hadoop.conf import has_hdfs_enabled
@@ -49,7 +49,7 @@ def has_access(fs=None, user=None):
     return has_gs_access(user)
 
 
-def is_enabled(fs=None):
+def is_enabled(fs):
   if fs == 'hdfs':
     return has_hdfs_enabled()
   elif fs == 'adl':
@@ -111,6 +111,7 @@ def get_filesystem(name='default'):
 def get_filesystems(user):
   return [fs for fs in SUPPORTED_FS if is_enabled(fs) and has_access(fs, user)]
 
+
 def _get_client(fs=None):
   if fs == 'hdfs':
     return get_hdfs

+ 5 - 5
desktop/core/src/desktop/lib/rest/http_client.py

@@ -123,17 +123,17 @@ class HttpClient(object):
     short_url = '%(scheme)s://%(netloc)s' % {'scheme': parsed_uri.scheme, 'netloc': parsed_uri.netloc}
     return short_url
 
-  def set_kerberos_auth(self):
+  def set_kerberos_auth(self, service="HTTP"):
     """Set up kerberos auth for the client, based on the current ticket."""
     mutual_auth = conf.KERBEROS.MUTUAL_AUTHENTICATION.get().upper()
     if mutual_auth == 'OPTIONAL':
-      self._session.auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
+      self._session.auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL, service=service)
     elif mutual_auth == 'REQUIRED':
-      self._session.auth = HTTPKerberosAuth(mutual_authentication=REQUIRED)
+      self._session.auth = HTTPKerberosAuth(mutual_authentication=REQUIRED, service=service)
     elif mutual_auth == 'DISABLED':
-      self._session.auth = HTTPKerberosAuth(mutual_authentication=DISABLED)
+      self._session.auth = HTTPKerberosAuth(mutual_authentication=DISABLED, service=service)
     else:
-      self._session.auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
+      self._session.auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL, service=service)
     return self
 
   def set_basic_auth(self, username, password):

+ 2 - 2
desktop/core/src/desktop/lib/thrift_/http_client.py

@@ -55,8 +55,8 @@ class THttpClient(TTransportBase):
   def open(self):
     pass
 
-  def set_kerberos_auth(self):
-    self._client.set_kerberos_auth()
+  def set_kerberos_auth(self, service="HTTP"):
+    self._client.set_kerberos_auth(service=service)
 
   def set_basic_auth(self, username, password):
     self._client.set_basic_auth(username, password)

+ 1 - 1
desktop/core/src/desktop/lib/thrift_util.py

@@ -317,7 +317,7 @@ def connect_to_thrift(conf):
 
   if conf.transport_mode == 'http':
     if conf.use_sasl and conf.mechanism != 'PLAIN':
-      mode.set_kerberos_auth()
+      mode.set_kerberos_auth(service=conf.kerberos_principal)
     else:
       mode.set_basic_auth(conf.username, conf.password)
 

+ 0 - 3
desktop/core/src/desktop/monkey_patches.py

@@ -40,6 +40,3 @@ def monkey_patch_username_validator():
   for validator in username.validators:
     if isinstance(validator, RegexValidator):
       validator.regex = regex
-
-
-monkey_patch_username_validator()

+ 5 - 2
desktop/core/src/desktop/tests.py

@@ -42,6 +42,7 @@ from django.urls import reverse
 from django.test.client import Client
 from django.views.static import serve
 from django.http import HttpResponse
+from mock import patch, Mock, MagicMock
 from nose.plugins.attrib import attr
 from nose.plugins.skip import SkipTest
 from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal, assert_raises, nottest
@@ -273,8 +274,10 @@ def test_dump_config():
 
   finish = desktop.conf.ENABLE_CONNECTORS.set_for_testing(True)
   try:
-    response = c.get(reverse('desktop.views.dump_config'))
-    assert_equal(1, len(response.context[0]['apps']), response.context[0])
+    with patch('desktop.lib.fsmanager.has_hdfs_enabled') as has_hdfs_enabled:
+      has_hdfs_enabled.return_value = True
+      response = c.get(reverse('desktop.views.dump_config'))
+      assert_equal(1, len(response.context[0]['apps']), response.context[0])
   finally:
     finish()
 

+ 3 - 4
desktop/core/src/desktop/urls.py

@@ -198,10 +198,9 @@ if METRICS.ENABLE_WEB_METRICS.get():
     url(r'^desktop/metrics/?', include('desktop.lib.metrics.urls'))
   ]
 
-if has_connectors():
-  dynamic_patterns += [
-    url(r'^desktop/connectors/?', include('desktop.lib.connectors.urls'))
-  ]
+dynamic_patterns += [
+  url(r'^desktop/connectors/?', include('desktop.lib.connectors.urls'))
+]
 
 if ANALYTICS.IS_ENABLED.get():
   dynamic_patterns += [

+ 3 - 2
desktop/core/src/desktop/views.py

@@ -68,6 +68,7 @@ if sys.version_info[0] > 2:
 else:
   from StringIO import StringIO as string_io
 
+
 LOG = logging.getLogger(__name__)
 
 
@@ -133,7 +134,7 @@ def home(request):
 
 def home2(request, is_embeddable=False):
   apps = appmanager.get_apps_dict(request.user)
-  
+
   return render('home2.mako', request, {
     'apps': apps,
     'is_embeddable': request.GET.get('is_embeddable', False)
@@ -303,7 +304,7 @@ def dump_config(request):
 
   apps = sorted(app_modules, key=lambda app: app.name)
   apps_names = [app.name for app in apps]
-  top_level = sorted(list(GLOBAL_CONFIG.get().values()), key=lambda obj: apps_names.index(obj.config.key))
+  top_level = sorted(config_modules, key=lambda obj: apps_names.index(obj.config.key))
 
   return render("dump_config.mako", request, {
       'show_private': show_private,

+ 2 - 2
desktop/libs/aws/src/aws/client.py

@@ -133,13 +133,13 @@ class CredentialProviderIDBroker(object):
 
 
 class Client(object):
-  def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=aws_conf.AWS_ACCOUNT_REGION_DEFAULT,
+  def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=None,
                timeout=HTTP_SOCKET_TIMEOUT_S, host=None, proxy_address=None, proxy_port=None, proxy_user=None,
                proxy_pass=None, calling_format=None, is_secure=True, expiration=None):
     self._access_key_id = aws_access_key_id
     self._secret_access_key = aws_secret_access_key
     self._security_token = aws_security_token
-    self._region = region.lower()
+    self._region = region.lower() if region else region
     self._timeout = timeout
     self._host = host
     self._proxy_address = proxy_address

+ 44 - 19
desktop/libs/aws/src/aws/conf.py

@@ -38,24 +38,29 @@ PERMISSION_ACTION_S3 = "s3_access"
 
 def get_locations():
   return ('EU',  # Ireland
-    'eu-central-1',  # Frankfurt
-    'eu-west-1',
-    'eu-west-2',
-    'eu-west-3',
-    'ca-central-1',
-    'us-east-1',
-    'us-east-2',
-    'us-west-1',
-    'us-west-2',
-    'sa-east-1',
+    'ap-east-1',
     'ap-northeast-1',
     'ap-northeast-2',
     'ap-northeast-3',
     'ap-southeast-1',
     'ap-southeast-2',
     'ap-south-1',
+    'ca-central-1',
     'cn-north-1',
-    'cn-northwest-1')
+    'cn-northwest-1',
+    'eu-central-1',  # Frankfurt
+    'eu-north-1',
+    'eu-west-1',
+    'eu-west-2',
+    'eu-west-3',
+    'me-south-1',
+    'sa-east-1',
+    'us-east-1',
+    'us-east-2',
+    'us-gov-east-1',
+    'us-gov-west-1',
+    'us-west-1',
+    'us-west-2')
 
 
 def get_default_access_key_id():
@@ -101,10 +106,22 @@ def get_region(conf):
     elif conf.REGION.get():
       region = conf.REGION.get()
 
-    # If the parsed out region is not in the list of supported regions, fallback to the default
-    if region not in get_locations():
-      LOG.warn("Region, %s, not found in the list of supported regions: %s" % (region, ', '.join(get_locations())))
-      region = ''
+  if not region and is_ec2_instance():
+    try:
+      import boto.utils
+      data = boto.utils.get_instance_identity(timeout=1, num_retries=1)
+      if data:
+        region = data['document']['region']
+    except Exception as e:
+      LOG.exception("Encountered error when fetching instance identity: %s" % e)
+
+  if not region:
+    region = AWS_ACCOUNT_REGION_DEFAULT
+
+  # If the parsed out region is not in the list of supported regions, fallback to the default
+  if region not in get_locations():
+    LOG.warn("Region, %s, not found in the list of supported regions: %s" % (region, ', '.join(get_locations())))
+    region = ''
 
   return region
 
@@ -160,7 +177,7 @@ AWS_ACCOUNTS = UnspecifiedConfigSection(
       ),
       REGION=Config(
         key='region',
-        default=AWS_ACCOUNT_REGION_DEFAULT,
+        default=None,
         type=str
       ),
       HOST=Config(
@@ -218,12 +235,20 @@ def is_enabled():
   return ('default' in list(AWS_ACCOUNTS.keys()) and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or has_iam_metadata() or conf_idbroker.is_idbroker_enabled('s3a')
 
 
+def is_ec2_instance():
+  # To avoid unnecessary network call, check if Hue is running on EC2 instance
+  # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
+  # /sys/hypervisor/uuid doesn't work on m5/c5, but /sys/devices/virtual/dmi/id/product_uuid does
+  try:
+    return (os.path.exists('/sys/hypervisor/uuid') and open('/sys/hypervisor/uuid', 'read').read()[:3].lower() == 'ec2') or (os.path.exists('/sys/devices/virtual/dmi/id/product_uuid') and open('/sys/devices/virtual/dmi/id/product_uuid', 'read').read()[:3].lower() == 'ec2')
+  except Exception as e:
+    LOG.exception("Failed to read /sys/hypervisor/uuid or /sys/devices/virtual/dmi/id/product_uuid: %s" % e)
+    return False
+
 def has_iam_metadata():
   try:
     import boto.utils
-    # To avoid unnecessary network call, check if Hue is running on EC2 instance
-    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
-    if os.path.exists('/sys/hypervisor/uuid') and open('/sys/hypervisor/uuid', 'read').read()[:3] == 'ec2':
+    if is_ec2_instance():
       metadata = boto.utils.get_instance_metadata(timeout=1, num_retries=1)
       return 'iam' in metadata
   except Exception as e:

+ 1 - 2
desktop/libs/hadoop/src/hadoop/conf.py

@@ -35,8 +35,7 @@ def find_file_recursive(desired_glob, root):
       matches = fnmatch.filter(filenames, desired_glob)
       if matches:
         if len(matches) != 1:
-          logging.warning("Found multiple jars matching %s: %s" %
-                          (desired_glob, matches))
+          logging.warning("Found multiple jars matching %s: %s" % (desired_glob, matches))
         return os.path.join(dirpath, matches[0])
 
     logging.error("Trouble finding jars matching %s" % (desired_glob,))

+ 5 - 0
desktop/libs/hadoop/src/hadoop/ssl_client_site.py

@@ -28,6 +28,7 @@ _SSL_SITE_PATH = None                  # Path to ssl-client.xml
 _SSL_SITE_DICT = None                  # A dictionary of name/value config options
 
 _CNF_TRUSTORE_LOCATION = 'ssl.client.truststore.location'
+_CNF_TRUSTORE_PASSWORD = 'ssl.client.truststore.password'
 
 LOG = logging.getLogger(__name__)
 
@@ -66,3 +67,7 @@ def _parse_ssl_client_site():
 
 def get_trustore_location():
   return get_conf().get(_CNF_TRUSTORE_LOCATION)
+
+
+def get_trustore_password():
+  return get_conf().get(_CNF_TRUSTORE_PASSWORD)

+ 0 - 2
desktop/libs/notebook/src/notebook/api.py

@@ -23,7 +23,6 @@ import logging
 import sqlparse
 import sys
 
-
 from django.urls import reverse
 from django.db.models import Q
 from django.utils.translation import ugettext as _
@@ -45,7 +44,6 @@ from notebook.connectors.hiveserver2 import HS2Api
 from notebook.connectors.oozie_batch import OozieApi
 from notebook.decorators import api_error_handler, check_document_access_permission, check_document_modify_permission
 from notebook.models import escape_rows, make_notebook, upgrade_session_properties, get_api
-from notebook.views import upgrade_session_properties, get_api
 
 if sys.version_info[0] > 2:
   import urllib.request, urllib.error

+ 0 - 1
desktop/libs/notebook/src/notebook/tasks.py

@@ -31,7 +31,6 @@ from celery.utils.log import get_task_logger
 from celery import states
 from django.core.cache import caches
 from django.core.files.storage import get_storage_class
-from django.contrib.auth.models import User
 from django.db import transaction
 from django.http import FileResponse, HttpRequest