Răsfoiți Sursa

HUE-1322 [useradmin] Add support for POSIX LDAP Groups

- Only import groups will import new users
- memberOf and isMemberOf are not generated for posix groups
Abraham Elmahrek 12 ani în urmă
părinte
comite
68eaa55

+ 107 - 37
apps/useradmin/src/useradmin/ldap_access.py

@@ -52,6 +52,7 @@ def get_ldap_username(username, nt_domain):
   else:
     return username
 
+
 class LdapConnection(object):
   """
   Constructor creates LDAP connection. Contains methods
@@ -103,26 +104,22 @@ class LdapConnection(object):
     else:
       return (base_dn, '(' + attr + '=' + name + ')')
 
-  def find_users(self, username_pattern, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+  def _transform_find_user_results(self, result_data, user_name_attr):
     """
-    LDAP search helper method finding users. This supports searching for users
-    by distinguished name, or the configured username attribute.
+    :param result_data: List of dictionaries that have ldap attributes and their associated values. Generally the result list from an ldapsearch request.
+    :param user_name_attr: The ldap attribute that is returned by the server to map to ``username`` in the return dictionary.
+    
+    :returns list of dictionaries that take on the following form: {
+      'dn': <distinguished name of entry>,
+      'username': <ldap attribute associated with user_name_attr>
+      'first': <first name>
+      'last': <last name>
+      'email': <email>
+      'groups': <list of DNs of groups that user is a member of>
+    }
     """
-    user_filter = desktop.conf.LDAP.USERS.USER_FILTER.get()
-    if not user_filter.startswith('('):
-      user_filter = '(' + user_filter + ')'
-    user_name_attr = desktop.conf.LDAP.USERS.USER_NAME_ATTR.get()
-
-    # Allow wild cards on non distinguished names
-    sanitized_name = ldap.filter.escape_filter_chars(username_pattern).replace(r'\2a', r'*')
-    search_dn, user_name_filter = self._get_search_params(sanitized_name, user_name_attr, find_by_dn)
-    ldap_filter = '(&' + user_filter + user_name_filter + ')'
-    attrlist = ['isMemberOf', 'memberOf', 'givenName', 'sn', 'mail', 'dn', user_name_attr]
-
-    ldap_result_id = self.ldap_handle.search(search_dn, scope, ldap_filter, attrlist)
-    result_type, result_data = self.ldap_handle.result(ldap_result_id)
     user_info = []
-    if result_data and result_type == ldap.RES_SEARCH_RESULT:
+    if result_data:
       for dn, data in result_data:
         # Skip Active Directory # refldap entries.
         if dn is not None:
@@ -150,30 +147,12 @@ class LdapConnection(object):
             ldap_info['groups'] = data['isMemberOf']
 
           user_info.append(ldap_info)
-
     return user_info
 
-  def find_groups(self, groupname_pattern, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
-    """
-    LDAP search helper method for finding groups
-    """
-    base_dn = self._get_root_dn()
-
-    group_filter = desktop.conf.LDAP.GROUPS.GROUP_FILTER.get()
-    if not group_filter.startswith('('):
-      group_filter = '(' + group_filter + ')'
-    group_name_attr = desktop.conf.LDAP.GROUPS.GROUP_NAME_ATTR.get()
-
-    # Allow wild cards on non distinguished names
-    sanitized_name = ldap.filter.escape_filter_chars(groupname_pattern).replace(r'\2a', r'*')
-    search_dn, group_name_filter = self._get_search_params(sanitized_name, group_name_attr, find_by_dn)
-    ldap_filter = '(&' + group_filter + group_name_filter + ')'
-
-    ldap_result_id = self.ldap_handle.search(search_dn, scope, ldap_filter)
-    result_type, result_data = self.ldap_handle.result(ldap_result_id)
 
+  def _transform_find_group_results(self, result_data, group_name_attr):
     group_info = []
-    if result_data and result_type == ldap.RES_SEARCH_RESULT:
+    if result_data:
       for dn, data in result_data:
 
         # Skip Active Directory # refldap entries.
@@ -195,10 +174,101 @@ class LdapConnection(object):
           else:
             ldap_info['members'] = []
 
+          if 'posixGroup' in data['objectClass'] and 'memberUid' in data:
+            ldap_info['posix_members'] = data['memberUid']
+          else:
+            ldap_info['posix_members'] = []
+
           group_info.append(ldap_info)
 
     return group_info
 
+  def find_users(self, username_pattern, search_attr=None, user_name_attr=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+    """
+    LDAP search helper method finding users. This supports searching for users
+    by distinguished name, or the configured username attribute.
+
+    :param username_pattern: The pattern to match ``search_attr`` against. Defaults to ``search_attr`` if none.
+    :param search_attr: The ldap attribute to search for ``username_pattern``. Defaults to LDAP -> USERS -> USER_NAME_ATTR config value.
+    :param user_name_attr: The ldap attribute that is returned by the server to map to ``username`` in the return dictionary.
+    :param find_by_dn: Search by distinguished name.
+    :param scope: ldapsearch scope.
+    
+    :returns: List of dictionaries that take on the following form: {
+      'dn': <distinguished name of entry>,
+      'username': <ldap attribute associated with user_name_attr>
+      'first': <first name>
+      'last': <last name>
+      'email': <email>
+      'groups': <list of DNs of groups that user is a member of>
+    }
+    ``
+    """
+    if not search_attr:
+      search_attr = desktop.conf.LDAP.USERS.USER_NAME_ATTR.get()
+    if not user_name_attr:
+      user_name_attr = search_attr
+
+    user_filter = desktop.conf.LDAP.USERS.USER_FILTER.get()
+    if not user_filter.startswith('('):
+      user_filter = '(' + user_filter + ')'
+
+    # Allow wild cards on non distinguished names
+    sanitized_name = ldap.filter.escape_filter_chars(username_pattern).replace(r'\2a', r'*')
+    search_dn, user_name_filter = self._get_search_params(sanitized_name, search_attr, find_by_dn)
+    ldap_filter = '(&' + user_filter + user_name_filter + ')'
+    attrlist = ['objectClass', 'isMemberOf', 'memberOf', 'givenName', 'sn', 'mail', 'dn', user_name_attr]
+
+    ldap_result_id = self.ldap_handle.search(search_dn, scope, ldap_filter, attrlist)
+    result_type, result_data = self.ldap_handle.result(ldap_result_id)
+
+    if result_type == ldap.RES_SEARCH_RESULT:
+      return self._transform_find_user_results(result_data, user_name_attr)
+    else:
+      return []
+
+  def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+    """
+    LDAP search helper method for finding groups
+
+    :param groupname_pattern: The pattern to match ``search_attr`` against. Defaults to ``search_attr`` if none.
+    :param search_attr: The ldap attribute to search for ``groupname_pattern``. Defaults to LDAP -> GROUPS -> GROUP_NAME_ATTR config value.
+    :param group_name_attr: The ldap attribute that is returned by the server to map to ``name`` in the return dictionary.
+    :param find_by_dn: Search by distinguished name.
+    :param scope: ldapsearch scope.
+    
+    :returns: List of dictionaries that take on the following form: {
+      'dn': <distinguished name of entry>,
+      'name': <ldap attribute associated with group_name_attr>
+      'first': <first name>
+      'last': <last name>
+      'email': <email>
+      'groups': <list of DNs of groups that user is a member of>
+    }
+    """
+    if not search_attr:
+      search_attr = desktop.conf.LDAP.GROUPS.GROUP_NAME_ATTR.get()
+    if not group_name_attr:
+      group_name_attr = search_attr
+
+    group_filter = desktop.conf.LDAP.GROUPS.GROUP_FILTER.get()
+    if not group_filter.startswith('('):
+      group_filter = '(' + group_filter + ')'
+
+    # Allow wild cards on non distinguished names
+    sanitized_name = ldap.filter.escape_filter_chars(groupname_pattern).replace(r'\2a', r'*')
+    search_dn, group_name_filter = self._get_search_params(sanitized_name, search_attr, find_by_dn)
+    ldap_filter = '(&' + group_filter + group_name_filter + ')'
+    attrlist = ['objectClass', 'dn', 'memberUid', desktop.conf.LDAP.GROUPS.GROUP_MEMBER_ATTR.get(), group_name_attr]
+
+    ldap_result_id = self.ldap_handle.search(search_dn, scope, ldap_filter, attrlist)
+    result_type, result_data = self.ldap_handle.result(ldap_result_id)
+
+    if result_type == ldap.RES_SEARCH_RESULT:
+      return self._transform_find_group_results(result_data, group_name_attr)
+    else:
+      return []
+
   def _get_root_dn(self):
     """
     Returns the configured base DN (DC=desktop,DC=local).

+ 89 - 9
apps/useradmin/src/useradmin/tests.py

@@ -62,6 +62,8 @@ class LdapTestConnection(object):
   Test class which mimics the behaviour of LdapConnection (from ldap_access.py).
   It also includes functionality to fake modifications to an LDAP server.  It is designed
   as a singleton, to allow for changes to persist across discrete connections.
+
+  This class assumes uid is the user_name_attr.
   """
   def __init__(self):
     self._instance = LdapTestConnection.Data()
@@ -72,17 +74,23 @@ class LdapTestConnection(object):
   def remove_user_group_for_test(self, user, group):
     self._instance.groups[group]['members'].remove(user)
 
-  def find_users(self, username_pattern, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+  def add_posix_user_group_for_test(self, user, group):
+    self._instance.groups[group]['posix_members'].append(user)
+
+  def remove_posix_user_group_for_test(self, user, group):
+    self._instance.groups[group]['posix_members'].remove(user)
+
+  def find_users(self, username_pattern, search_attr=None, user_name_attr=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
     """ Returns info for a particular user """
     if find_by_dn:
       data = filter(lambda attrs: attrs['dn'] == username_pattern, self._instance.users.values())
     else:
-      username_pattern = username_pattern.replace('.','\\.').replace('*', '.*')
+      username_pattern = "^%s$" % username_pattern.replace('.','\\.').replace('*', '.*')
       usernames = filter(lambda username: re.match(username_pattern, username), self._instance.users.keys())
       data = [self._instance.users.get(username) for username in usernames]
     return data
 
-  def find_groups(self, groupname_pattern, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+  def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
     """ Return all groups in the system with parents and children """
     if find_by_dn:
       data = filter(lambda attrs: attrs['dn'] == groupname_pattern, self._instance.groups.values())
@@ -91,7 +99,7 @@ class LdapTestConnection(object):
         sub_data = filter(lambda attrs: attrs['dn'].endswith(data[0]['dn']), self._instance.groups.values())
         data.extend(sub_data)
     else:
-      groupname_pattern = groupname_pattern.replace('.','\\.').replace('*', '.*')
+      groupname_pattern = "^%s$" % groupname_pattern.replace('.','\\.').replace('*', '.*')
       groupnames = filter(lambda username: re.match(groupname_pattern, username), self._instance.groups.keys())
       data = [self._instance.groups.get(groupname) for groupname in groupnames]
     return data
@@ -102,21 +110,36 @@ class LdapTestConnection(object):
                     'lårry': {'dn': 'uid=lårry,ou=People,dc=example,dc=com', 'username':'lårry', 'first':'Larry', 'last':'Stooge', 'email':'larry@stooges.com', 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com', 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com']},
                     'curly': {'dn': 'uid=curly,ou=People,dc=example,dc=com', 'username':'curly', 'first':'Curly', 'last':'Stooge', 'email':'curly@stooges.com', 'groups': ['cn=TestUsers,ou=Groups,dc=example,dc=com', 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com']},
                     'rock': {'dn': 'uid=rock,ou=People,dc=example,dc=com', 'username':'rock', 'first':'rock', 'last':'man', 'email':'rockman@stooges.com', 'groups': ['cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com']},
-                    'otherguy': {'dn': 'uid=otherguy,ou=People,dc=example,dc=com', 'username':'otherguy', 'first':'Other', 'last':'Guy', 'email':'other@guy.com'}}
+                    'otherguy': {'dn': 'uid=otherguy,ou=People,dc=example,dc=com', 'username':'otherguy', 'first':'Other', 'last':'Guy', 'email':'other@guy.com'},
+                    'posix_person': {'dn': 'uid=posix_person,ou=People,dc=example,dc=com', 'username': 'posix_person', 'first': 'pos', 'last': 'ix', 'email': 'pos@ix.com'},
+                    'posix_person2': {'dn': 'uid=posix_person2,ou=People,dc=example,dc=com', 'username': 'posix_person2', 'first': 'pos', 'last': 'ix', 'email': 'pos@ix.com'}}
 
       self.groups = {'TestUsers': {
                         'dn': 'cn=TestUsers,ou=Groups,dc=example,dc=com',
                         'name':'TestUsers',
-                        'members':['uid=moe,ou=People,dc=example,dc=com','uid=lårry,ou=People,dc=example,dc=com','uid=curly,ou=People,dc=example,dc=com']},
+                        'members':['uid=moe,ou=People,dc=example,dc=com','uid=lårry,ou=People,dc=example,dc=com','uid=curly,ou=People,dc=example,dc=com'],
+                        'posix_members':[]},
                      'Test Administrators': {
                         'dn': 'cn=Test Administrators,cn=TestUsers,ou=Groups,dc=example,dc=com',
                         'name':'Test Administrators',
-                        'members':['uid=rock,ou=People,dc=example,dc=com','uid=lårry,ou=People,dc=example,dc=com','uid=curly,ou=People,dc=example,dc=com']},
+                        'members':['uid=rock,ou=People,dc=example,dc=com','uid=lårry,ou=People,dc=example,dc=com','uid=curly,ou=People,dc=example,dc=com'],
+                        'posix_members':[]},
                      'OtherGroup': {
                         'dn': 'cn=OtherGroup,cn=TestUsers,ou=Groups,dc=example,dc=com',
                         'name':'OtherGroup',
-                        'members':[]}}
-
+                        'members':[],
+                        'posix_members':[]},
+                     'PosixGroup': {
+                        'dn': 'cn=PosixGroup,ou=Groups,dc=example,dc=com',
+                        'name':'PosixGroup',
+                        'members':[],
+                        'posix_members':['posix_person','lårry']},
+                     'PosixGroup1': {
+                        'dn': 'cn=PosixGroup1,cn=PosixGroup,ou=Groups,dc=example,dc=com',
+                        'name':'PosixGroup1',
+                        'members':[],
+                        'posix_members':['posix_person2']},
+                     }
 
 
 def test_invalid_username():
@@ -540,6 +563,63 @@ def test_useradmin_ldap_group_integration():
   assert_false(LdapGroup.objects.filter(group=hue_group).exists())
   assert_true(hue_group.user_set.filter(username=hue_user.username).exists())
 
+
+def test_useradmin_ldap_posix_group_integration():
+  reset_all_users()
+  reset_all_groups()
+
+  # Set up LDAP tests to use a LdapTestConnection instead of an actual LDAP connection
+  ldap_access.CACHED_LDAP_CONN = LdapTestConnection()
+
+  # Import groups only
+  import_ldap_groups('PosixGroup', import_members=False, import_members_recursive=False, sync_users=False, import_by_dn=False)
+  test_users = Group.objects.get(name='PosixGroup')
+  assert_true(LdapGroup.objects.filter(group=test_users).exists())
+  assert_equal(test_users.user_set.all().count(), 0)
+
+  # Import all members of TestUsers
+  import_ldap_groups('PosixGroup', import_members=True, import_members_recursive=False, sync_users=True, import_by_dn=False)
+  test_users = Group.objects.get(name='PosixGroup')
+  assert_true(LdapGroup.objects.filter(group=test_users).exists())
+  assert_equal(test_users.user_set.all().count(), 2)
+
+  # Should import a group, but will only sync already-imported members
+  import_ldap_groups('Test Administrators', import_members=False, import_members_recursive=False, sync_users=True, import_by_dn=False)
+  assert_equal(User.objects.all().count(), 2, User.objects.all())
+  assert_equal(Group.objects.all().count(), 2, Group.objects.all())
+  test_admins = Group.objects.get(name='Test Administrators')
+  assert_equal(test_admins.user_set.all().count(), 1)
+  larry = User.objects.get(username='lårry')
+  assert_equal(test_admins.user_set.all()[0].username, larry.username)
+
+  # Only sync already imported
+  ldap_access.CACHED_LDAP_CONN.remove_posix_user_group_for_test('posix_person', 'PosixGroup')
+  import_ldap_groups('PosixGroup', import_members=False, import_members_recursive=False, sync_users=True, import_by_dn=False)
+  assert_equal(test_users.user_set.all().count(), 1)
+  assert_equal(User.objects.get(username='posix_person').groups.all().count(), 0)
+
+  # Import missing user
+  ldap_access.CACHED_LDAP_CONN.add_posix_user_group_for_test('posix_person', 'PosixGroup')
+  import_ldap_groups('PosixGroup', import_members=True, import_members_recursive=False, sync_users=True, import_by_dn=False)
+  assert_equal(test_users.user_set.all().count(), 2)
+  assert_equal(User.objects.get(username='posix_person').groups.all().count(), 1)
+
+  # Import all members of PosixGroup and members of subgroups
+  import_ldap_groups('PosixGroup', import_members=True, import_members_recursive=True, sync_users=True, import_by_dn=False)
+  test_users = Group.objects.get(name='PosixGroup')
+  assert_true(LdapGroup.objects.filter(group=test_users).exists())
+  assert_equal(test_users.user_set.all().count(), 3)
+
+  # Make sure Hue groups with naming collisions don't get marked as LDAP groups
+  hue_user = User.objects.create(username='otherguy', first_name='Different', last_name='Guy')
+  hue_group = Group.objects.create(name='OtherGroup')
+  hue_group.user_set.add(hue_user)
+  hue_group.save()
+  import_ldap_groups('OtherGroup', import_members=False, import_members_recursive=False, sync_users=True, import_by_dn=False)
+  assert_false(LdapGroup.objects.filter(group=hue_group).exists())
+  assert_true(hue_group.user_set.filter(username=hue_user.username).exists())
+
+
 def test_useradmin_ldap_user_integration():
   reset_all_users()
   reset_all_groups()

+ 51 - 18
apps/useradmin/src/useradmin/views.py

@@ -40,6 +40,7 @@ from django.utils.translation import ugettext as _
 from django.forms.util import ErrorList
 from django.shortcuts import redirect
 
+import desktop.conf
 from hadoop.fs.exceptions import WebHdfsException
 from useradmin.models import HuePermission, UserProfile, LdapGroup
 from useradmin.models import get_profile, get_default_user_group
@@ -463,20 +464,26 @@ def _import_ldap_users(username_pattern, sync_groups=False, import_by_dn=False):
   the distinguished name, rather than the configured username attribute.
   """
   conn = ldap_access.get_connection()
-  user_info = conn.find_users(username_pattern, import_by_dn)
+  user_info = conn.find_users(username_pattern, find_by_dn=import_by_dn)
   if not user_info:
-    LOG.warn(_("Could not get LDAP details for users with pattern %s") % username_pattern)
+    LOG.warn("Could not get LDAP details for users with pattern %s" % username_pattern)
     return None
 
+  return _import_ldap_users_info(user_info, sync_groups, import_by_dn)
+
+
+def _import_ldap_users_info(user_info, sync_groups=False, import_by_dn=False):
+  """
+  Import user_info found through ldap_access.find_users.
+  """
   imported_users = []
   for ldap_info in user_info:
     user, created = User.objects.get_or_create(username=ldap_info['username'])
     profile = get_profile(user)
     if not created and profile.creation_method == str(UserProfile.CreationMethod.HUE):
       # This is a Hue user, and shouldn't be overwritten
-      LOG.warn(_('There was a naming conflict while importing user %(username)s in pattern %(username_pattern)s') % {
-        'username': ldap_info['username'],
-        'username_pattern': username_pattern
+      LOG.warn(_('There was a naming conflict while importing user %(username)s') % {
+        'username': ldap_info['username']
       })
       return None
 
@@ -498,6 +505,7 @@ def _import_ldap_users(username_pattern, sync_groups=False, import_by_dn=False):
 
     # sync groups
     if sync_groups and 'groups' in ldap_info:
+      conn = ldap_access.get_connection()
       old_groups = set(user.groups.all())
       new_groups = set()
       # Skip if 'memberOf' or 'isMemberOf' are not set
@@ -532,10 +540,10 @@ def _import_ldap_groups(groupname_pattern, import_members=False, recursive_impor
     scope = ldap.SCOPE_BASE
   else:
     scope = ldap.SCOPE_SUBTREE
-  group_info = conn.find_groups(groupname_pattern, import_by_dn, scope)
+  group_info = conn.find_groups(groupname_pattern, find_by_dn=import_by_dn, scope=scope)
 
   if not group_info:
-    LOG.warn(_("Could not get LDAP details for group pattern %s") % groupname_pattern)
+    LOG.warn("Could not get LDAP details for group pattern %s" % groupname_pattern)
     return None
 
   groups = []
@@ -552,12 +560,14 @@ def _import_ldap_groups(groupname_pattern, import_members=False, recursive_impor
     LdapGroup.objects.get_or_create(group=group)
     group.user_set.clear()
 
-    # Find members for group and subgoups
+    # Find members and posix members for group and subgoups
     members = ldap_info['members']
+    posix_members = ldap_info['posix_members']
     if recursive_import_members:
-      sub_group_info = conn.find_groups(ldap_info['dn'], True)
+      sub_group_info = conn.find_groups(ldap_info['dn'], find_by_dn=True)
       for sub_ldap_info in sub_group_info:
         members.extend(sub_ldap_info['members'])
+        posix_members.extend(sub_ldap_info['posix_members'])
 
     # Import/fetch users
     for member in members:
@@ -575,18 +585,41 @@ def _import_ldap_groups(groupname_pattern, import_members=False, recursive_impor
           for ldap_info in user_info:
             try:
               user = User.objects.get(username=ldap_info['username'])
+              users.append(user)
             except User.DoesNotExist:
-              continue
-            users.append(user)
+              pass
 
-      if not users:
-        # There was a naming conflict, or for some other reason, we couldn't get
-        # at the user
-        continue
+      if users:
+        LOG.debug("Adding member %s represented as users (should be a single user) %s to group %s" % (str(member), str(users), str(group.name)))
+        group.user_set.add(*users)
+
+
+    # Import/fetch posix users
+    for posix_member in posix_members:
+      users = []
+
+      if import_members:
+        LOG.debug("Importing user %s" % str(posix_member))
+        # posixGroup class defines 'memberUid' to be login names,
+        # which are defined by 'uid'.
+        user_info = conn.find_users(posix_member, search_attr='uid', user_name_attr=desktop.conf.LDAP.USERS.USER_NAME_ATTR.get(), find_by_dn=False)
+        users = _import_ldap_users_info(user_info, import_by_dn=False)
+
+      elif sync_users:
+        user_info = conn.find_users(posix_member, search_attr='uid', user_name_attr=desktop.conf.LDAP.USERS.USER_NAME_ATTR.get(), find_by_dn=False)
+        if len(user_info) > 1:
+          LOG.warn('Found multiple users for member %s.' % member)
+        else:
+          for ldap_info in user_info:
+            try:
+              user = User.objects.get(username=ldap_info['username'])
+              users.append(user)
+            except User.DoesNotExist:
+              pass
 
-      LOG.debug("Adding member %s represented as users (should be a single user) %s to group %s" % (str(member), str(users), str(group.name)))
-      for user in users:
-        group.user_set.add(user)
+      if users:
+        LOG.debug("Adding member %s represented as users (should be a single user) %s to group %s" % (str(posix_member), str(users), str(group.name)))
+        group.user_set.add(*users)
 
     group.save()
     groups.append(group)