فهرست منبع

HUE-2040 [useradmin] Synchronize nested ldap groups

Split up nested group and suboordinate group handling.
Add group synchronization logic in a separate method for nested groups.
Posix groups cannot have nested groups, so if it is nested,
its users are imported and we don't go any further.
NOTE: Synchronization should on assign existing users to existing groups.
Abraham Elmahrek 11 سال پیش
والد
کامیت
5c7ef36

+ 4 - 7
apps/useradmin/src/useradmin/forms.py

@@ -165,6 +165,10 @@ class AddLdapGroupsForm(forms.Form):
                                                 help_text=_t('Create home directories for every member imported, if members are being imported.'),
                                                 initial=True,
                                                 required=False)
+  import_members_recursive = forms.BooleanField(label=_t('Import new members from all subgroups'),
+                                                help_text=_t('Import unimported or new users from the all subgroups.'),
+                                                initial=False,
+                                                required=False)
 
   def clean(self):
     cleaned_data = super(AddLdapGroupsForm, self).clean()
@@ -181,13 +185,6 @@ class AddLdapGroupsForm(forms.Form):
     return cleaned_data
 
 
-class AddLdapSuboordinateGroupsForm(AddLdapGroupsForm):
-  import_members_recursive = forms.BooleanField(label=_t('Import new members from all subgroups'),
-                                                help_text=_t('Import unimported or new users from the all subgroups.'),
-                                                initial=False,
-                                                required=False)
-
-
 class GroupEditForm(forms.ModelForm):
   """
   Form to manipulate a group.  This manages the group name and its membership.

+ 96 - 4
apps/useradmin/src/useradmin/tests.py

@@ -188,6 +188,12 @@ class LdapTestConnection(object):
                         'members':['uid=nestedguy,ou=People,dc=example,dc=com'],
                         'posix_members':[]
                       },
+                      'NestedPosixGroups': {
+                        'dn': 'cn=NestedPosixGroups,ou=Groups,dc=example,dc=com',
+                        'name':'NestedPosixGroups',
+                        'members':['cn=PosixGroup,ou=Groups,dc=example,dc=com'],
+                        'posix_members':[]
+                      },
                       'PosixGroup': {
                         'dn': 'cn=PosixGroup,ou=Groups,dc=example,dc=com',
                         'name':'PosixGroup',
@@ -564,7 +570,7 @@ def test_useradmin_ldap_user_group_membership_sync():
     settings.MIDDLEWARE_CLASSES.remove('useradmin.middleware.LdapSynchronizationMiddleware')
 
 
-def test_useradmin_ldap_group_integration():
+def test_useradmin_ldap_suboordinate_group_integration():
   reset_all_users()
   reset_all_groups()
 
@@ -664,6 +670,7 @@ def test_useradmin_ldap_nested_group_integration():
     assert_equal(test_admins.user_set.all()[0].username, larry.username)
 
     # Only sync already imported
+    assert_equal(test_users.user_set.all().count(), 3)
     ldap_access.CACHED_LDAP_CONN.remove_user_group_for_test('uid=moe,ou=People,dc=example,dc=com', 'TestUsers')
     import_ldap_groups('TestUsers', import_members=False, import_members_recursive=False, sync_users=True, import_by_dn=False)
     assert_equal(test_users.user_set.all().count(), 2)
@@ -682,13 +689,22 @@ def test_useradmin_ldap_nested_group_integration():
     assert_equal(test_users.user_set.all().count(), 3)
 
     # Nested group import
+    # First without recursive import, then with.
     import_ldap_groups('NestedGroups', import_members=True, import_members_recursive=False, sync_users=True, import_by_dn=False)
     nested_groups = Group.objects.get(name='NestedGroups')
     nested_group = Group.objects.get(name='NestedGroup')
     assert_true(LdapGroup.objects.filter(group=nested_groups).exists())
     assert_true(LdapGroup.objects.filter(group=nested_group).exists())
-    assert_equal(nested_groups.user_set.all().count(), 0)
-    assert_equal(nested_group.user_set.all().count(), 1)
+    assert_equal(nested_groups.user_set.all().count(), 0, nested_groups.user_set.all())
+    assert_equal(nested_group.user_set.all().count(), 0, nested_group.user_set.all())
+
+    import_ldap_groups('NestedGroups', import_members=True, import_members_recursive=True, sync_users=True, import_by_dn=False)
+    nested_groups = Group.objects.get(name='NestedGroups')
+    nested_group = Group.objects.get(name='NestedGroup')
+    assert_true(LdapGroup.objects.filter(group=nested_groups).exists())
+    assert_true(LdapGroup.objects.filter(group=nested_group).exists())
+    assert_equal(nested_groups.user_set.all().count(), 0, nested_groups.user_set.all())
+    assert_equal(nested_group.user_set.all().count(), 1, nested_group.user_set.all())
 
     # Make sure Hue groups with naming collisions don't get marked as LDAP groups
     hue_user = User.objects.create(username='otherguy', first_name='Different', last_name='Guy')
@@ -703,7 +719,7 @@ def test_useradmin_ldap_nested_group_integration():
       finish()
 
 
-def test_useradmin_ldap_posix_group_integration():
+def test_useradmin_ldap_suboordinate_posix_group_integration():
   reset_all_users()
   reset_all_groups()
 
@@ -768,6 +784,82 @@ def test_useradmin_ldap_posix_group_integration():
       finish()
 
 
+def test_useradmin_ldap_nested_posix_group_integration():
+  reset_all_users()
+  reset_all_groups()
+
+  reset = []
+
+  # Set up LDAP tests to use a LdapTestConnection instead of an actual LDAP connection
+  ldap_access.CACHED_LDAP_CONN = LdapTestConnection()
+
+  # Test nested groups
+  reset.append(desktop.conf.LDAP.SUBGROUPS.set_for_testing("nested"))
+
+  try:
+    # Import groups only
+    import_ldap_groups('PosixGroup', import_members=False, import_members_recursive=False, sync_users=False, import_by_dn=False)
+    test_users = Group.objects.get(name='PosixGroup')
+    assert_true(LdapGroup.objects.filter(group=test_users).exists())
+    assert_equal(test_users.user_set.all().count(), 0)
+
+    # Import all members of TestUsers
+    import_ldap_groups('PosixGroup', import_members=True, import_members_recursive=False, sync_users=True, import_by_dn=False)
+    test_users = Group.objects.get(name='PosixGroup')
+    assert_true(LdapGroup.objects.filter(group=test_users).exists())
+    assert_equal(test_users.user_set.all().count(), 2)
+
+    # Should import a group, but will only sync already-imported members
+    import_ldap_groups('Test Administrators', import_members=False, import_members_recursive=False, sync_users=True, import_by_dn=False)
+    assert_equal(User.objects.all().count(), 2, User.objects.all())
+    assert_equal(Group.objects.all().count(), 2, Group.objects.all())
+    test_admins = Group.objects.get(name='Test Administrators')
+    assert_equal(test_admins.user_set.all().count(), 1)
+    larry = User.objects.get(username='lårry')
+    assert_equal(test_admins.user_set.all()[0].username, larry.username)
+
+    # Only sync already imported
+    ldap_access.CACHED_LDAP_CONN.remove_posix_user_group_for_test('posix_person', 'PosixGroup')
+    import_ldap_groups('PosixGroup', import_members=False, import_members_recursive=False, sync_users=True, import_by_dn=False)
+    assert_equal(test_users.user_set.all().count(), 1)
+    assert_equal(User.objects.get(username='posix_person').groups.all().count(), 0)
+
+    # Import missing user
+    ldap_access.CACHED_LDAP_CONN.add_posix_user_group_for_test('posix_person', 'PosixGroup')
+    import_ldap_groups('PosixGroup', import_members=True, import_members_recursive=False, sync_users=True, import_by_dn=False)
+    assert_equal(test_users.user_set.all().count(), 2)
+    assert_equal(User.objects.get(username='posix_person').groups.all().count(), 1)
+
+    # Import all members of PosixGroup and members of subgroups (there should be no subgroups)
+    import_ldap_groups('PosixGroup', import_members=True, import_members_recursive=True, sync_users=True, import_by_dn=False)
+    test_users = Group.objects.get(name='PosixGroup')
+    assert_true(LdapGroup.objects.filter(group=test_users).exists())
+    assert_equal(test_users.user_set.all().count(), 2)
+
+    # Import all members of NestedPosixGroups and members of subgroups
+    reset_all_users()
+    reset_all_groups()
+    import_ldap_groups('NestedPosixGroups', import_members=True, import_members_recursive=True, sync_users=True, import_by_dn=False)
+    test_users = Group.objects.get(name='NestedPosixGroups')
+    assert_true(LdapGroup.objects.filter(group=test_users).exists())
+    assert_equal(test_users.user_set.all().count(), 0)
+    test_users = Group.objects.get(name='PosixGroup')
+    assert_true(LdapGroup.objects.filter(group=test_users).exists())
+    assert_equal(test_users.user_set.all().count(), 2)
+
+    # Make sure Hue groups with naming collisions don't get marked as LDAP groups
+    hue_user = User.objects.create(username='otherguy', first_name='Different', last_name='Guy')
+    hue_group = Group.objects.create(name='OtherGroup')
+    hue_group.user_set.add(hue_user)
+    hue_group.save()
+    import_ldap_groups('OtherGroup', import_members=False, import_members_recursive=False, sync_users=True, import_by_dn=False)
+    assert_false(LdapGroup.objects.filter(group=hue_group).exists())
+    assert_true(hue_group.user_set.filter(username=hue_user.username).exists())
+  finally:
+    for finish in reset:
+      finish()
+
+
 def test_useradmin_ldap_user_integration():
   done = []
   try:

+ 143 - 41
apps/useradmin/src/useradmin/views.py

@@ -40,9 +40,8 @@ from desktop.conf import LDAP
 from hadoop.fs.exceptions import WebHdfsException
 from useradmin.models import HuePermission, UserProfile, LdapGroup
 from useradmin.models import get_profile, get_default_user_group
-from useradmin.forms import SyncLdapUsersGroupsForm, AddLdapGroupsForm,AddLdapSuboordinateGroupsForm,\
-  AddLdapUsersForm, PermissionsEditForm, GroupEditForm, SuperUserChangeForm,\
-  UserChangeForm
+from useradmin.forms import SyncLdapUsersGroupsForm, AddLdapGroupsForm, AddLdapUsersForm,\
+  PermissionsEditForm, GroupEditForm, SuperUserChangeForm, UserChangeForm
 
 
 
@@ -349,18 +348,13 @@ def add_ldap_groups(request):
   if not request.user.is_superuser:
     raise PopupException(_("You must be a superuser to add another group."), error_code=401)
 
-  if desktop.conf.LDAP.SUBGROUPS.get() == 'suboordinate':
-    FormClass = AddLdapSuboordinateGroupsForm
-  else:
-    FormClass = AddLdapGroupsForm
-
   if request.method == 'POST':
-    form = FormClass(request.POST)
+    form = AddLdapGroupsForm(request.POST)
     if form.is_valid():
       groupname_pattern = form.cleaned_data['groupname_pattern']
       import_by_dn = form.cleaned_data['dn']
       import_members = form.cleaned_data['import_members']
-      import_members_recursive = form.cleaned_data.get('import_members_recursive', False)
+      import_members_recursive = form.cleaned_data['import_members_recursive']
       try:
         groups = import_ldap_groups(groupname_pattern, import_members=import_members, import_members_recursive=import_members_recursive, sync_users=True, import_by_dn=import_by_dn)
       except ldap.LDAPError, e:
@@ -374,7 +368,7 @@ def add_ldap_groups(request):
         errors.append(_('Could not get LDAP details for groups in pattern %s') % groupname_pattern)
 
   else:
-    form = FormClass()
+    form = AddLdapGroupsForm()
 
   return render('edit_group.mako', request, dict(form=form, action=request.path, ldap=True))
 
@@ -574,41 +568,135 @@ def _import_ldap_users_info(user_info, sync_groups=False, import_by_dn=False):
   return imported_users
 
 
-def _import_ldap_members(conn, group, ldap_info, count=0):
-  if count >= desktop.conf.LDAP.NESTED_MEMBERS_SEARCH_DEPTH.get():
+def _import_ldap_members(conn, group, ldap_info, count=0, max_count=1):
+  if count >= max_count:
     return None
 
-  user_set = set()
-  group_set = set()
-
   # Find all users and groups of group.
   users_info = conn.find_users_of_group(ldap_info['dn'])
   groups_info = conn.find_groups_of_group(ldap_info['dn'])
+  posix_members = ldap_info['posix_members']
+
   for user_info in users_info:
-    LOG.debug("Importing user %s" % smart_str(user_info['dn']))
+    LOG.debug("Importing user %s into group %s" % (smart_str(user_info['dn']), smart_str(group.name)))
     users = _import_ldap_users(smart_str(user_info['dn']), import_by_dn=True)
     group.user_set.add(*users)
-    user_set |= set(users)
+
   for group_info in groups_info:
     LOG.debug("Importing group %s" % smart_str(group_info['dn']))
-    group_set |= set(_import_ldap_groups(smart_str(group_info['dn']), import_by_dn=True))
+    groups = _import_ldap_groups(smart_str(group_info['dn']), import_by_dn=True)
 
     # Must find all members of subgroups
-    if len(group_set) > 1:
+    if len(groups) > 1:
       LOG.warn('Found multiple groups for member %s.' % smart_str(group_info['dn']))
     else:
-      for group in group_set:
-        new_user_set, new_group_set = _import_ldap_members(conn, group, group_info, count+1)
-        user_set |= new_user_set
-        group_set |= new_group_set
+      for group in groups:
+        _import_ldap_members(conn, group, group_info, count+1, max_count)
 
-  return user_set, group_set
+  for posix_member in posix_members:
+    LOG.debug("Importing posix user %s into group %s" % (smart_str(posix_member), smart_str(group.name)))
+    user_info = conn.find_users(posix_member, search_attr='uid', user_name_attr=desktop.conf.LDAP.USERS.USER_NAME_ATTR.get(), find_by_dn=False)
+    users = _import_ldap_users_info(user_info)
 
+    if users:
+      LOG.debug("Adding member %s represented as users (should be a single user) %s to group %s" % (str(posix_member), str(users), str(group.name)))
+      group.user_set.add(*users)
 
-def _import_ldap_groups(groupname_pattern, import_members=False, recursive_import_members=False, sync_users=True, import_by_dn=False):
+
+def _sync_ldap_members(conn, group, ldap_info, count=0, max_count=1):
+  if count >= max_count:
+    return None
+
+  # Find all users and groups of group.
+  users_info = conn.find_users_of_group(ldap_info['dn'])
+  groups_info = conn.find_groups_of_group(ldap_info['dn'])
+  posix_members = ldap_info['posix_members']
+
+  for user_info in users_info:
+    LOG.debug("Synchronizing user %s with group %s" % (smart_str(user_info['dn']), smart_str(group.name)))
+    try:
+      user = ldap_access.get_ldap_user(username=user_info['username'])
+      group.user_set.add(user)
+    except User.DoesNotExist:
+      LOG.debug("Synchronizing user %s with group %s failed. User does not exist." % (smart_str(user_info['dn']), smart_str(group.name)))
+
+  for group_info in groups_info:
+    LOG.debug("Synchronizing group %s" % smart_str(group_info['dn']))
+
+    try:
+      group = Group.objects.get(name=group_info['name'])
+      _sync_ldap_members(conn, group, group_info, count+1, max_count)
+    except Group.DoesNotExist:
+      LOG.debug("Synchronizing group %s failed. Group does not exist." % smart_str(group.name))
+
+  for posix_member in posix_members:
+    LOG.debug("Synchronizing posix user %s with group %s" % (smart_str(posix_member), smart_str(group.name)))
+    users_info = conn.find_users(posix_member, search_attr='uid', user_name_attr=desktop.conf.LDAP.USERS.USER_NAME_ATTR.get(), find_by_dn=False)
+    for user_info in users_info:
+      try:
+        user = ldap_access.get_ldap_user(username=user_info['username'])
+        group.user_set.add(user)
+      except User.DoesNotExist:
+        LOG.debug("Synchronizing posix user %s with group %s failed. User does not exist." % (smart_str(posix_member), smart_str(group.name)))
+
+
+def _import_ldap_nested_groups(groupname_pattern, import_members=False, recursive_import_members=False, sync_users=True, import_by_dn=False):
   """
   Import a group from LDAP. If import_members is true, this will also import any
-  LDAP users that exist within the group.
+  LDAP users that exist within the group. This will use nested groups logic.
+  A nested group is a group that is a member of another group.
+  e.g. CN=subtest,OU=groups,DC=exampe,DC=COM is a member of CN=test,OU=groups,DC=exampe,DC=COM
+  and they both of the object class "groupOfNames" (or some other object class for groups).
+  """
+  conn = ldap_access.get_connection()
+  if import_by_dn:
+    scope = ldap.SCOPE_BASE
+  else:
+    scope = ldap.SCOPE_SUBTREE
+  group_info = conn.find_groups(groupname_pattern, find_by_dn=import_by_dn, scope=scope)
+
+  if not group_info:
+    LOG.warn("Could not get LDAP details for group pattern %s" % groupname_pattern)
+    return None
+
+  groups = []
+  for ldap_info in group_info:
+    group, created = Group.objects.get_or_create(name=ldap_info['name'])
+    if not created and not LdapGroup.objects.filter(group=group).exists():
+      # This is a Hue group, and shouldn't be overwritten
+      LOG.warn(_('There was a naming conflict while importing group %(groupname)s in pattern %(groupname_pattern)s') % {
+        'groupname': ldap_info['name'],
+        'groupname_pattern': groupname_pattern
+      })
+      return None
+
+    LdapGroup.objects.get_or_create(group=group)
+    group.user_set.clear()
+
+    # Find members and posix members for group and subgoups
+    # Import users and groups
+    max_count = recursive_import_members and desktop.conf.LDAP.NESTED_MEMBERS_SEARCH_DEPTH.get() or 1
+
+    if import_members:
+      _import_ldap_members(conn, group, ldap_info, max_count=max_count)
+
+    # Sync users
+    if sync_users:
+      _sync_ldap_members(conn, group, ldap_info, max_count=max_count)
+
+    group.save()
+    groups.append(group)
+
+  return groups
+
+
+def _import_ldap_suboordinate_groups(groupname_pattern, import_members=False, recursive_import_members=False, sync_users=True, import_by_dn=False):
+  """
+  Import a group from LDAP. If import_members is true, this will also import any
+  LDAP users that exist within the group. This will use suboordinate group logic.
+  A suboordinate group is a group that is a subentry of another group.
+  e.g. CN=subtest,CN=test,OU=groups,DC=exampe,DC=COM is a suboordinate group of
+  CN=test,OU=groups,DC=exampe,DC=COM
   """
   conn = ldap_access.get_connection()
   if import_by_dn:
@@ -640,20 +728,15 @@ def _import_ldap_groups(groupname_pattern, import_members=False, recursive_impor
     posix_members = ldap_info['posix_members']
 
     # @TODO: Deprecate recursive_import_members as it may not be useful.
-    if desktop.conf.LDAP.SUBGROUPS.get() == 'suboordinate':
-      if import_members:
-        if recursive_import_members:
-          for sub_ldap_info in conn.find_groups(ldap_info['dn'], find_by_dn=True):
-            members += sub_ldap_info['members']
-            posix_members += sub_ldap_info['posix_members']
-
-        for member in members:
-          LOG.debug("Importing user %s" % smart_str(member))
-          group.user_set.add( *( _import_ldap_users(member, import_by_dn=True) or [] ) )
-    else:
-      # Import users and groups
-      if import_members:
-        new_users_set, new_groups_set = _import_ldap_members(conn, group, ldap_info)
+    if import_members:
+      if recursive_import_members:
+        for sub_ldap_info in conn.find_groups(ldap_info['dn'], find_by_dn=True):
+          members += sub_ldap_info['members']
+          posix_members += sub_ldap_info['posix_members']
+
+      for member in members:
+        LOG.debug("Importing user %s" % smart_str(member))
+        group.user_set.add( *( _import_ldap_users(member, import_by_dn=True) or [] ) )
 
     # Sync users
     if sync_users:
@@ -703,6 +786,25 @@ def _import_ldap_groups(groupname_pattern, import_members=False, recursive_impor
   return groups
 
 
+def _import_ldap_groups(groupname_pattern, import_members=False, recursive_import_members=False, sync_users=True, import_by_dn=False):
+  """
+  Import a group from LDAP. If import_members is true, this will also import any
+  LDAP users that exist within the group.
+  """
+  if desktop.conf.LDAP.SUBGROUPS.get().lower() == 'suboordinate':
+    return _import_ldap_suboordinate_groups(groupname_pattern=groupname_pattern,
+                                            import_members=import_members,
+                                            recursive_import_members=recursive_import_members,
+                                            sync_users=sync_users,
+                                            import_by_dn=import_by_dn)
+  else:
+    return _import_ldap_nested_groups(groupname_pattern=groupname_pattern,
+                                      import_members=import_members,
+                                      recursive_import_members=recursive_import_members,
+                                      sync_users=sync_users,
+                                      import_by_dn=import_by_dn)
+
+
 def import_ldap_users(user_pattern, sync_groups, import_by_dn):
   return _import_ldap_users(user_pattern, sync_groups=sync_groups, import_by_dn=import_by_dn)
 

+ 6 - 0
desktop/conf.dist/hue.ini

@@ -201,6 +201,12 @@
     # Use search bind authentication.
     ## search_bind_authentication=true
 
+    # Choose which kind of subgrouping to use: nested or suboordinate (deprecated).
+    ## subgroups=suboordinate
+
+    # Define the number of levels to search for nested members.
+    ## nested_members_search_depth=10
+
     [[[users]]]
 
       # Base filter for searching for users

+ 6 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -209,6 +209,12 @@
     # Use search bind authentication.
     ## search_bind_authentication=true
 
+    # Choose which kind of subgrouping to use: nested or suboordinate (deprecated).
+    ## subgroups=suboordinate
+
+    # Define the number of levels to search for nested members.
+    ## nested_members_search_depth=10
+
     [[[users]]]
 
       # Base filter for searching for users

+ 0 - 2
desktop/core/src/desktop/conf.py

@@ -470,12 +470,10 @@ LDAP = ConfigSection(
       default=False),
 
     SUBGROUPS = Config("subgroups",
-      private=True,
       help=_("Choose which kind of subgrouping to use: nested or suboordinate (deprecated)."),
       type=coerce_str_lowercase,
       default="suboordinate"),
     NESTED_MEMBERS_SEARCH_DEPTH = Config("nested_members_search_depth",
-      private=True,
       help=_("Define the number of levels to search for nested members."),
       type=int,
       default=10),