Pārlūkot izejas kodu

HUE-5917 [metadata] Add Django caching of security permissions

Romain Rigaux 8 gadi atpakaļ
vecāks
revīzija
e704615

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -1382,6 +1382,9 @@
   # Sentry configuration directory, where sentry-site.xml is located.
   ## sentry_conf_dir=/etc/sentry/conf
 
+  # Number of seconds when the privilege list of a user is cached.
+  ## privilege_checker_caching=30
+
 
 ###########################################################################
 # Settings to configure the ZooKeeper Lib

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1384,6 +1384,9 @@
   # Sentry configuration directory, where sentry-site.xml is located.
   ## sentry_conf_dir=/etc/sentry/conf
 
+  # Number of seconds when the privilege list of a user is cached.
+  ## privilege_checker_caching=30
+
 
 ###########################################################################
 # Settings to configure the ZooKeeper Lib

+ 7 - 0
desktop/libs/libsentry/src/libsentry/conf.py

@@ -44,6 +44,13 @@ SENTRY_CONF_DIR = Config(
   default=os.environ.get("SENTRY_CONF_DIR", '/etc/sentry/conf')
 )
 
+PRIVILEGE_CHECKER_CACHING=Config(
+  key='privilege_checker_caching',
+  help=_t('Number of seconds when the privilege list of a user is cached.'),
+  type=int,
+  default=30,
+)
+
 
 def is_enabled():
   from hadoop import cluster # Avoid dependencies conflicts

+ 1 - 0
desktop/libs/libsentry/src/libsentry/privilege_checker.py

@@ -47,6 +47,7 @@ SENTRY_OBJECTS = (
 )
 
 SENTRY_PRIVILEGE_KEY = 'SENTRY_PRIVILEGE'
+SENTRY_PRIVILEGE_CACHE_KEY = 'checker-%(username)s'
 
 
 class PrivilegeChecker(object):

+ 12 - 3
desktop/libs/metadata/src/metadata/navigator_client.py

@@ -22,11 +22,14 @@ import re
 
 from itertools import islice
 
+from django.core.cache import cache
+
 from desktop.lib.rest import resource
 from desktop.lib.rest.http_client import HttpClient, RestException
 
 from hadoop.conf import HDFS_CLUSTERS
-from libsentry.privilege_checker import PrivilegeChecker
+from libsentry.conf import PRIVILEGE_CHECKER_CACHING
+from libsentry.privilege_checker import PrivilegeChecker, SENTRY_PRIVILEGE_CACHE_KEY
 from libsentry.sentry_site import get_hive_sentry_provider
 
 from metadata.conf import NAVIGATOR, get_navigator_auth_password, get_navigator_auth_username
@@ -222,9 +225,15 @@ class NavigatorApi(object):
       raise NavigatorApiException(msg)
 
 
-  def _secure_results(self, results):
+  def _secure_results(self, results, checker=None):
     if NAVIGATOR.APPLY_SENTRY_PERMISSIONS.get():
-      checker = PrivilegeChecker(user=self.user)
+
+      cache_key = SENTRY_PRIVILEGE_CACHE_KEY % {'username': self.user.username}
+      checker = checker or cache.get(cache_key)
+      if not checker:
+        checker = PrivilegeChecker(user=self.user)
+        cache.set(cache_key, checker, PRIVILEGE_CHECKER_CACHING.get())
+
       action = 'SELECT'
 
       def getkey(result):

+ 72 - 9
desktop/libs/metadata/src/metadata/navigator_client_tests.py

@@ -20,6 +20,7 @@ import logging
 from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal
 
+from django.core.cache import cache
 from django.contrib.auth.models import User
 
 from desktop.auth.backend import rewrite_user
@@ -29,6 +30,9 @@ from hadoop.pseudo_hdfs4 import is_live_cluster
 
 from metadata.conf import has_navigator, NAVIGATOR
 from metadata.navigator_client import NavigatorApi
+from libsentry.privilege_checker import SENTRY_PRIVILEGE_CACHE_KEY,\
+  PrivilegeChecker
+from libsentry.test_privilege_checker import MockSentryApiV2
 
 
 LOG = logging.getLogger(__name__)
@@ -56,6 +60,21 @@ class NavigatorClientTest:
     cls.api._root = MockedRoot()
 
 
+class MockSentryApiHive(object):
+
+  def __init__(self, privileges=None):
+    self.privileges = privileges or []
+
+
+  def list_sentry_roles_by_group(self, *args, **kwargs):
+    return [{'name': 'test', 'group': 'test'}]
+
+
+  def list_sentry_privileges_by_role(self, *args, **kwargs):
+    return self.privileges
+
+
+
 class TestNavigatorClientSecure(NavigatorClientTest):
 
   def setUp(self):
@@ -66,16 +85,60 @@ class TestNavigatorClientSecure(NavigatorClientTest):
 
 
   def test_secure_results(self):
-    records = [
-      {u'type': u'DATABASE', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None, u'originalName': u'marketsriskcalc_work'}, {u'customProperties': None, u'deleteTime': None, u'fileSystemPath': u'hdfs://Enchilada/data/catssolprn/work/hive', u'description': None, u'params': None, u'type': u'DATABASE', u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51188932', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None, u'originalName': u'catssolprn_work'},
-      {u'type': u'TABLE', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'parentPath': u'/finance_db', u'originalName': u'securitydataval_20150812', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
-      {u'type': u'FIELD', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': u'/elt_db/finance', u'originalName': u'valc3'},
-      {u'type': u'VIEW', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalName': u'dsmt_frs_bu', u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'parentPath': u'/elt2', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
-    ]
-
-    results = list(self.api._secure_results(records))
 
-    assert_equal(len(records), len(results), results)
+    cache_key = SENTRY_PRIVILEGE_CACHE_KEY % {'username': self.user.username}
+
+    try:
+      api_v1 = MockSentryApiHive(privileges=[
+        {'column': '', 'grantOption': False, 'timestamp': 1478810513849, 'database': 'etl', 'action': 'SELECT', 'scope': 'DATABASE', 'table': '', 'URI': '', 'server': 'server1'},
+        {'column': '', 'grantOption': False, 'timestamp': 1478810422058, 'database': 'etl', 'action': 'SELECT', 'scope': 'TABLE', 'table': 'finance', 'URI': '', 'server': 'server1'},
+        {'column': 'col3', 'grantOption': False, 'timestamp': 1478810590335, 'database': 'etl', 'action': 'SELECT', 'scope': 'TABLE', 'table': 'finance', 'URI': '', 'server': 'server1'},
+      ])
+      api_v2 = MockSentryApiV2()
+      checker = PrivilegeChecker(user=self.user, api_v1=api_v1, api_v2=api_v2)
+
+      records = [
+        {u'type': u'DATABASE', u'originalName': u'etl', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None},
+        {u'type': u'TABLE', u'parentPath': u'/etl', u'originalName': u'finance', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
+        {u'type': u'FIELD', u'parentPath': u'/etl/finance', u'originalName': u'col1', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav'},
+        {u'type': u'VIEW', u'parentPath': u'/etl', u'originalName': u'finance_view', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
+      ]
+
+      results = list(self.api._secure_results(records, checker=checker))
+      assert_equal(len(records), len(results), results)
+
+      # No privilege
+      api_v1 = MockSentryApiHive()
+      checker = PrivilegeChecker(user=self.user, api_v1=api_v1, api_v2=api_v2)
+
+      records = [
+        {u'type': u'DATABASE', u'originalName': u'etl', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None},
+        {u'type': u'TABLE', u'parentPath': u'/etl', u'originalName': u'finance', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
+        {u'type': u'FIELD', u'parentPath': u'/etl/finance', u'originalName': u'col1', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav'},
+        {u'type': u'VIEW', u'parentPath': u'/etl', u'originalName': u'finance_view', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
+      ]
+
+      results = list(self.api._secure_results(records, checker=checker))
+      assert_equal(0, len(results), results)
+
+      # All privileges
+      api_v1 = MockSentryApiHive(privileges=[
+        # Table SELECT
+        {'column': '', 'grantOption': False, 'timestamp': 1478810422058, 'database': 'etl', 'action': 'SELECT', 'scope': 'TABLE', 'table': 'finance', 'URI': '', 'server': 'server1'},
+      ])
+      checker = PrivilegeChecker(user=self.user, api_v1=api_v1, api_v2=api_v2)
+
+      records = [
+        {u'type': u'DATABASE', u'originalName': u'etl', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None},
+        {u'type': u'TABLE', u'parentPath': u'/etl', u'originalName': u'finance', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
+        {u'type': u'FIELD', u'parentPath': u'/etl/finance', u'originalName': u'col1', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav'},
+        {u'type': u'VIEW', u'parentPath': u'/etl', u'originalName': u'finance_view', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
+      ]
+
+      results = list(self.api._secure_results(records, checker=checker))
+      assert_equal(2, len(results), results) # Table + its Column
+    finally:
+      cache.delete(cache_key)
 
 
 class TestNavigatorClientTest(NavigatorClientTest):