|
|
@@ -20,6 +20,7 @@ import logging
|
|
|
from nose.plugins.skip import SkipTest
|
|
|
from nose.tools import assert_equal
|
|
|
|
|
|
+from django.core.cache import cache
|
|
|
from django.contrib.auth.models import User
|
|
|
|
|
|
from desktop.auth.backend import rewrite_user
|
|
|
@@ -29,6 +30,9 @@ from hadoop.pseudo_hdfs4 import is_live_cluster
|
|
|
|
|
|
from metadata.conf import has_navigator, NAVIGATOR
|
|
|
from metadata.navigator_client import NavigatorApi
|
|
|
+from libsentry.privilege_checker import SENTRY_PRIVILEGE_CACHE_KEY,\
|
|
|
+ PrivilegeChecker
|
|
|
+from libsentry.test_privilege_checker import MockSentryApiV2
|
|
|
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
@@ -56,6 +60,21 @@ class NavigatorClientTest:
|
|
|
cls.api._root = MockedRoot()
|
|
|
|
|
|
|
|
|
+class MockSentryApiHive(object):
|
|
|
+
|
|
|
+ def __init__(self, privileges=None):
|
|
|
+ self.privileges = privileges or []
|
|
|
+
|
|
|
+
|
|
|
+ def list_sentry_roles_by_group(self, *args, **kwargs):
|
|
|
+ return [{'name': 'test', 'group': 'test'}]
|
|
|
+
|
|
|
+
|
|
|
+ def list_sentry_privileges_by_role(self, *args, **kwargs):
|
|
|
+ return self.privileges
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
class TestNavigatorClientSecure(NavigatorClientTest):
|
|
|
|
|
|
def setUp(self):
|
|
|
@@ -66,16 +85,60 @@ class TestNavigatorClientSecure(NavigatorClientTest):
|
|
|
|
|
|
|
|
|
def test_secure_results(self):
|
|
|
- records = [
|
|
|
- {u'type': u'DATABASE', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None, u'originalName': u'marketsriskcalc_work'}, {u'customProperties': None, u'deleteTime': None, u'fileSystemPath': u'hdfs://Enchilada/data/catssolprn/work/hive', u'description': None, u'params': None, u'type': u'DATABASE', u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51188932', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None, u'originalName': u'catssolprn_work'},
|
|
|
- {u'type': u'TABLE', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'parentPath': u'/finance_db', u'originalName': u'securitydataval_20150812', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
|
|
|
- {u'type': u'FIELD', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': u'/elt_db/finance', u'originalName': u'valc3'},
|
|
|
- {u'type': u'VIEW', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalName': u'dsmt_frs_bu', u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'parentPath': u'/elt2', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
|
|
|
- ]
|
|
|
-
|
|
|
- results = list(self.api._secure_results(records))
|
|
|
|
|
|
- assert_equal(len(records), len(results), results)
|
|
|
+ cache_key = SENTRY_PRIVILEGE_CACHE_KEY % {'username': self.user.username}
|
|
|
+
|
|
|
+ try:
|
|
|
+ api_v1 = MockSentryApiHive(privileges=[
|
|
|
+ {'column': '', 'grantOption': False, 'timestamp': 1478810513849, 'database': 'etl', 'action': 'SELECT', 'scope': 'DATABASE', 'table': '', 'URI': '', 'server': 'server1'},
|
|
|
+ {'column': '', 'grantOption': False, 'timestamp': 1478810422058, 'database': 'etl', 'action': 'SELECT', 'scope': 'TABLE', 'table': 'finance', 'URI': '', 'server': 'server1'},
|
|
|
+ {'column': 'col3', 'grantOption': False, 'timestamp': 1478810590335, 'database': 'etl', 'action': 'SELECT', 'scope': 'TABLE', 'table': 'finance', 'URI': '', 'server': 'server1'},
|
|
|
+ ])
|
|
|
+ api_v2 = MockSentryApiV2()
|
|
|
+ checker = PrivilegeChecker(user=self.user, api_v1=api_v1, api_v2=api_v2)
|
|
|
+
|
|
|
+ records = [
|
|
|
+ {u'type': u'DATABASE', u'originalName': u'etl', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None},
|
|
|
+ {u'type': u'TABLE', u'parentPath': u'/etl', u'originalName': u'finance', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
|
|
|
+ {u'type': u'FIELD', u'parentPath': u'/etl/finance', u'originalName': u'col1', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav'},
|
|
|
+ {u'type': u'VIEW', u'parentPath': u'/etl', u'originalName': u'finance_view', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
|
|
|
+ ]
|
|
|
+
|
|
|
+ results = list(self.api._secure_results(records, checker=checker))
|
|
|
+ assert_equal(len(records), len(results), results)
|
|
|
+
|
|
|
+ # No privilege
|
|
|
+ api_v1 = MockSentryApiHive()
|
|
|
+ checker = PrivilegeChecker(user=self.user, api_v1=api_v1, api_v2=api_v2)
|
|
|
+
|
|
|
+ records = [
|
|
|
+ {u'type': u'DATABASE', u'originalName': u'etl', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None},
|
|
|
+ {u'type': u'TABLE', u'parentPath': u'/etl', u'originalName': u'finance', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
|
|
|
+ {u'type': u'FIELD', u'parentPath': u'/etl/finance', u'originalName': u'col1', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav'},
|
|
|
+ {u'type': u'VIEW', u'parentPath': u'/etl', u'originalName': u'finance_view', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
|
|
|
+ ]
|
|
|
+
|
|
|
+ results = list(self.api._secure_results(records, checker=checker))
|
|
|
+ assert_equal(0, len(results), results)
|
|
|
+
|
|
|
+ # All privileges
|
|
|
+ api_v1 = MockSentryApiHive(privileges=[
|
|
|
+ # Table SELECT
|
|
|
+ {'column': '', 'grantOption': False, 'timestamp': 1478810422058, 'database': 'etl', 'action': 'SELECT', 'scope': 'TABLE', 'table': 'finance', 'URI': '', 'server': 'server1'},
|
|
|
+ ])
|
|
|
+ checker = PrivilegeChecker(user=self.user, api_v1=api_v1, api_v2=api_v2)
|
|
|
+
|
|
|
+ records = [
|
|
|
+ {u'type': u'DATABASE', u'originalName': u'etl', u'description': None, u'params': None, u'internalType': u'hv_database', u'sourceType': u'HIVE', u'tags': None, u'originalDescription': None, u'metaClassName': u'hv_database', u'properties': None, u'identity': u'51002517', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##3', u'sourceId': u'56850544', u'packageName': u'nav', u'parentPath': None},
|
|
|
+ {u'type': u'TABLE', u'parentPath': u'/etl', u'originalName': u'finance', u'clusteredByColNames': None, u'customProperties': None, u'owner': u'elt', u'serdeName': None, u'sourceType': u'HIVE', u'serdeLibName': u'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', u'internalType': u'hv_table', u'description': None, u'tags': None, u'originalDescription': None, u'compressed': False, u'metaClassName': u'hv_table', u'properties': None, u'identity': u'51340470', u'outputFormat': u'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1185', u'created': u'2015-08-14T00:04:01.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'lastAccessed': u'1970-01-01T00:00:00.000Z'},
|
|
|
+ {u'type': u'FIELD', u'parentPath': u'/etl/finance', u'originalName': u'col1', u'customProperties': None, u'deleteTime': None, u'description': None, u'dataType': u'string', u'internalType': u'hv_column', u'sourceType': u'HIVE', u'tags': None, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_column', u'properties': None, u'identity': u'51001004', u'firstClassParentId': u'59444965', u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##1582', u'sourceId': u'56850544', u'packageName': u'nav'},
|
|
|
+ {u'type': u'VIEW', u'parentPath': u'/etl', u'originalName': u'finance_view', u'customProperties': None, u'deleteTime': None, u'description': None, u'lastModifiedBy': None, u'internalType': u'hv_view', u'sourceType': u'HIVE', u'tags': None, u'deleted': False, u'technicalProperties': None, u'userEntity': False, u'originalDescription': None, u'metaClassName': u'hv_view', u'properties': None, u'identity': u'51012354', u'firstClassParentId': None, u'name': None, u'extractorRunId': u'845beb21b95783c4f55276a4ae38a332##394', u'created': u'2015-09-02T08:01:14.000Z', u'sourceId': u'56850544', u'lastModified': None, u'packageName': u'nav', u'queryText': u"SELECT * FROM etl.finance LIMIT 10", u'lastAccessed': u'1970-01-01T00:00:00.000Z'}
|
|
|
+ ]
|
|
|
+
|
|
|
+ results = list(self.api._secure_results(records, checker=checker))
|
|
|
+ assert_equal(2, len(results), results) # Table + its Column
|
|
|
+ finally:
|
|
|
+ cache.delete(cache_key)
|
|
|
|
|
|
|
|
|
class TestNavigatorClientTest(NavigatorClientTest):
|