Browse Source

HUE-6249 [metadata] Support Sentry filtering on query hints and popular values

Romain Rigaux 8 years ago
parent
commit
30aa745

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -1637,6 +1637,10 @@
     # The email of the Optimizer account you want to associate with the Product.
     ## email=hue@gethue.com
 
+    # Perform Sentry privilege filtering.
+    # Default to true automatically if the cluster is secure.
+    ## apply_sentry_permissions=False
+
     # In secure mode (HTTPS), if Optimizer SSL certificates have to be verified against certificate authority.
     ## ssl_cert_ca_verify=True
 

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1641,6 +1641,10 @@
     # The email of the Optimizer account you want to associate with the Product.
     ## email=hue@gethue.com
 
+    # Perform Sentry privilege filtering.
+    # Default to true automatically if the cluster is secure.
+    ## apply_sentry_permissions=False
+
     # In secure mode (HTTPS), if Optimizer SSL certificates have to be verified against certificate authority.
     ## ssl_cert_ca_verify=True
 

+ 20 - 0
desktop/libs/libsentry/src/libsentry/privilege_checker.py

@@ -19,8 +19,11 @@ import logging
 
 from collections import defaultdict
 
+from django.core.cache import cache
+
 from libsentry.api import get_api as get_api_v1
 from libsentry.api2 import get_api as get_api_v2
+from libsentry.conf import PRIVILEGE_CHECKER_CACHING
 from libsentry.sentry_site import get_hive_sentry_provider
 
 
@@ -51,6 +54,23 @@ SENTRY_PRIVILEGE_KEY = 'SENTRY_PRIVILEGE'
 SENTRY_PRIVILEGE_CACHE_KEY = 'checker-%(username)s'
 
 
+class MissingSentryPrivilegeException(Exception):
+  def __init__(self, objects=None):
+    self.objects = objects
+
+  def __str__(self):
+    return str(self.objects)
+
+
+def get_checker(user, checker=None):
+  cache_key = SENTRY_PRIVILEGE_CACHE_KEY % {'username': user.username}
+  checker = checker or cache.get(cache_key)
+  if not checker:
+    checker = PrivilegeChecker(user=user)
+    cache.set(cache_key, checker, PRIVILEGE_CHECKER_CACHING.get())
+  return checker
+
+
 class PrivilegeChecker(object):
   """
   Given a user, checks and applies Sentry privilege and authorization rules against Sentry objects

+ 15 - 8
desktop/libs/metadata/src/metadata/conf.py

@@ -60,6 +60,14 @@ def has_navigator(user):
       and (user.is_superuser or user.has_hue_permission(action="access", app=DJANGO_APPS[0]))
 
 
+def get_security_default():
+  '''Get default security value from Hadoop'''
+  from hadoop import cluster # Avoid dependencies conflicts
+  cluster = cluster.get_cluster_conf_for_job_submission()
+
+  return cluster.SECURITY_ENABLED.get()
+
+
 OPTIMIZER = ConfigSection(
   key='optimizer',
   help=_t("""Configuration options for Optimizer API"""),
@@ -103,6 +111,13 @@ OPTIMIZER = ConfigSection(
       type=coerce_password_from_script,
       default=None),
 
+    APPLY_SENTRY_PERMISSIONS = Config(
+      key="apply_sentry_permissions",
+      help=_t("Perform Sentry privilege filtering. Default to true automatically if the cluster is secure."),
+      dynamic_default=get_security_default,
+      type=coerce_bool
+    ),
+
     EMAIL=Config(
       key="email",
       help=_t("The email of the Optimizer account you want to associate with the Product."),
@@ -168,14 +183,6 @@ def get_navigator_saml_password():
   return NAVIGATOR.AUTH_SAML_PASSWORD_SCRIPT.get()
 
 
-def get_security_default():
-  '''Get default security value from Hadoop'''
-  from hadoop import cluster # Avoid dependencies conflicts
-  cluster = cluster.get_cluster_conf_for_job_submission()
-
-  return cluster.SECURITY_ENABLED.get()
-
-
 NAVIGATOR = ConfigSection(
   key='navigator',
   help=_t("""Configuration options for Navigator API"""),

+ 2 - 10
desktop/libs/metadata/src/metadata/navigator_client.py

@@ -22,7 +22,6 @@ import re
 
 from itertools import islice
 
-from django.core.cache import cache
 from django.utils.translation import ugettext as _
 
 from desktop.lib.i18n import smart_unicode
@@ -31,8 +30,7 @@ from desktop.lib.rest.unsecure_http_client import UnsecureHttpClient
 from desktop.lib.rest.http_client import RestException
 
 from hadoop.conf import HDFS_CLUSTERS
-from libsentry.conf import PRIVILEGE_CHECKER_CACHING
-from libsentry.privilege_checker import PrivilegeChecker, SENTRY_PRIVILEGE_CACHE_KEY
+from libsentry.privilege_checker import get_checker
 from libsentry.sentry_site import get_hive_sentry_provider
 
 from metadata.conf import NAVIGATOR, get_navigator_auth_password, get_navigator_auth_username
@@ -299,13 +297,7 @@ class NavigatorApi(object):
 
   def _secure_results(self, results, checker=None):
     if NAVIGATOR.APPLY_SENTRY_PERMISSIONS.get():
-
-      cache_key = SENTRY_PRIVILEGE_CACHE_KEY % {'username': self.user.username}
-      checker = checker or cache.get(cache_key)
-      if not checker:
-        checker = PrivilegeChecker(user=self.user)
-        cache.set(cache_key, checker, PRIVILEGE_CHECKER_CACHING.get())
-
+      checker = get_checker(self.user, checker)
       action = 'SELECT'
 
       def getkey(result):

+ 2 - 22
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -26,12 +26,10 @@ from django.views.decorators.http import require_POST
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.i18n import force_unicode
 from desktop.models import Document2
-from libsentry.privilege_checker import PrivilegeChecker
 from notebook.api import _get_statement
 from notebook.models import Notebook
 
-from metadata.conf import NAVIGATOR
-from metadata.optimizer_client import OptimizerApi, NavOptException
+from metadata.optimizer_client import OptimizerApi, NavOptException, _get_table_name
 
 
 LOG = logging.getLogger(__name__)
@@ -94,7 +92,7 @@ def top_tables(request):
   database = request.POST.get('database', 'default')
   limit = request.POST.get('len', 1000)
 
-  api = OptimizerApi()
+  api = OptimizerApi(user=request.user)
   data = api.top_tables(database_name=database, page_size=limit)
 
   tables = [{
@@ -109,15 +107,6 @@ def top_tables(request):
     } for table in data['results']
   ]
 
-  if NAVIGATOR.APPLY_SENTRY_PERMISSIONS.get():
-    checker = PrivilegeChecker(user=request.user)
-    action = 'SELECT'
-
-    for table in tables:
-      paths = _get_table_name(table['name'])
-      table.update({u'db': paths['database'], u'table': paths['table'], u'column': None, u'server': u'server1'})
-    tables = list(checker.filter_objects(tables, action)) #, getkey=getkey)
-
   response['top_tables'] = tables
   response['status'] = 0
 
@@ -431,12 +420,3 @@ def upload_status(request):
   response['status'] = 0
 
   return JsonResponse(response)
-
-
-def _get_table_name(path):
-  if '.' in path:
-    database, table = path.split('.', 1)
-  else:
-    database, table = 'default', path
-
-  return {'database': database, 'table': table}

+ 31 - 2
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -30,6 +30,8 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib import export_csvxls
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.rest.http_client import RestException
+from libsentry.sentry_site import get_hive_sentry_provider
+from libsentry.privilege_checker import get_checker, MissingSentryPrivilegeException
 from navoptapi.api_lib import ApiLib
 
 from metadata.conf import OPTIMIZER, get_optimizer_url
@@ -54,7 +56,8 @@ class NavOptException(Exception):
 
 class OptimizerApi(object):
 
-  def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
+  def __init__(self, user=None, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
+    self.user = user
     self._api_url = (api_url or get_optimizer_url()).strip('/')
     self._email = OPTIMIZER.EMAIL.get()
     self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
@@ -159,8 +162,26 @@ class OptimizerApi(object):
 
 
   def top_tables(self, workfloadId=None, database_name='default', page_size=1000, startingToken=None):
-    return self._call('getTopTables', {'tenant' : self._product_name, 'dbName': database_name.lower(), 'pageSize': page_size, startingToken: None})
+    if OPTIMIZER.APPLY_SENTRY_PERMISSIONS.get():
+      checker = get_checker(user=self.user)
+      action = 'SELECT'
+      objects = [{'server': get_hive_sentry_provider(), 'db': database_name}]
+      if not checker.filter_objects(objects, action):
+        raise MissingSentryPrivilegeException(objects)
 
+    data = self._call('getTopTables', {'tenant' : self._product_name, 'dbName': database_name.lower(), 'pageSize': page_size, startingToken: None})
+
+    if OPTIMIZER.APPLY_SENTRY_PERMISSIONS.get():
+      checker = get_checker(user=self.user)
+      action = 'SELECT'
+
+      def getkey(table):
+        names = _get_table_name(table['name']),
+        return {'server': get_hive_sentry_provider(), 'db': names['database'], 'table': names['table']}
+
+      data['results'] = checker.filter_objects(data['results'], action, key=getkey)
+
+    return data
 
   def table_details(self, database_name, table_name, page_size=100, startingToken=None):
     return self._call('getTablesDetail', {'tenant' : self._product_name, 'dbName': database_name.lower(), 'tableName': table_name.lower(), 'pageSize': page_size, startingToken: None})
@@ -263,3 +284,11 @@ def OptimizerQueryDataAdapter(data):
     rows = ([str(uuid.uuid4()), 0.0, q, 'default'] for q in data)
 
   yield headers, rows
+
+def _get_table_name(path):
+  if '.' in path:
+    database, table = path.split('.', 1)
+  else:
+    database, table = 'default', path
+
+  return {'database': database, 'table': table}