瀏覽代碼

HUE-8824 [metadata] Adding missing base file

Romain 6 年之前
父節點
當前提交
60e0a48b1e

+ 16 - 0
desktop/libs/metadata/src/metadata/optimizer/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 82 - 0
desktop/libs/metadata/src/metadata/optimizer/base.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from builtins import object
+from django.utils.translation import ugettext as _
+
+from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.i18n import smart_unicode
+
+
+def get_api(request, interface):
+
+  if interface == 'optimizer':
+    from metadata.optimizer.optimizer_client import OptimizerClient
+    return OptimizerClient(request.user)
+  elif interface == 'dummy':
+    from metadata.optimizer.dummy_client import DummyClient
+    return DummyClient(user=request.user)
+  else:
+    raise PopupException(_('Optimizer connector interface not recognized: %s') % interface)
+
+
+class OptimizerApiException(Exception):
+  def __init__(self, message=None):
+    self.message = message or _('No error message, please check the logs.')
+
+  def __str__(self):
+    return str(self.message)
+
+  def __unicode__(self):
+    return smart_unicode(self.message)
+
+
+def check_privileges(view_func):
+  def decorate(*args, **kwargs):
+
+    if OPTIMIZER.APPLY_SENTRY_PERMISSIONS.get():
+      checker = get_checker(user=args[0].user)
+      action = 'SELECT'
+      objects = []
+
+      if kwargs.get('db_tables'):
+        for db_table in kwargs['db_tables']:
+          objects.append({'server': get_hive_sentry_provider(), 'db': _get_table_name(db_table)['database'], 'table': _get_table_name(db_table)['table']})
+      else:
+        objects = [{'server': get_hive_sentry_provider()}]
+        if kwargs.get('database_name'):
+          objects[0]['db'] = kwargs['database_name']
+        if kwargs.get('table_name'):
+          objects[0]['table'] = kwargs['table_name']
+
+      filtered = list(checker.filter_objects(objects, action))
+      if len(filtered) != len(objects):
+        raise MissingSentryPrivilegeException({'pre_filtering': objects, 'post_filtering': filtered, 'diff': len(objects) - len(filtered)})
+
+    return view_func(*args, **kwargs)
+  return wraps(view_func)(decorate)
+
+
+class Api(object):
+  '''
+  Base Api
+  '''
+
+  def __init__(self, user=None):
+    self.user = user
+
+  # To implement

+ 39 - 39
desktop/libs/metadata/src/metadata/optimizer/optimizer_client_tests.py

@@ -29,13 +29,13 @@ from desktop.models import uuid_default
 from useradmin.models import User
 
 from metadata.conf import OPTIMIZER, has_optimizer
-from metadata.optimizer_client import OptimizerApi
+from metadata.optimizer.optimizer_client import OptimizerClient
 
 
 LOG = logging.getLogger(__name__)
 
 
-class BaseTestOptimizerApi(object):
+class BaseTestOptimizerClient(object):
   integration = True
   UPLOADED = False
   DATABASE = 'db1'
@@ -52,25 +52,25 @@ class BaseTestOptimizerApi(object):
     grant_access("test", "test", "metadata")
     grant_access("test", "test", "optimizer")
 
-    cls.api = OptimizerApi(user=cls.user)
+    cls.api = OptimizerClient(user=cls.user)
 
-    if not BaseTestOptimizerApi.UPLOADED:
+    if not BaseTestOptimizerClient.UPLOADED:
       cls.upload()
-      BaseTestOptimizerApi.UPLOADED = True
+      BaseTestOptimizerClient.UPLOADED = True
 
 
   # Should run first
   @classmethod
   def upload(cls):
     queries = [
-        (uuid_default(), 0, "select emps.id from emps where emps.name = 'Joe' group by emps.mgr, emps.id;", BaseTestOptimizerApi.DATABASE),
-        (uuid_default(), 0, "select emps.name from emps where emps.num = 007 group by emps.state, emps.name;", BaseTestOptimizerApi.DATABASE),
-        (uuid_default(), 0, "select Part.partkey, max(Part.salary), Part.name, Part.type from %s.Part where Part.yyprice > 2095" % BaseTestOptimizerApi.DATABASE, BaseTestOptimizerApi.DATABASE),
-        (uuid_default(), 0, "select Part.partkey, Part.name, Part.mfgr FROM Part WHERE Part.name LIKE '%red';", BaseTestOptimizerApi.DATABASE),
-        (uuid_default(), 0, "select count(*) as loans from account a where a.account_state_id in (5,9);", BaseTestOptimizerApi.DATABASE),
-        (uuid_default(), 0, "select orders.key, orders.id from orders where orders.price < 9999", BaseTestOptimizerApi.DATABASE),
+        (uuid_default(), 0, "select emps.id from emps where emps.name = 'Joe' group by emps.mgr, emps.id;", BaseTestOptimizerClient.DATABASE),
+        (uuid_default(), 0, "select emps.name from emps where emps.num = 007 group by emps.state, emps.name;", BaseTestOptimizerClient.DATABASE),
+        (uuid_default(), 0, "select Part.partkey, max(Part.salary), Part.name, Part.type from %s.Part where Part.yyprice > 2095" % BaseTestOptimizerClient.DATABASE, BaseTestOptimizerClient.DATABASE),
+        (uuid_default(), 0, "select Part.partkey, Part.name, Part.mfgr FROM Part WHERE Part.name LIKE '%red';", BaseTestOptimizerClient.DATABASE),
+        (uuid_default(), 0, "select count(*) as loans from account a where a.account_state_id in (5,9);", BaseTestOptimizerClient.DATABASE),
+        (uuid_default(), 0, "select orders.key, orders.id from orders where orders.price < 9999", BaseTestOptimizerClient.DATABASE),
 
-        (uuid_default(), 0, "select x from x join y where x.a = y.a;", BaseTestOptimizerApi.DATABASE),
+        (uuid_default(), 0, "select x from x join y where x.a = y.a;", BaseTestOptimizerClient.DATABASE),
 
         # DDL
         (uuid_default(), 0, ' '.join('''CREATE TABLE `web_logs`(
@@ -105,7 +105,7 @@ class BaseTestOptimizerApi(object):
 COMMENT 'http://demo.gethue.com/ rocks!'
 PARTITIONED BY (
   `date` string)
-'''.splitlines()), BaseTestOptimizerApi.DATABASE)
+'''.splitlines()), BaseTestOptimizerClient.DATABASE)
     ]
 
     resp = cls.api.upload(data=queries, data_type='queries', source_platform='hive')
@@ -142,7 +142,7 @@ PARTITIONED BY (
     cls.user.save()
 
 
-class TestOptimizerApi(BaseTestOptimizerApi):
+class TestOptimizerClient(BaseTestOptimizerClient):
 
   def test_tenant(self):
     resp = self.api.get_tenant(cluster_id=OPTIMIZER.CLUSTER_ID.get())
@@ -159,7 +159,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
     assert_true('eid' in resp['results'][0], resp)
     assert_true('name' in resp['results'][0], resp)
 
-    database_name = BaseTestOptimizerApi.DATABASE
+    database_name = BaseTestOptimizerClient.DATABASE
     resp = self.api.top_tables(database_name=database_name)
 
     assert_true(isinstance(resp['results'], list), resp)
@@ -185,7 +185,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
     assert_true('type' in resp, resp)
     assert_true('name' in resp, resp)
 
-    resp = self.api.table_details(database_name=BaseTestOptimizerApi.DATABASE, table_name='Part')
+    resp = self.api.table_details(database_name=BaseTestOptimizerClient.DATABASE, table_name='Part')
 
     assert_true('tid' in resp, resp)
     assert_true('columnCount' in resp, resp)
@@ -194,7 +194,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
   def test_query_risk(self):
     query = 'Select * from items'
 
-    resp = self.api.query_risk(query=query, source_platform='hive', db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform='hive', db_name=BaseTestOptimizerClient.DATABASE)
 
     assert_true(len(resp) > 0, resp)
     assert_true('riskAnalysis' in resp['hints'][0], resp)
@@ -216,26 +216,26 @@ class TestOptimizerApi(BaseTestOptimizerApi):
 
 
   def test_top_filters(self):
-    resp = self.api.top_filters(db_tables=['%s.Part' % BaseTestOptimizerApi.DATABASE])
+    resp = self.api.top_filters(db_tables=['%s.Part' % BaseTestOptimizerClient.DATABASE])
 
     assert_true(len(resp['results']) > 0, resp)
 
 
   def test_top_joins(self):
-    resp = self.api.top_joins(db_tables=['%s.x' % BaseTestOptimizerApi.DATABASE])
+    resp = self.api.top_joins(db_tables=['%s.x' % BaseTestOptimizerClient.DATABASE])
 
     assert_true(len(resp['results']) > 0, resp)
 
-    assert_true(resp['results'][0]['tables'], [u'%s.x', u'%s.y' % (BaseTestOptimizerApi.DATABASE, BaseTestOptimizerApi.DATABASE)])
+    assert_true(resp['results'][0]['tables'], [u'%s.x', u'%s.y' % (BaseTestOptimizerClient.DATABASE, BaseTestOptimizerClient.DATABASE)])
     assert_true('queryIds' in resp['results'][0], resp)
     assert_true('totalTableCount' in resp['results'][0], resp)
     assert_true('totalQueryCount' in resp['results'][0], resp)
     assert_true('joinType' in resp['results'][0], resp)
-    assert_equal(resp['results'][0]['joinCols'], [{u'columns': [u'%s.x.a' % BaseTestOptimizerApi.DATABASE, u'%s.y.a' % BaseTestOptimizerApi.DATABASE]}])
+    assert_equal(resp['results'][0]['joinCols'], [{u'columns': [u'%s.x.a' % BaseTestOptimizerClient.DATABASE, u'%s.y.a' % BaseTestOptimizerClient.DATABASE]}])
 
 
   def test_top_aggs(self):
-    resp = self.api.top_aggs(db_tables=['%s.Part' % BaseTestOptimizerApi.DATABASE])
+    resp = self.api.top_aggs(db_tables=['%s.Part' % BaseTestOptimizerClient.DATABASE])
 
     assert_true(len(resp['results']) > 0, resp)
 
@@ -248,7 +248,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
 
 
   def test_top_columns(self):
-    resp = self.api.top_columns(db_tables=['%s.Part' % BaseTestOptimizerApi.DATABASE])
+    resp = self.api.top_columns(db_tables=['%s.Part' % BaseTestOptimizerClient.DATABASE])
 
     assert_true('orderbyColumns' in resp, resp)
     assert_true('selectColumns' in resp, resp)
@@ -285,7 +285,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
 
 
 
-class TestOptimizerRiskApi(BaseTestOptimizerApi):
+class TestOptimizerRiskApi(BaseTestOptimizerClient):
 
   def test_risk_10_views(self):
     source_platform = 'hive'
@@ -317,7 +317,7 @@ FROM
                                     FROM sample_01) t1) t2) t3) t4) t5) t6) t7) t8) t9) t10) t11) t12
 '''
 
-    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
     _assert_risks(['>=10 Inline Views present in query.'], resp['hints'])
 
 
@@ -326,7 +326,7 @@ FROM
     query = '''SELECT ID, NAME, AMOUNT, DATE FROM CUSTOMERS, ORDERS
 '''
 
-    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
     _assert_risks(['Cartesian or CROSS join found.'], resp['hints'])
 
     source_platform = 'hive'
@@ -340,7 +340,7 @@ WHERE s07.salary > 88
 ORDER BY s07.salary DESC
 '''
 
-    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
     _assert_risks(['Cartesian or CROSS join found.'], resp['hints'])
 
 
@@ -378,7 +378,7 @@ ORDER BY s07.salary DESC
 LIMIT 1000
 '''
 
-    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
     _assert_risks(['>=5 table joins or >=10 join conditions found.'], resp['hints'])
 
 
@@ -401,11 +401,11 @@ GROUP BY account_client,
          limit 5
 '''
 
-    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
     _assert_risks(['>=10 columns present in GROUP BY list.'], resp['hints'])
 
-    assert_equal(resp['noDDL'], ['%s.transactions' % BaseTestOptimizerApi.DATABASE])
-    assert_equal(resp['noStats'], ['%s.transactions' % BaseTestOptimizerApi.DATABASE])
+    assert_equal(resp['noDDL'], ['%s.transactions' % BaseTestOptimizerClient.DATABASE])
+    assert_equal(resp['noStats'], ['%s.transactions' % BaseTestOptimizerClient.DATABASE])
 
 
   def test_risk_cross_join_false_positive(self):
@@ -422,27 +422,27 @@ ORDER BY s07.salary DESC
 LIMIT 1000
 '''
 
-    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
     _assert_risks(['Cartesian or CROSS join found.'], resp['hints'], present=False)
 
 
     source_platform = 'hive'
     query = '''select x from x join y where x.a = y.a'''
 
-    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
+    resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
     _assert_risks(['Cartesian or CROSS join found.'], resp['hints'], present=False)
 
 
   def test_risk_no_filter_on_any_partitioned_column(self):
     source_platform = 'hive'
     query = '''SELECT * FROM web_logs'''
-    db_name = BaseTestOptimizerApi.DATABASE
+    db_name = BaseTestOptimizerClient.DATABASE
 
     resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=db_name)
     _assert_risks(['Query on partitioned table is missing filters on partioning columns.'], resp['hints'])
 
     assert_false(resp['noDDL'], resp) # DDL was uploaded already
-    assert_equal(resp['noStats'], ['%s.web_logs' % BaseTestOptimizerApi.DATABASE])
+    assert_equal(resp['noStats'], ['%s.web_logs' % BaseTestOptimizerClient.DATABASE])
 
 
     source_platform = 'hive'
@@ -475,14 +475,14 @@ LIMIT 1000
   def test_risk_listing_all_risk_tables_all_the_time(self):
     source_platform = 'hive'
     query = '''SELECT * FROM web_logs JOIN a ON web_logs.id = a.id LIMIT 100'''
-    db_name = BaseTestOptimizerApi.DATABASE
+    db_name = BaseTestOptimizerClient.DATABASE
 
     resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=db_name)
     _assert_risks(['Query on partitioned table is missing filters on partioning columns.'], resp['hints'])
 
-    assert_equal([suggestion for suggestion in resp['hints'] if suggestion['riskId'] == 22][0]['riskTables'], ['%s.web_logs' % BaseTestOptimizerApi.DATABASE])
-    assert_equal(resp['noDDL'], ['%s.a' % BaseTestOptimizerApi.DATABASE])
-    assert_equal(resp['noStats'], ['%s.a' % BaseTestOptimizerApi.DATABASE, '%s.web_logs' % BaseTestOptimizerApi.DATABASE])
+    assert_equal([suggestion for suggestion in resp['hints'] if suggestion['riskId'] == 22][0]['riskTables'], ['%s.web_logs' % BaseTestOptimizerClient.DATABASE])
+    assert_equal(resp['noDDL'], ['%s.a' % BaseTestOptimizerClient.DATABASE])
+    assert_equal(resp['noStats'], ['%s.a' % BaseTestOptimizerClient.DATABASE, '%s.web_logs' % BaseTestOptimizerClient.DATABASE])
 
 
 def _assert_risks(risks, suggestions, present=True):