|
|
@@ -29,13 +29,13 @@ from desktop.models import uuid_default
|
|
|
from useradmin.models import User
|
|
|
|
|
|
from metadata.conf import OPTIMIZER, has_optimizer
|
|
|
-from metadata.optimizer_client import OptimizerApi
|
|
|
+from metadata.optimizer.optimizer_client import OptimizerClient
|
|
|
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
-class BaseTestOptimizerApi(object):
|
|
|
+class BaseTestOptimizerClient(object):
|
|
|
integration = True
|
|
|
UPLOADED = False
|
|
|
DATABASE = 'db1'
|
|
|
@@ -52,25 +52,25 @@ class BaseTestOptimizerApi(object):
|
|
|
grant_access("test", "test", "metadata")
|
|
|
grant_access("test", "test", "optimizer")
|
|
|
|
|
|
- cls.api = OptimizerApi(user=cls.user)
|
|
|
+ cls.api = OptimizerClient(user=cls.user)
|
|
|
|
|
|
- if not BaseTestOptimizerApi.UPLOADED:
|
|
|
+ if not BaseTestOptimizerClient.UPLOADED:
|
|
|
cls.upload()
|
|
|
- BaseTestOptimizerApi.UPLOADED = True
|
|
|
+ BaseTestOptimizerClient.UPLOADED = True
|
|
|
|
|
|
|
|
|
# Should run first
|
|
|
@classmethod
|
|
|
def upload(cls):
|
|
|
queries = [
|
|
|
- (uuid_default(), 0, "select emps.id from emps where emps.name = 'Joe' group by emps.mgr, emps.id;", BaseTestOptimizerApi.DATABASE),
|
|
|
- (uuid_default(), 0, "select emps.name from emps where emps.num = 007 group by emps.state, emps.name;", BaseTestOptimizerApi.DATABASE),
|
|
|
- (uuid_default(), 0, "select Part.partkey, max(Part.salary), Part.name, Part.type from %s.Part where Part.yyprice > 2095" % BaseTestOptimizerApi.DATABASE, BaseTestOptimizerApi.DATABASE),
|
|
|
- (uuid_default(), 0, "select Part.partkey, Part.name, Part.mfgr FROM Part WHERE Part.name LIKE '%red';", BaseTestOptimizerApi.DATABASE),
|
|
|
- (uuid_default(), 0, "select count(*) as loans from account a where a.account_state_id in (5,9);", BaseTestOptimizerApi.DATABASE),
|
|
|
- (uuid_default(), 0, "select orders.key, orders.id from orders where orders.price < 9999", BaseTestOptimizerApi.DATABASE),
|
|
|
+ (uuid_default(), 0, "select emps.id from emps where emps.name = 'Joe' group by emps.mgr, emps.id;", BaseTestOptimizerClient.DATABASE),
|
|
|
+ (uuid_default(), 0, "select emps.name from emps where emps.num = 007 group by emps.state, emps.name;", BaseTestOptimizerClient.DATABASE),
|
|
|
+ (uuid_default(), 0, "select Part.partkey, max(Part.salary), Part.name, Part.type from %s.Part where Part.yyprice > 2095" % BaseTestOptimizerClient.DATABASE, BaseTestOptimizerClient.DATABASE),
|
|
|
+ (uuid_default(), 0, "select Part.partkey, Part.name, Part.mfgr FROM Part WHERE Part.name LIKE '%red';", BaseTestOptimizerClient.DATABASE),
|
|
|
+ (uuid_default(), 0, "select count(*) as loans from account a where a.account_state_id in (5,9);", BaseTestOptimizerClient.DATABASE),
|
|
|
+ (uuid_default(), 0, "select orders.key, orders.id from orders where orders.price < 9999", BaseTestOptimizerClient.DATABASE),
|
|
|
|
|
|
- (uuid_default(), 0, "select x from x join y where x.a = y.a;", BaseTestOptimizerApi.DATABASE),
|
|
|
+ (uuid_default(), 0, "select x from x join y where x.a = y.a;", BaseTestOptimizerClient.DATABASE),
|
|
|
|
|
|
# DDL
|
|
|
(uuid_default(), 0, ' '.join('''CREATE TABLE `web_logs`(
|
|
|
@@ -105,7 +105,7 @@ class BaseTestOptimizerApi(object):
|
|
|
COMMENT 'http://demo.gethue.com/ rocks!'
|
|
|
PARTITIONED BY (
|
|
|
`date` string)
|
|
|
-'''.splitlines()), BaseTestOptimizerApi.DATABASE)
|
|
|
+'''.splitlines()), BaseTestOptimizerClient.DATABASE)
|
|
|
]
|
|
|
|
|
|
resp = cls.api.upload(data=queries, data_type='queries', source_platform='hive')
|
|
|
@@ -142,7 +142,7 @@ PARTITIONED BY (
|
|
|
cls.user.save()
|
|
|
|
|
|
|
|
|
-class TestOptimizerApi(BaseTestOptimizerApi):
|
|
|
+class TestOptimizerClient(BaseTestOptimizerClient):
|
|
|
|
|
|
def test_tenant(self):
|
|
|
resp = self.api.get_tenant(cluster_id=OPTIMIZER.CLUSTER_ID.get())
|
|
|
@@ -159,7 +159,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
|
|
|
assert_true('eid' in resp['results'][0], resp)
|
|
|
assert_true('name' in resp['results'][0], resp)
|
|
|
|
|
|
- database_name = BaseTestOptimizerApi.DATABASE
|
|
|
+ database_name = BaseTestOptimizerClient.DATABASE
|
|
|
resp = self.api.top_tables(database_name=database_name)
|
|
|
|
|
|
assert_true(isinstance(resp['results'], list), resp)
|
|
|
@@ -185,7 +185,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
|
|
|
assert_true('type' in resp, resp)
|
|
|
assert_true('name' in resp, resp)
|
|
|
|
|
|
- resp = self.api.table_details(database_name=BaseTestOptimizerApi.DATABASE, table_name='Part')
|
|
|
+ resp = self.api.table_details(database_name=BaseTestOptimizerClient.DATABASE, table_name='Part')
|
|
|
|
|
|
assert_true('tid' in resp, resp)
|
|
|
assert_true('columnCount' in resp, resp)
|
|
|
@@ -194,7 +194,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
|
|
|
def test_query_risk(self):
|
|
|
query = 'Select * from items'
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform='hive', db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform='hive', db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
|
|
|
assert_true(len(resp) > 0, resp)
|
|
|
assert_true('riskAnalysis' in resp['hints'][0], resp)
|
|
|
@@ -216,26 +216,26 @@ class TestOptimizerApi(BaseTestOptimizerApi):
|
|
|
|
|
|
|
|
|
def test_top_filters(self):
|
|
|
- resp = self.api.top_filters(db_tables=['%s.Part' % BaseTestOptimizerApi.DATABASE])
|
|
|
+ resp = self.api.top_filters(db_tables=['%s.Part' % BaseTestOptimizerClient.DATABASE])
|
|
|
|
|
|
assert_true(len(resp['results']) > 0, resp)
|
|
|
|
|
|
|
|
|
def test_top_joins(self):
|
|
|
- resp = self.api.top_joins(db_tables=['%s.x' % BaseTestOptimizerApi.DATABASE])
|
|
|
+ resp = self.api.top_joins(db_tables=['%s.x' % BaseTestOptimizerClient.DATABASE])
|
|
|
|
|
|
assert_true(len(resp['results']) > 0, resp)
|
|
|
|
|
|
- assert_true(resp['results'][0]['tables'], [u'%s.x', u'%s.y' % (BaseTestOptimizerApi.DATABASE, BaseTestOptimizerApi.DATABASE)])
|
|
|
+ assert_true(resp['results'][0]['tables'], [u'%s.x', u'%s.y' % (BaseTestOptimizerClient.DATABASE, BaseTestOptimizerClient.DATABASE)])
|
|
|
assert_true('queryIds' in resp['results'][0], resp)
|
|
|
assert_true('totalTableCount' in resp['results'][0], resp)
|
|
|
assert_true('totalQueryCount' in resp['results'][0], resp)
|
|
|
assert_true('joinType' in resp['results'][0], resp)
|
|
|
- assert_equal(resp['results'][0]['joinCols'], [{u'columns': [u'%s.x.a' % BaseTestOptimizerApi.DATABASE, u'%s.y.a' % BaseTestOptimizerApi.DATABASE]}])
|
|
|
+ assert_equal(resp['results'][0]['joinCols'], [{u'columns': [u'%s.x.a' % BaseTestOptimizerClient.DATABASE, u'%s.y.a' % BaseTestOptimizerClient.DATABASE]}])
|
|
|
|
|
|
|
|
|
def test_top_aggs(self):
|
|
|
- resp = self.api.top_aggs(db_tables=['%s.Part' % BaseTestOptimizerApi.DATABASE])
|
|
|
+ resp = self.api.top_aggs(db_tables=['%s.Part' % BaseTestOptimizerClient.DATABASE])
|
|
|
|
|
|
assert_true(len(resp['results']) > 0, resp)
|
|
|
|
|
|
@@ -248,7 +248,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
|
|
|
|
|
|
|
|
|
def test_top_columns(self):
|
|
|
- resp = self.api.top_columns(db_tables=['%s.Part' % BaseTestOptimizerApi.DATABASE])
|
|
|
+ resp = self.api.top_columns(db_tables=['%s.Part' % BaseTestOptimizerClient.DATABASE])
|
|
|
|
|
|
assert_true('orderbyColumns' in resp, resp)
|
|
|
assert_true('selectColumns' in resp, resp)
|
|
|
@@ -285,7 +285,7 @@ class TestOptimizerApi(BaseTestOptimizerApi):
|
|
|
|
|
|
|
|
|
|
|
|
-class TestOptimizerRiskApi(BaseTestOptimizerApi):
|
|
|
+class TestOptimizerRiskApi(BaseTestOptimizerClient):
|
|
|
|
|
|
def test_risk_10_views(self):
|
|
|
source_platform = 'hive'
|
|
|
@@ -317,7 +317,7 @@ FROM
|
|
|
FROM sample_01) t1) t2) t3) t4) t5) t6) t7) t8) t9) t10) t11) t12
|
|
|
'''
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
_assert_risks(['>=10 Inline Views present in query.'], resp['hints'])
|
|
|
|
|
|
|
|
|
@@ -326,7 +326,7 @@ FROM
|
|
|
query = '''SELECT ID, NAME, AMOUNT, DATE FROM CUSTOMERS, ORDERS
|
|
|
'''
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
_assert_risks(['Cartesian or CROSS join found.'], resp['hints'])
|
|
|
|
|
|
source_platform = 'hive'
|
|
|
@@ -340,7 +340,7 @@ WHERE s07.salary > 88
|
|
|
ORDER BY s07.salary DESC
|
|
|
'''
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
_assert_risks(['Cartesian or CROSS join found.'], resp['hints'])
|
|
|
|
|
|
|
|
|
@@ -378,7 +378,7 @@ ORDER BY s07.salary DESC
|
|
|
LIMIT 1000
|
|
|
'''
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
_assert_risks(['>=5 table joins or >=10 join conditions found.'], resp['hints'])
|
|
|
|
|
|
|
|
|
@@ -401,11 +401,11 @@ GROUP BY account_client,
|
|
|
limit 5
|
|
|
'''
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
_assert_risks(['>=10 columns present in GROUP BY list.'], resp['hints'])
|
|
|
|
|
|
- assert_equal(resp['noDDL'], ['%s.transactions' % BaseTestOptimizerApi.DATABASE])
|
|
|
- assert_equal(resp['noStats'], ['%s.transactions' % BaseTestOptimizerApi.DATABASE])
|
|
|
+ assert_equal(resp['noDDL'], ['%s.transactions' % BaseTestOptimizerClient.DATABASE])
|
|
|
+ assert_equal(resp['noStats'], ['%s.transactions' % BaseTestOptimizerClient.DATABASE])
|
|
|
|
|
|
|
|
|
def test_risk_cross_join_false_positive(self):
|
|
|
@@ -422,27 +422,27 @@ ORDER BY s07.salary DESC
|
|
|
LIMIT 1000
|
|
|
'''
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
_assert_risks(['Cartesian or CROSS join found.'], resp['hints'], present=False)
|
|
|
|
|
|
|
|
|
source_platform = 'hive'
|
|
|
query = '''select x from x join y where x.a = y.a'''
|
|
|
|
|
|
- resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerApi.DATABASE)
|
|
|
+ resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=BaseTestOptimizerClient.DATABASE)
|
|
|
_assert_risks(['Cartesian or CROSS join found.'], resp['hints'], present=False)
|
|
|
|
|
|
|
|
|
def test_risk_no_filter_on_any_partitioned_column(self):
|
|
|
source_platform = 'hive'
|
|
|
query = '''SELECT * FROM web_logs'''
|
|
|
- db_name = BaseTestOptimizerApi.DATABASE
|
|
|
+ db_name = BaseTestOptimizerClient.DATABASE
|
|
|
|
|
|
resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=db_name)
|
|
|
_assert_risks(['Query on partitioned table is missing filters on partioning columns.'], resp['hints'])
|
|
|
|
|
|
assert_false(resp['noDDL'], resp) # DDL was uploaded already
|
|
|
- assert_equal(resp['noStats'], ['%s.web_logs' % BaseTestOptimizerApi.DATABASE])
|
|
|
+ assert_equal(resp['noStats'], ['%s.web_logs' % BaseTestOptimizerClient.DATABASE])
|
|
|
|
|
|
|
|
|
source_platform = 'hive'
|
|
|
@@ -475,14 +475,14 @@ LIMIT 1000
|
|
|
def test_risk_listing_all_risk_tables_all_the_time(self):
|
|
|
source_platform = 'hive'
|
|
|
query = '''SELECT * FROM web_logs JOIN a ON web_logs.id = a.id LIMIT 100'''
|
|
|
- db_name = BaseTestOptimizerApi.DATABASE
|
|
|
+ db_name = BaseTestOptimizerClient.DATABASE
|
|
|
|
|
|
resp = self.api.query_risk(query=query, source_platform=source_platform, db_name=db_name)
|
|
|
_assert_risks(['Query on partitioned table is missing filters on partioning columns.'], resp['hints'])
|
|
|
|
|
|
- assert_equal([suggestion for suggestion in resp['hints'] if suggestion['riskId'] == 22][0]['riskTables'], ['%s.web_logs' % BaseTestOptimizerApi.DATABASE])
|
|
|
- assert_equal(resp['noDDL'], ['%s.a' % BaseTestOptimizerApi.DATABASE])
|
|
|
- assert_equal(resp['noStats'], ['%s.a' % BaseTestOptimizerApi.DATABASE, '%s.web_logs' % BaseTestOptimizerApi.DATABASE])
|
|
|
+ assert_equal([suggestion for suggestion in resp['hints'] if suggestion['riskId'] == 22][0]['riskTables'], ['%s.web_logs' % BaseTestOptimizerClient.DATABASE])
|
|
|
+ assert_equal(resp['noDDL'], ['%s.a' % BaseTestOptimizerClient.DATABASE])
|
|
|
+ assert_equal(resp['noStats'], ['%s.a' % BaseTestOptimizerClient.DATABASE, '%s.web_logs' % BaseTestOptimizerClient.DATABASE])
|
|
|
|
|
|
|
|
|
def _assert_risks(risks, suggestions, present=True):
|