|
@@ -17,6 +17,7 @@
|
|
|
# limitations under the License.
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
from future import standard_library
|
|
from future import standard_library
|
|
|
|
|
+
|
|
|
standard_library.install_aliases()
|
|
standard_library.install_aliases()
|
|
|
from builtins import object
|
|
from builtins import object
|
|
|
import json
|
|
import json
|
|
@@ -38,7 +39,8 @@ from useradmin.models import HuePermission, GroupPermission, User, Group
|
|
|
|
|
|
|
|
from beeswax.conf import LIST_PARTITIONS_LIMIT
|
|
from beeswax.conf import LIST_PARTITIONS_LIMIT
|
|
|
from beeswax.views import collapse_whitespace
|
|
from beeswax.views import collapse_whitespace
|
|
|
-from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config, fetch_query_result_data
|
|
|
|
|
|
|
+from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config, \
|
|
|
|
|
+ fetch_query_result_data
|
|
|
from beeswax.models import QueryHistory
|
|
from beeswax.models import QueryHistory
|
|
|
from beeswax.server import dbms
|
|
from beeswax.server import dbms
|
|
|
from beeswax.test_base import BeeswaxSampleProvider
|
|
from beeswax.test_base import BeeswaxSampleProvider
|
|
@@ -59,23 +61,21 @@ def _make_query(client, query, submission_type="Execute",
|
|
|
res = make_query(client, query, submission_type,
|
|
res = make_query(client, query, submission_type,
|
|
|
udfs, settings, resources,
|
|
udfs, settings, resources,
|
|
|
wait, name, desc, local, is_parameterized, max, database, email_notify, **kwargs)
|
|
wait, name, desc, local, is_parameterized, max, database, email_notify, **kwargs)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Should be in the history if it's submitted.
|
|
# Should be in the history if it's submitted.
|
|
|
if submission_type == 'Execute':
|
|
if submission_type == 'Execute':
|
|
|
fragment = collapse_whitespace(smart_str(query[:20]))
|
|
fragment = collapse_whitespace(smart_str(query[:20]))
|
|
|
verify_history(client, fragment=fragment)
|
|
verify_history(client, fragment=fragment)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
return res
|
|
return res
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestApi():
|
|
class TestApi():
|
|
|
-
|
|
|
|
|
def setUp(self):
|
|
def setUp(self):
|
|
|
self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
|
|
self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
self.user = User.objects.get(username="test")
|
|
self.user = User.objects.get(username="test")
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_show_tables(self):
|
|
def test_show_tables(self):
|
|
|
grant_access("test", "default", "metastore")
|
|
grant_access("test", "default", "metastore")
|
|
|
with patch('beeswax.server.dbms.get') as get:
|
|
with patch('beeswax.server.dbms.get') as get:
|
|
@@ -91,11 +91,43 @@ class TestApi():
|
|
|
),
|
|
),
|
|
|
server_name='hive'
|
|
server_name='hive'
|
|
|
)
|
|
)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
response = self.client.post('/metastore/tables/sfdc?format=json')
|
|
response = self.client.post('/metastore/tables/sfdc?format=json')
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
get.assert_called()
|
|
get.assert_called()
|
|
|
|
|
+
|
|
|
|
|
+ assert_equal(response.status_code, 200)
|
|
|
|
|
+ data = json.loads(response.content)
|
|
|
|
|
+ assert_equal(data['status'], 0)
|
|
|
|
|
+ assert_equal(data['table_names'], ['customer', 'opportunities'])
|
|
|
|
|
+ assert_equal(data['tables'], [{'name': 'customer'}, {'name': 'opportunities'}])
|
|
|
|
|
|
|
|
|
|
+ def test_show_tables_hs2(self):
|
|
|
|
|
+ grant_access("test", "default", "metastore")
|
|
|
|
|
+ with patch('beeswax.server.dbms.get') as get:
|
|
|
|
|
+ with patch('beeswax.server.dbms.KazooClient') as KazooClient:
|
|
|
|
|
+ server_config = get_query_server_config(name='beeswax')
|
|
|
|
|
+ KazooClient.return_value = Mock(
|
|
|
|
|
+ # Bug "TypeError: expected string or buffer" if False, to add a new test case and fix
|
|
|
|
|
+ exists=Mock(return_value=True),
|
|
|
|
|
+ get_children=Mock(
|
|
|
|
|
+ return_value=['serverUri=hive-llap-1.gethue.com:10000;serverUri=hive-llap-2.gethue.com:10000'])
|
|
|
|
|
+ )
|
|
|
|
|
+ get.return_value = Mock(
|
|
|
|
|
+ get_databases=Mock(
|
|
|
|
|
+ return_value=['sfdc']
|
|
|
|
|
+ ),
|
|
|
|
|
+ get_database=Mock(
|
|
|
|
|
+ return_value={}
|
|
|
|
|
+ ),
|
|
|
|
|
+ get_tables_meta=Mock(
|
|
|
|
|
+ return_value=[{'name': 'customer'}, {'name': 'opportunities'}]
|
|
|
|
|
+ ),
|
|
|
|
|
+ server_name='hive'
|
|
|
|
|
+ )
|
|
|
|
|
+ response = self.client.post('/metastore/tables/sfdc?format=json')
|
|
|
|
|
+ get.assert_called()
|
|
|
|
|
+
|
|
|
assert_equal(response.status_code, 200)
|
|
assert_equal(response.status_code, 200)
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_equal(data['status'], 0)
|
|
assert_equal(data['status'], 0)
|
|
@@ -106,51 +138,53 @@ class TestApi():
|
|
|
class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
requires_hadoop = True
|
|
requires_hadoop = True
|
|
|
integration = True
|
|
integration = True
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def setUp(self):
|
|
def setUp(self):
|
|
|
user = User.objects.get(username='test')
|
|
user = User.objects.get(username='test')
|
|
|
self.db = dbms.get(user, get_query_server_config())
|
|
self.db = dbms.get(user, get_query_server_config())
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
add_permission("test", "test", "write", "metastore")
|
|
add_permission("test", "test", "write", "metastore")
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_basic_flow(self):
|
|
def test_basic_flow(self):
|
|
|
# Default database should exist
|
|
# Default database should exist
|
|
|
response = self.client.get("/metastore/databases")
|
|
response = self.client.get("/metastore/databases")
|
|
|
assert_true(self.db_name in response.context[0]["databases"])
|
|
assert_true(self.db_name in response.context[0]["databases"])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Table should have been created
|
|
# Table should have been created
|
|
|
response = self.client.get("/metastore/tables/")
|
|
response = self.client.get("/metastore/tables/")
|
|
|
assert_equal(200, response.status_code)
|
|
assert_equal(200, response.status_code)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Switch databases
|
|
# Switch databases
|
|
|
response = self.client.get("/metastore/tables/%s?format=json" % self.db_name)
|
|
response = self.client.get("/metastore/tables/%s?format=json" % self.db_name)
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_true('name' in data["tables"][0])
|
|
assert_true('name' in data["tables"][0])
|
|
|
assert_true("test" in data["table_names"])
|
|
assert_true("test" in data["table_names"])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Should default to "default" database
|
|
# Should default to "default" database
|
|
|
response = self.client.get("/metastore/tables/not_there")
|
|
response = self.client.get("/metastore/tables/not_there")
|
|
|
assert_equal(200, response.status_code)
|
|
assert_equal(200, response.status_code)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# And have detail
|
|
# And have detail
|
|
|
response = self.client.post("/metastore/table/%s/test/?format=json" % self.db_name, {'format': 'json'})
|
|
response = self.client.post("/metastore/table/%s/test/?format=json" % self.db_name, {'format': 'json'})
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_true("foo" in [col['name'] for col in data['cols']])
|
|
assert_true("foo" in [col['name'] for col in data['cols']])
|
|
|
assert_true("SerDe Library:" in [prop['col_name'] for prop in data['properties']], data)
|
|
assert_true("SerDe Library:" in [prop['col_name'] for prop in data['properties']], data)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Remember the number of history items. Use a generic fragment 'test' to pass verification.
|
|
# Remember the number of history items. Use a generic fragment 'test' to pass verification.
|
|
|
history_cnt = verify_history(self.client, fragment='test')
|
|
history_cnt = verify_history(self.client, fragment='test')
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Show table data.
|
|
# Show table data.
|
|
|
response = self.client.get("/metastore/table/%s/test/read" % self.db_name, follow=True)
|
|
response = self.client.get("/metastore/table/%s/test/read" % self.db_name, follow=True)
|
|
|
- response = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context[0]['query'].id}), follow=True)
|
|
|
|
|
|
|
+ response = self.client.get(
|
|
|
|
|
+ reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context[0]['query'].id}), follow=True)
|
|
|
response = wait_for_query_to_finish(self.client, response, max=30.0)
|
|
response = wait_for_query_to_finish(self.client, response, max=30.0)
|
|
|
# Note that it may not return all rows at once. But we expect at least 10.
|
|
# Note that it may not return all rows at once. But we expect at least 10.
|
|
|
results = fetch_query_result_data(self.client, response)
|
|
results = fetch_query_result_data(self.client, response)
|
|
|
assert_true(len(results['results']) > 0)
|
|
assert_true(len(results['results']) > 0)
|
|
|
# This should NOT go into the query history.
|
|
# This should NOT go into the query history.
|
|
|
- assert_equal(verify_history(self.client, fragment='test'), history_cnt, 'Implicit queries should not be saved in the history')
|
|
|
|
|
-
|
|
|
|
|
|
|
+ assert_equal(verify_history(self.client, fragment='test'), history_cnt,
|
|
|
|
|
+ 'Implicit queries should not be saved in the history')
|
|
|
|
|
+
|
|
|
def test_show_tables(self):
|
|
def test_show_tables(self):
|
|
|
hql = """
|
|
hql = """
|
|
|
CREATE TABLE test_show_tables_1 (a int) COMMENT 'Test for show_tables';
|
|
CREATE TABLE test_show_tables_1 (a int) COMMENT 'Test for show_tables';
|
|
@@ -159,7 +193,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
"""
|
|
"""
|
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Table should have been created
|
|
# Table should have been created
|
|
|
response = self.client.get("/metastore/tables/%s?filter=show_tables&format=json" % self.db_name)
|
|
response = self.client.get("/metastore/tables/%s?filter=show_tables&format=json" % self.db_name)
|
|
|
assert_equal(200, response.status_code)
|
|
assert_equal(200, response.status_code)
|
|
@@ -168,14 +202,14 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
assert_true('name' in data["tables"][0])
|
|
assert_true('name' in data["tables"][0])
|
|
|
assert_true('comment' in data["tables"][0])
|
|
assert_true('comment' in data["tables"][0])
|
|
|
assert_true('type' in data["tables"][0])
|
|
assert_true('type' in data["tables"][0])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
hql = """
|
|
hql = """
|
|
|
CREATE TABLE test_show_tables_4 (a int) COMMENT 'Test for show_tables';
|
|
CREATE TABLE test_show_tables_4 (a int) COMMENT 'Test for show_tables';
|
|
|
CREATE TABLE test_show_tables_5 (a int) COMMENT 'Test for show_tables';
|
|
CREATE TABLE test_show_tables_5 (a int) COMMENT 'Test for show_tables';
|
|
|
"""
|
|
"""
|
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Table should have been created
|
|
# Table should have been created
|
|
|
response = self.client.get("/metastore/tables/%s?filter=show_tables&format=json" % self.db_name)
|
|
response = self.client.get("/metastore/tables/%s?filter=show_tables&format=json" % self.db_name)
|
|
|
assert_equal(200, response.status_code)
|
|
assert_equal(200, response.status_code)
|
|
@@ -184,42 +218,43 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
assert_true('name' in data["tables"][0])
|
|
assert_true('name' in data["tables"][0])
|
|
|
assert_true('comment' in data["tables"][0])
|
|
assert_true('comment' in data["tables"][0])
|
|
|
assert_true('type' in data["tables"][0])
|
|
assert_true('type' in data["tables"][0])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
hql = """
|
|
hql = """
|
|
|
CREATE INDEX test_index ON TABLE test_show_tables_1 (a) AS 'COMPACT' WITH DEFERRED REBUILD;
|
|
CREATE INDEX test_index ON TABLE test_show_tables_1 (a) AS 'COMPACT' WITH DEFERRED REBUILD;
|
|
|
"""
|
|
"""
|
|
|
resp = _make_query(self.client, hql, wait=True, local=False, max=30.0, database=self.db_name)
|
|
resp = _make_query(self.client, hql, wait=True, local=False, max=30.0, database=self.db_name)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# By default, index table should not appear in show tables view
|
|
# By default, index table should not appear in show tables view
|
|
|
response = self.client.get("/metastore/tables/%s?format=json" % self.db_name)
|
|
response = self.client.get("/metastore/tables/%s?format=json" % self.db_name)
|
|
|
assert_equal(200, response.status_code)
|
|
assert_equal(200, response.status_code)
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_false('test_index' in data['tables'])
|
|
assert_false('test_index' in data['tables'])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_describe_view(self):
|
|
def test_describe_view(self):
|
|
|
resp = self.client.post('/metastore/table/%s/myview' % self.db_name, data={'format': 'json'})
|
|
resp = self.client.post('/metastore/table/%s/myview' % self.db_name, data={'format': 'json'})
|
|
|
assert_equal(200, resp.status_code, resp.content)
|
|
assert_equal(200, resp.status_code, resp.content)
|
|
|
data = json.loads(resp.content)
|
|
data = json.loads(resp.content)
|
|
|
assert_true(data['is_view'])
|
|
assert_true(data['is_view'])
|
|
|
assert_equal("myview", data['name'])
|
|
assert_equal("myview", data['name'])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_describe_partitions(self):
|
|
def test_describe_partitions(self):
|
|
|
response = self.client.post("/metastore/table/%s/test_partitions" % self.db_name, data={'format': 'json'})
|
|
response = self.client.post("/metastore/table/%s/test_partitions" % self.db_name, data={'format': 'json'})
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_equal(2, len(data['partition_keys']), data)
|
|
assert_equal(2, len(data['partition_keys']), data)
|
|
|
-
|
|
|
|
|
- response = self.client.post("/metastore/table/%s/test_partitions/partitions" % self.db_name, data={'format': 'json'}, follow=True)
|
|
|
|
|
|
|
+
|
|
|
|
|
+ response = self.client.post("/metastore/table/%s/test_partitions/partitions" % self.db_name,
|
|
|
|
|
+ data={'format': 'json'}, follow=True)
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
partition_columns = [col for cols in data['partition_values_json'] for col in cols['columns']]
|
|
partition_columns = [col for cols in data['partition_values_json'] for col in cols['columns']]
|
|
|
assert_true("baz_one" in partition_columns)
|
|
assert_true("baz_one" in partition_columns)
|
|
|
assert_true('12345' in partition_columns, partition_columns)
|
|
assert_true('12345' in partition_columns, partition_columns)
|
|
|
assert_true("baz_foo" in partition_columns)
|
|
assert_true("baz_foo" in partition_columns)
|
|
|
assert_true('67890' in partition_columns)
|
|
assert_true('67890' in partition_columns)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Not partitioned
|
|
# Not partitioned
|
|
|
response = self.client.get("/metastore/table/%s/test/partitions" % self.db_name, follow=True)
|
|
response = self.client.get("/metastore/table/%s/test/partitions" % self.db_name, follow=True)
|
|
|
assert_true("is not partitioned." in response.content)
|
|
assert_true("is not partitioned." in response.content)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_describe_partitioned_table_with_limit(self):
|
|
def test_describe_partitioned_table_with_limit(self):
|
|
|
# We have 2 partitions in the test table
|
|
# We have 2 partitions in the test table
|
|
|
finish = LIST_PARTITIONS_LIMIT.set_for_testing("1")
|
|
finish = LIST_PARTITIONS_LIMIT.set_for_testing("1")
|
|
@@ -229,7 +264,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
assert_equal(1, len(partition_values_json))
|
|
assert_equal(1, len(partition_values_json))
|
|
|
finally:
|
|
finally:
|
|
|
finish()
|
|
finish()
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
finish = LIST_PARTITIONS_LIMIT.set_for_testing("3")
|
|
finish = LIST_PARTITIONS_LIMIT.set_for_testing("3")
|
|
|
try:
|
|
try:
|
|
|
response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
|
|
response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
|
|
@@ -237,40 +272,43 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
assert_equal(2, len(partition_values_json))
|
|
assert_equal(2, len(partition_values_json))
|
|
|
finally:
|
|
finally:
|
|
|
finish()
|
|
finish()
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_read_partitions(self):
|
|
def test_read_partitions(self):
|
|
|
if not is_live_cluster():
|
|
if not is_live_cluster():
|
|
|
raise SkipTest
|
|
raise SkipTest
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
partition_spec = "baz='baz_one',boom=12345"
|
|
partition_spec = "baz='baz_one',boom=12345"
|
|
|
- response = self.client.get("/metastore/table/%s/test_partitions/partitions/%s/read" % (self.db_name, partition_spec), follow=True)
|
|
|
|
|
- response = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context[0]['query'].id}), follow=True)
|
|
|
|
|
|
|
+ response = self.client.get(
|
|
|
|
|
+ "/metastore/table/%s/test_partitions/partitions/%s/read" % (self.db_name, partition_spec), follow=True)
|
|
|
|
|
+ response = self.client.get(
|
|
|
|
|
+ reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': response.context[0]['query'].id}), follow=True)
|
|
|
response = wait_for_query_to_finish(self.client, response, max=30.0)
|
|
response = wait_for_query_to_finish(self.client, response, max=30.0)
|
|
|
results = fetch_query_result_data(self.client, response)
|
|
results = fetch_query_result_data(self.client, response)
|
|
|
assert_true(len(results['results']) > 0, results)
|
|
assert_true(len(results['results']) > 0, results)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_browse_partition(self):
|
|
def test_browse_partition(self):
|
|
|
partition_spec = "baz='baz_one',boom=12345"
|
|
partition_spec = "baz='baz_one',boom=12345"
|
|
|
- response = self.client.get("/metastore/table/%s/test_partitions/partitions/%s/browse" % (self.db_name, partition_spec), follow=True)
|
|
|
|
|
|
|
+ response = self.client.get(
|
|
|
|
|
+ "/metastore/table/%s/test_partitions/partitions/%s/browse" % (self.db_name, partition_spec), follow=True)
|
|
|
if is_live_cluster():
|
|
if is_live_cluster():
|
|
|
path = '/user/hive/warehouse/%s.db/test_partitions/baz=baz_one/boom=12345' % self.db_name
|
|
path = '/user/hive/warehouse/%s.db/test_partitions/baz=baz_one/boom=12345' % self.db_name
|
|
|
else:
|
|
else:
|
|
|
path = '/user/hive/warehouse/test_partitions/baz=baz_one/boom=12345'
|
|
path = '/user/hive/warehouse/test_partitions/baz=baz_one/boom=12345'
|
|
|
filebrowser_path = urllib.parse.unquote(reverse("filebrowser:filebrowser.views.view", kwargs={'path': path}))
|
|
filebrowser_path = urllib.parse.unquote(reverse("filebrowser:filebrowser.views.view", kwargs={'path': path}))
|
|
|
assert_equal(response.request['PATH_INFO'], filebrowser_path)
|
|
assert_equal(response.request['PATH_INFO'], filebrowser_path)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_drop_partition(self):
|
|
def test_drop_partition(self):
|
|
|
# Create partition first
|
|
# Create partition first
|
|
|
partition_spec = "baz='baz_drop',boom=54321"
|
|
partition_spec = "baz='baz_drop',boom=54321"
|
|
|
hql = 'ALTER TABLE `%s`.`test_partitions` ADD IF NOT EXISTS PARTITION (%s);' % (self.db_name, partition_spec)
|
|
hql = 'ALTER TABLE `%s`.`test_partitions` ADD IF NOT EXISTS PARTITION (%s);' % (self.db_name, partition_spec)
|
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
|
wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Assert partition exists
|
|
# Assert partition exists
|
|
|
response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name, {'format': 'json'})
|
|
response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name, {'format': 'json'})
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_true("baz_drop" in [part['columns'][0] for part in data['partition_values_json']], data)
|
|
assert_true("baz_drop" in [part['columns'][0] for part in data['partition_values_json']], data)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Drop partition
|
|
# Drop partition
|
|
|
self.client.post(
|
|
self.client.post(
|
|
|
"/metastore/table/%s/test_partitions/partitions/drop" % self.db_name,
|
|
"/metastore/table/%s/test_partitions/partitions/drop" % self.db_name,
|
|
@@ -285,7 +323,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name, {'format': 'json'})
|
|
response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name, {'format': 'json'})
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_false("baz_drop" in [part['columns'][0] for part in data['partition_values_json']], data)
|
|
assert_false("baz_drop" in [part['columns'][0] for part in data['partition_values_json']], data)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_drop_multi_tables(self):
|
|
def test_drop_multi_tables(self):
|
|
|
hql = """
|
|
hql = """
|
|
|
CREATE TABLE test_drop_1 (a int);
|
|
CREATE TABLE test_drop_1 (a int);
|
|
@@ -294,7 +332,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
"""
|
|
"""
|
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Drop them
|
|
# Drop them
|
|
|
resp = self.client.get('/metastore/tables/drop/%s' % self.db_name, follow=True)
|
|
resp = self.client.get('/metastore/tables/drop/%s' % self.db_name, follow=True)
|
|
|
assert_true('want to delete' in resp.content, resp.content)
|
|
assert_true('want to delete' in resp.content, resp.content)
|
|
@@ -303,7 +341,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
{u'table_selection': [u'test_drop_1', u'test_drop_2', u'test_drop_3'], 'is_embeddable': True}
|
|
{u'table_selection': [u'test_drop_1', u'test_drop_2', u'test_drop_3'], 'is_embeddable': True}
|
|
|
)
|
|
)
|
|
|
assert_equal(resp.status_code, 302)
|
|
assert_equal(resp.status_code, 302)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_drop_multi_tables_with_skip_trash(self):
|
|
def test_drop_multi_tables_with_skip_trash(self):
|
|
|
hql = """
|
|
hql = """
|
|
|
CREATE TABLE test_drop_multi_tables_with_skip_trash_1 (a int);
|
|
CREATE TABLE test_drop_multi_tables_with_skip_trash_1 (a int);
|
|
@@ -312,7 +350,7 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
"""
|
|
"""
|
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
resp = _make_query(self.client, hql, database=self.db_name)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Drop them
|
|
# Drop them
|
|
|
resp = self.client.get('/metastore/tables/drop/%s' % self.db_name, follow=True)
|
|
resp = self.client.get('/metastore/tables/drop/%s' % self.db_name, follow=True)
|
|
|
assert_true('want to delete' in resp.content, resp.content)
|
|
assert_true('want to delete' in resp.content, resp.content)
|
|
@@ -320,25 +358,25 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
'/metastore/tables/drop/%s' % self.db_name,
|
|
'/metastore/tables/drop/%s' % self.db_name,
|
|
|
{
|
|
{
|
|
|
u'table_selection': [u'test_drop_multi_tables_with_skip_trash_1',
|
|
u'table_selection': [u'test_drop_multi_tables_with_skip_trash_1',
|
|
|
- u'test_drop_multi_tables_with_skip_trash_2', u'test_drop_multi_tables_with_skip_trash_3'],
|
|
|
|
|
|
|
+ u'test_drop_multi_tables_with_skip_trash_2', u'test_drop_multi_tables_with_skip_trash_3'],
|
|
|
u'skip_trash': u'on',
|
|
u'skip_trash': u'on',
|
|
|
'is_embeddable': True
|
|
'is_embeddable': True
|
|
|
}
|
|
}
|
|
|
)
|
|
)
|
|
|
assert_equal(resp.status_code, 302)
|
|
assert_equal(resp.status_code, 302)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
response = self.client.get("/metastore/tables/%s?format=json" % self.db_name)
|
|
response = self.client.get("/metastore/tables/%s?format=json" % self.db_name)
|
|
|
assert_equal(200, response.status_code)
|
|
assert_equal(200, response.status_code)
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
assert_false('test_drop_multi_tables_with_skip_trash_1' in data['tables'])
|
|
assert_false('test_drop_multi_tables_with_skip_trash_1' in data['tables'])
|
|
|
assert_false('test_drop_multi_tables_with_skip_trash_2' in data['tables'])
|
|
assert_false('test_drop_multi_tables_with_skip_trash_2' in data['tables'])
|
|
|
assert_false('test_drop_multi_tables_with_skip_trash_3' in data['tables'])
|
|
assert_false('test_drop_multi_tables_with_skip_trash_3' in data['tables'])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_drop_multi_databases(self):
|
|
def test_drop_multi_databases(self):
|
|
|
db1 = '%s_test_drop_1' % self.db_name
|
|
db1 = '%s_test_drop_1' % self.db_name
|
|
|
db2 = '%s_test_drop_2' % self.db_name
|
|
db2 = '%s_test_drop_2' % self.db_name
|
|
|
db3 = '%s_test_drop_3' % self.db_name
|
|
db3 = '%s_test_drop_3' % self.db_name
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
try:
|
|
try:
|
|
|
hql = """
|
|
hql = """
|
|
|
CREATE DATABASE %(db1)s;
|
|
CREATE DATABASE %(db1)s;
|
|
@@ -347,55 +385,59 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
""" % {'db1': db1, 'db2': db2, 'db3': db3}
|
|
""" % {'db1': db1, 'db2': db2, 'db3': db3}
|
|
|
resp = _make_query(self.client, hql)
|
|
resp = _make_query(self.client, hql)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Add a table to db1
|
|
# Add a table to db1
|
|
|
hql = "CREATE TABLE " + "`" + db1 + "`." + "`test_drop_1` (a int);"
|
|
hql = "CREATE TABLE " + "`" + db1 + "`." + "`test_drop_1` (a int);"
|
|
|
resp = _make_query(self.client, hql, database=db1)
|
|
resp = _make_query(self.client, hql, database=db1)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
assert_equal(resp.status_code, 200)
|
|
assert_equal(resp.status_code, 200)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Drop them
|
|
# Drop them
|
|
|
resp = self.client.get('/metastore/databases/drop', follow=True)
|
|
resp = self.client.get('/metastore/databases/drop', follow=True)
|
|
|
assert_true('want to delete' in resp.content, resp.content)
|
|
assert_true('want to delete' in resp.content, resp.content)
|
|
|
- resp = self.client.post('/metastore/databases/drop', {u'database_selection': [db1, db2, db3], 'is_embeddable': True})
|
|
|
|
|
|
|
+ resp = self.client.post('/metastore/databases/drop',
|
|
|
|
|
+ {u'database_selection': [db1, db2, db3], 'is_embeddable': True})
|
|
|
assert_equal(resp.status_code, 302)
|
|
assert_equal(resp.status_code, 302)
|
|
|
finally:
|
|
finally:
|
|
|
make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db1}, wait=True)
|
|
make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db1}, wait=True)
|
|
|
make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db2}, wait=True)
|
|
make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db2}, wait=True)
|
|
|
make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db3}, wait=True)
|
|
make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db3}, wait=True)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_load_data(self):
|
|
def test_load_data(self):
|
|
|
"""
|
|
"""
|
|
|
Test load data queries.
|
|
Test load data queries.
|
|
|
These require Hadoop, because they ask the metastore
|
|
These require Hadoop, because they ask the metastore
|
|
|
about whether a table is partitioned.
|
|
about whether a table is partitioned.
|
|
|
"""
|
|
"""
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Check that view works
|
|
# Check that view works
|
|
|
resp = self.client.get("/metastore/table/%s/test/load" % self.db_name, follow=True)
|
|
resp = self.client.get("/metastore/table/%s/test/load" % self.db_name, follow=True)
|
|
|
assert_true('Path' in resp.content)
|
|
assert_true('Path' in resp.content)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
data_dir = '%(prefix)s/tmp' % {'prefix': self.cluster.fs_prefix}
|
|
data_dir = '%(prefix)s/tmp' % {'prefix': self.cluster.fs_prefix}
|
|
|
data_path = data_dir + '/foo'
|
|
data_path = data_dir + '/foo'
|
|
|
self.cluster.fs.mkdir(data_dir)
|
|
self.cluster.fs.mkdir(data_dir)
|
|
|
self.cluster.fs.create(data_path, data='123')
|
|
self.cluster.fs.create(data_path, data='123')
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Try the submission
|
|
# Try the submission
|
|
|
- response = self.client.post("/metastore/table/%s/test/load" % self.db_name, {'path': data_path, 'overwrite': True}, follow=True)
|
|
|
|
|
|
|
+ response = self.client.post("/metastore/table/%s/test/load" % self.db_name, {'path': data_path, 'overwrite': True},
|
|
|
|
|
+ follow=True)
|
|
|
data = json.loads(response.content)
|
|
data = json.loads(response.content)
|
|
|
query = QueryHistory.objects.get(id=data['query_history_id'])
|
|
query = QueryHistory.objects.get(id=data['query_history_id'])
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
assert_equal_mod_whitespace(
|
|
assert_equal_mod_whitespace(
|
|
|
- "LOAD DATA INPATH '%(data_path)s' OVERWRITE INTO TABLE `%(db)s`.`test`" % {'data_path': data_path, 'db': self.db_name}, query.query
|
|
|
|
|
|
|
+ "LOAD DATA INPATH '%(data_path)s' OVERWRITE INTO TABLE `%(db)s`.`test`" % {'data_path': data_path,
|
|
|
|
|
+ 'db': self.db_name}, query.query
|
|
|
)
|
|
)
|
|
|
-
|
|
|
|
|
- resp = self.client.post("/metastore/table/%s/test/load" % self.db_name, {'path': data_path, 'overwrite': False}, follow=True)
|
|
|
|
|
|
|
+
|
|
|
|
|
+ resp = self.client.post("/metastore/table/%s/test/load" % self.db_name, {'path': data_path, 'overwrite': False},
|
|
|
|
|
+ follow=True)
|
|
|
query = QueryHistory.objects.latest('id')
|
|
query = QueryHistory.objects.latest('id')
|
|
|
assert_equal_mod_whitespace(
|
|
assert_equal_mod_whitespace(
|
|
|
- "LOAD DATA INPATH '%(data_path)s' INTO TABLE `%(db)s`.`test`" % {'data_path': data_path, 'db': self.db_name}, query.query
|
|
|
|
|
|
|
+ "LOAD DATA INPATH '%(data_path)s' INTO TABLE `%(db)s`.`test`" % {'data_path': data_path, 'db': self.db_name},
|
|
|
|
|
+ query.query
|
|
|
)
|
|
)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Try it with partitions
|
|
# Try it with partitions
|
|
|
resp = self.client.post(
|
|
resp = self.client.post(
|
|
|
"/metastore/table/%s/test_partitions/load" % self.db_name,
|
|
"/metastore/table/%s/test_partitions/load" % self.db_name,
|
|
@@ -409,96 +451,95 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
'data_path': data_path, 'db': self.db_name
|
|
'data_path': data_path, 'db': self.db_name
|
|
|
}
|
|
}
|
|
|
)
|
|
)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_has_write_access_frontend(self):
|
|
def test_has_write_access_frontend(self):
|
|
|
- client = make_logged_in_client(username='write_access_frontend', groupname='write_access_frontend', is_superuser=False)
|
|
|
|
|
|
|
+ client = make_logged_in_client(username='write_access_frontend', groupname='write_access_frontend',
|
|
|
|
|
+ is_superuser=False)
|
|
|
grant_access("write_access_frontend", "write_access_frontend", "metastore")
|
|
grant_access("write_access_frontend", "write_access_frontend", "metastore")
|
|
|
user = User.objects.get(username='write_access_frontend')
|
|
user = User.objects.get(username='write_access_frontend')
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def check(client, assertz):
|
|
def check(client, assertz):
|
|
|
response = client.get("/metastore/databases")
|
|
response = client.get("/metastore/databases")
|
|
|
assertz("Drop</button>" in response.content, response.content)
|
|
assertz("Drop</button>" in response.content, response.content)
|
|
|
assertz("Create a new database" in response.content, response.content)
|
|
assertz("Create a new database" in response.content, response.content)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
response = client.get("/metastore/tables/")
|
|
response = client.get("/metastore/tables/")
|
|
|
assertz("Drop</button>" in response.content, response.content)
|
|
assertz("Drop</button>" in response.content, response.content)
|
|
|
assertz("Create a new table" in response.content, response.content)
|
|
assertz("Create a new table" in response.content, response.content)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
check(client, assert_false)
|
|
check(client, assert_false)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Add access
|
|
# Add access
|
|
|
group, created = Group.objects.get_or_create(name='write_access_frontend')
|
|
group, created = Group.objects.get_or_create(name='write_access_frontend')
|
|
|
perm, created = HuePermission.objects.get_or_create(app='metastore', action='write')
|
|
perm, created = HuePermission.objects.get_or_create(app='metastore', action='write')
|
|
|
GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
|
|
GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
check(client, assert_true)
|
|
check(client, assert_true)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_has_write_access_backend(self):
|
|
def test_has_write_access_backend(self):
|
|
|
- client = make_logged_in_client(username='write_access_backend', groupname='write_access_backend', is_superuser=False)
|
|
|
|
|
|
|
+ client = make_logged_in_client(username='write_access_backend', groupname='write_access_backend',
|
|
|
|
|
+ is_superuser=False)
|
|
|
grant_access("write_access_backend", "write_access_backend", "metastore")
|
|
grant_access("write_access_backend", "write_access_backend", "metastore")
|
|
|
grant_access("write_access_backend", "write_access_backend", "beeswax")
|
|
grant_access("write_access_backend", "write_access_backend", "beeswax")
|
|
|
user = User.objects.get(username='write_access_backend')
|
|
user = User.objects.get(username='write_access_backend')
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Only fails if we were using Sentry and won't allow SELECT to user
|
|
# Only fails if we were using Sentry and won't allow SELECT to user
|
|
|
resp = _make_query(client, 'CREATE TABLE test_perm_1 (a int);', database=self.db_name)
|
|
resp = _make_query(client, 'CREATE TABLE test_perm_1 (a int);', database=self.db_name)
|
|
|
resp = wait_for_query_to_finish(client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def check(client, http_codes):
|
|
def check(client, http_codes):
|
|
|
resp = client.get('/metastore/tables/drop/%s' % self.db_name)
|
|
resp = client.get('/metastore/tables/drop/%s' % self.db_name)
|
|
|
assert_true(resp.status_code in http_codes, resp.content)
|
|
assert_true(resp.status_code in http_codes, resp.content)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
resp = client.post('/metastore/tables/drop/%s' % self.db_name, {u'table_selection': [u'test_perm_1']})
|
|
resp = client.post('/metastore/tables/drop/%s' % self.db_name, {u'table_selection': [u'test_perm_1']})
|
|
|
assert_true(resp.status_code in http_codes, resp.content)
|
|
assert_true(resp.status_code in http_codes, resp.content)
|
|
|
-
|
|
|
|
|
- check(client, [301]) # Denied
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
|
|
+ check(client, [301]) # Denied
|
|
|
|
|
+
|
|
|
# Add access
|
|
# Add access
|
|
|
group, created = Group.objects.get_or_create(name='write_access_backend')
|
|
group, created = Group.objects.get_or_create(name='write_access_backend')
|
|
|
perm, created = HuePermission.objects.get_or_create(app='metastore', action='write')
|
|
perm, created = HuePermission.objects.get_or_create(app='metastore', action='write')
|
|
|
GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
|
|
GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
|
|
|
-
|
|
|
|
|
- check(client, [200, 302]) # Ok
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
|
|
+ check(client, [200, 302]) # Ok
|
|
|
|
|
+
|
|
|
def test_alter_database(self):
|
|
def test_alter_database(self):
|
|
|
resp = self.client.post(reverse("metastore:get_database_metadata", kwargs={'database': self.db_name}))
|
|
resp = self.client.post(reverse("metastore:get_database_metadata", kwargs={'database': self.db_name}))
|
|
|
json_resp = json.loads(resp.content)
|
|
json_resp = json.loads(resp.content)
|
|
|
assert_true('data' in json_resp, json_resp)
|
|
assert_true('data' in json_resp, json_resp)
|
|
|
assert_true('parameters' in json_resp['data'], json_resp)
|
|
assert_true('parameters' in json_resp['data'], json_resp)
|
|
|
assert_false('message=After Alter' in json_resp['data']['parameters'], json_resp)
|
|
assert_false('message=After Alter' in json_resp['data']['parameters'], json_resp)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Alter message
|
|
# Alter message
|
|
|
resp = self.client.post(reverse("metastore:alter_database", kwargs={'database': self.db_name}),
|
|
resp = self.client.post(reverse("metastore:alter_database", kwargs={'database': self.db_name}),
|
|
|
{'properties': json.dumps({'message': 'After Alter'})})
|
|
{'properties': json.dumps({'message': 'After Alter'})})
|
|
|
json_resp = json.loads(resp.content)
|
|
json_resp = json.loads(resp.content)
|
|
|
assert_equal(0, json_resp['status'], json_resp)
|
|
assert_equal(0, json_resp['status'], json_resp)
|
|
|
assert_equal('{message=After Alter}', json_resp['data']['parameters'], json_resp)
|
|
assert_equal('{message=After Alter}', json_resp['data']['parameters'], json_resp)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_alter_table(self):
|
|
def test_alter_table(self):
|
|
|
- resp = _make_query(self.client, "CREATE TABLE test_alter_table (a int) COMMENT 'Before Alter';", database=self.db_name)
|
|
|
|
|
|
|
+ resp = _make_query(self.client, "CREATE TABLE test_alter_table (a int) COMMENT 'Before Alter';",
|
|
|
|
|
+ database=self.db_name)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
resp = self.client.get('/metastore/table/%s/test_alter_table' % self.db_name)
|
|
resp = self.client.get('/metastore/table/%s/test_alter_table' % self.db_name)
|
|
|
assert_true('test_alter_table', resp.content)
|
|
assert_true('test_alter_table', resp.content)
|
|
|
assert_true('Before Alter', resp.content)
|
|
assert_true('Before Alter', resp.content)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Alter name
|
|
# Alter name
|
|
|
resp = self.client.post(reverse("metastore:alter_table",
|
|
resp = self.client.post(reverse("metastore:alter_table",
|
|
|
kwargs={'database': self.db_name, 'table': 'test_alter_table'}),
|
|
kwargs={'database': self.db_name, 'table': 'test_alter_table'}),
|
|
|
{'new_table_name': 'table_altered'})
|
|
{'new_table_name': 'table_altered'})
|
|
|
json_resp = json.loads(resp.content)
|
|
json_resp = json.loads(resp.content)
|
|
|
assert_equal('table_altered', json_resp['data']['name'], json_resp)
|
|
assert_equal('table_altered', json_resp['data']['name'], json_resp)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Alter comment
|
|
# Alter comment
|
|
|
resp = self.client.post(reverse("metastore:alter_table",
|
|
resp = self.client.post(reverse("metastore:alter_table",
|
|
|
kwargs={'database': self.db_name, 'table': 'table_altered'}),
|
|
kwargs={'database': self.db_name, 'table': 'table_altered'}),
|
|
|
{'comment': 'After Alter'})
|
|
{'comment': 'After Alter'})
|
|
|
json_resp = json.loads(resp.content)
|
|
json_resp = json.loads(resp.content)
|
|
|
assert_equal('After Alter', json_resp['data']['comment'], json_resp)
|
|
assert_equal('After Alter', json_resp['data']['comment'], json_resp)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Invalid table name returns error response
|
|
# Invalid table name returns error response
|
|
|
resp = self.client.post(reverse("metastore:alter_table",
|
|
resp = self.client.post(reverse("metastore:alter_table",
|
|
|
kwargs={'database': self.db_name, 'table': 'table_altered'}),
|
|
kwargs={'database': self.db_name, 'table': 'table_altered'}),
|
|
@@ -506,26 +547,26 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
json_resp = json.loads(resp.content)
|
|
json_resp = json.loads(resp.content)
|
|
|
assert_equal(1, json_resp['status'], json_resp)
|
|
assert_equal(1, json_resp['status'], json_resp)
|
|
|
assert_true('Failed to alter table' in json_resp['data'], json_resp)
|
|
assert_true('Failed to alter table' in json_resp['data'], json_resp)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_alter_column(self):
|
|
def test_alter_column(self):
|
|
|
resp = _make_query(self.client, 'CREATE TABLE test_alter_column (before_alter int);', database=self.db_name)
|
|
resp = _make_query(self.client, 'CREATE TABLE test_alter_column (before_alter int);', database=self.db_name)
|
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
resp = self.client.get('/metastore/table/%s/test_alter_column' % self.db_name)
|
|
resp = self.client.get('/metastore/table/%s/test_alter_column' % self.db_name)
|
|
|
assert_true('before_alter', resp.content)
|
|
assert_true('before_alter', resp.content)
|
|
|
assert_true('int', resp.content)
|
|
assert_true('int', resp.content)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Alter name, type and comment
|
|
# Alter name, type and comment
|
|
|
resp = self.client.post(
|
|
resp = self.client.post(
|
|
|
reverse("metastore:alter_column", kwargs={'database': self.db_name, 'table': 'test_alter_column'}),
|
|
reverse("metastore:alter_column", kwargs={'database': self.db_name, 'table': 'test_alter_column'}),
|
|
|
- {'column': 'before_alter', 'new_column_name': 'after_alter', 'new_column_type': 'string', 'comment': 'alter comment'}
|
|
|
|
|
|
|
+ {'column': 'before_alter', 'new_column_name': 'after_alter', 'new_column_type': 'string',
|
|
|
|
|
+ 'comment': 'alter comment'}
|
|
|
)
|
|
)
|
|
|
json_resp = json.loads(resp.content)
|
|
json_resp = json.loads(resp.content)
|
|
|
assert_equal('after_alter', json_resp['data']['name'], json_resp)
|
|
assert_equal('after_alter', json_resp['data']['name'], json_resp)
|
|
|
assert_equal('string', json_resp['data']['type'], json_resp)
|
|
assert_equal('string', json_resp['data']['type'], json_resp)
|
|
|
assert_equal('alter comment', json_resp['data']['comment'], json_resp)
|
|
assert_equal('alter comment', json_resp['data']['comment'], json_resp)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
# Invalid column type returns error response
|
|
# Invalid column type returns error response
|
|
|
resp = self.client.post(reverse("metastore:alter_column",
|
|
resp = self.client.post(reverse("metastore:alter_column",
|
|
|
kwargs={'database': self.db_name, 'table': 'test_alter_column'}),
|
|
kwargs={'database': self.db_name, 'table': 'test_alter_column'}),
|
|
@@ -536,7 +577,6 @@ class TestMetastoreWithHadoop(BeeswaxSampleProvider):
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestParser(object):
|
|
class TestParser(object):
|
|
|
-
|
|
|
|
|
def test_parse_simple(self):
|
|
def test_parse_simple(self):
|
|
|
name = 'simple'
|
|
name = 'simple'
|
|
|
type = 'string'
|
|
type = 'string'
|
|
@@ -544,8 +584,7 @@ class TestParser(object):
|
|
|
column = {'name': name, 'type': type, 'comment': comment}
|
|
column = {'name': name, 'type': type, 'comment': comment}
|
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
|
assert_equal(parse_tree, column)
|
|
assert_equal(parse_tree, column)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_parse_varchar(self):
|
|
def test_parse_varchar(self):
|
|
|
name = 'varchar'
|
|
name = 'varchar'
|
|
|
type = 'varchar(1000)'
|
|
type = 'varchar(1000)'
|
|
@@ -553,8 +592,7 @@ class TestParser(object):
|
|
|
column = {'name': name, 'type': type, 'comment': comment}
|
|
column = {'name': name, 'type': type, 'comment': comment}
|
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
|
assert_equal(parse_tree, column)
|
|
assert_equal(parse_tree, column)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_parse_decimal(self):
|
|
def test_parse_decimal(self):
|
|
|
name = 'simple'
|
|
name = 'simple'
|
|
|
type = 'decimal(12,2)'
|
|
type = 'decimal(12,2)'
|
|
@@ -562,8 +600,7 @@ class TestParser(object):
|
|
|
column = {'name': name, 'type': type, 'comment': comment}
|
|
column = {'name': name, 'type': type, 'comment': comment}
|
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
|
assert_equal(parse_tree, column)
|
|
assert_equal(parse_tree, column)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_parse_array(self):
|
|
def test_parse_array(self):
|
|
|
name = 'array'
|
|
name = 'array'
|
|
|
type = 'array<string>'
|
|
type = 'array<string>'
|
|
@@ -571,8 +608,7 @@ class TestParser(object):
|
|
|
column = {'name': name, 'type': 'array', 'comment': comment, 'item': {'type': 'string'}}
|
|
column = {'name': name, 'type': 'array', 'comment': comment, 'item': {'type': 'string'}}
|
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
|
assert_equal(parse_tree, column)
|
|
assert_equal(parse_tree, column)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_parse_map(self):
|
|
def test_parse_map(self):
|
|
|
name = 'map'
|
|
name = 'map'
|
|
|
type = 'map<string,int>'
|
|
type = 'map<string,int>'
|
|
@@ -580,8 +616,7 @@ class TestParser(object):
|
|
|
column = {'name': name, 'type': 'map', 'comment': comment, 'key': {'type': 'string'}, 'value': {'type': 'int'}}
|
|
column = {'name': name, 'type': 'map', 'comment': comment, 'key': {'type': 'string'}, 'value': {'type': 'int'}}
|
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
|
assert_equal(parse_tree, column)
|
|
assert_equal(parse_tree, column)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_parse_struct(self):
|
|
def test_parse_struct(self):
|
|
|
name = 'struct'
|
|
name = 'struct'
|
|
|
type = 'struct<name:string,age:int>'
|
|
type = 'struct<name:string,age:int>'
|
|
@@ -592,8 +627,7 @@ class TestParser(object):
|
|
|
}
|
|
}
|
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
|
assert_equal(parse_tree, column)
|
|
assert_equal(parse_tree, column)
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_parse_nested(self):
|
|
def test_parse_nested(self):
|
|
|
name = 'nested'
|
|
name = 'nested'
|
|
|
type = 'array<struct<name:string,age:int>>'
|
|
type = 'array<struct<name:string,age:int>>'
|
|
@@ -604,11 +638,11 @@ class TestParser(object):
|
|
|
}
|
|
}
|
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
parse_tree = parser.parse_column(name, type, comment)
|
|
|
assert_equal(parse_tree, column)
|
|
assert_equal(parse_tree, column)
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
def test_parse_nested_with_array(self):
|
|
def test_parse_nested_with_array(self):
|
|
|
name = 'nested'
|
|
name = 'nested'
|
|
|
type = ('struct<fieldname1:bigint,fieldname2:int,fieldname3:int,fieldname4:array<bigint>,'
|
|
type = ('struct<fieldname1:bigint,fieldname2:int,fieldname3:int,fieldname4:array<bigint>,'
|
|
|
- 'fieldname5:bigint,fieldname6:array<struct<array_elem:string>>,fieldname7:string>')
|
|
|
|
|
|
|
+ 'fieldname5:bigint,fieldname6:array<struct<array_elem:string>>,fieldname7:string>')
|
|
|
comment = 'test_parse_nested'
|
|
comment = 'test_parse_nested'
|
|
|
column = {
|
|
column = {
|
|
|
'comment': 'test_parse_nested',
|
|
'comment': 'test_parse_nested',
|
|
@@ -618,7 +652,8 @@ class TestParser(object):
|
|
|
{'type': 'int', 'name': 'fieldname3'},
|
|
{'type': 'int', 'name': 'fieldname3'},
|
|
|
{'item': {'type': 'bigint'}, 'type': 'array', 'name': 'fieldname4'},
|
|
{'item': {'type': 'bigint'}, 'type': 'array', 'name': 'fieldname4'},
|
|
|
{'type': 'bigint', 'name': 'fieldname5'},
|
|
{'type': 'bigint', 'name': 'fieldname5'},
|
|
|
- {'item': {'fields': [{'type': 'string', 'name': 'array_elem'}], 'type': 'struct'}, 'type': 'array', 'name': 'fieldname6'},
|
|
|
|
|
|
|
+ {'item': {'fields': [{'type': 'string', 'name': 'array_elem'}], 'type': 'struct'}, 'type': 'array',
|
|
|
|
|
+ 'name': 'fieldname6'},
|
|
|
{'type': 'string', 'name': 'fieldname7'}
|
|
{'type': 'string', 'name': 'fieldname7'}
|
|
|
],
|
|
],
|
|
|
'type': 'struct', 'name': 'nested'
|
|
'type': 'struct', 'name': 'nested'
|