tests.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # Licensed to Cloudera, Inc. under one
  4. # or more contributor license agreements. See the NOTICE file
  5. # distributed with this work for additional information
  6. # regarding copyright ownership. Cloudera, Inc. licenses this file
  7. # to you under the Apache License, Version 2.0 (the
  8. # "License"); you may not use this file except in compliance
  9. # with the License. You may obtain a copy of the License at
  10. #
  11. # http://www.apache.org/licenses/LICENSE-2.0
  12. #
  13. # Unless required by applicable law or agreed to in writing, software
  14. # distributed under the License is distributed on an "AS IS" BASIS,
  15. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16. # See the License for the specific language governing permissions and
  17. # limitations under the License.
  18. import logging
  19. from nose.tools import assert_true, assert_equal, assert_false
  20. from nose.plugins.skip import SkipTest
  21. from django.utils.encoding import smart_str
  22. from django.contrib.auth.models import User, Group
  23. from django.core.urlresolvers import reverse
  24. import hadoop
  25. from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod_whitespace
  26. from desktop.lib.test_utils import add_permission, grant_access
  27. from useradmin.models import HuePermission, GroupPermission,\
  28. group_has_permission
  29. from beeswax.conf import BROWSE_PARTITIONED_TABLE_LIMIT
  30. from beeswax.views import collapse_whitespace
  31. from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config
  32. from beeswax.models import QueryHistory
  33. from beeswax.server import dbms
  34. from beeswax.test_base import BeeswaxSampleProvider
  35. LOG = logging.getLogger(__name__)
  36. def _make_query(client, query, submission_type="Execute",
  37. udfs=None, settings=None, resources=[],
  38. wait=False, name=None, desc=None, local=True,
  39. is_parameterized=True, max=30.0, database='default', email_notify=False, **kwargs):
  40. """Wrapper around the real make_query"""
  41. res = make_query(client, query, submission_type,
  42. udfs, settings, resources,
  43. wait, name, desc, local, is_parameterized, max, database, email_notify, **kwargs)
  44. # Should be in the history if it's submitted.
  45. if submission_type == 'Execute':
  46. fragment = collapse_whitespace(smart_str(query[:20]))
  47. verify_history(client, fragment=fragment)
  48. return res
  49. class TestMetastoreWithHadoop(BeeswaxSampleProvider):
  50. requires_hadoop = True
  51. def setUp(self):
  52. user = User.objects.get(username='test')
  53. self.db = dbms.get(user, get_query_server_config())
  54. def test_basic_flow(self):
  55. # Default database should exist
  56. response = self.client.get("/metastore/databases")
  57. assert_true("default" in response.context["databases"])
  58. # Table should have been created
  59. response = self.client.get("/metastore/tables/")
  60. assert_true("test" in response.context["tables"])
  61. # Switch databases
  62. response = self.client.get("/metastore/tables/default")
  63. assert_true("test" in response.context["tables"])
  64. response = self.client.get("/metastore/tables/not_there")
  65. assert_false("test" in response.context["tables"])
  66. # And have detail
  67. response = self.client.get("/metastore/table/default/test")
  68. assert_true("foo" in response.content)
  69. assert_true("serdeInfo:SerDeInfo" in response.content, response.content)
  70. # Remember the number of history items. Use a generic fragment 'test' to pass verification.
  71. history_cnt = verify_history(self.client, fragment='test')
  72. # Show table data.
  73. response = self.client.get("/metastore/table/default/test/read", follow=True)
  74. response = wait_for_query_to_finish(self.client, response, max=30.0)
  75. # Note that it may not return all rows at once. But we expect at least 10.
  76. assert_true(len(response.context['results']) > 10)
  77. # Column names
  78. assert_true("foo" in response.content)
  79. assert_true("bar" in response.content)
  80. # This should NOT go into the query history.
  81. assert_equal(verify_history(self.client, fragment='test'), history_cnt,
  82. 'Implicit queries should not be saved in the history')
  83. assert_equal(str(response.context['query_context'][0]), 'table')
  84. assert_equal(str(response.context['query_context'][1]), 'test:default')
  85. def test_describe_view(self):
  86. resp = self.client.get('/metastore/table/default/myview')
  87. assert_equal(None, resp.context['sample'])
  88. assert_true(resp.context['table'].is_view)
  89. assert_true("View" in resp.content)
  90. assert_true("Drop View" in resp.content)
  91. # Breadcrumbs
  92. assert_true("default" in resp.content)
  93. assert_true("myview" in resp.content)
  94. def test_describe_partitions(self):
  95. response = self.client.get("/metastore/table/default/test_partitions")
  96. assert_true("Show Partitions (1)" in response.content, response.content)
  97. response = self.client.get("/metastore/table/default/test_partitions/partitions", follow=True)
  98. assert_true("baz_one" in response.content)
  99. assert_true("boom_two" in response.content)
  100. # Breadcrumbs
  101. assert_true("default" in response.content)
  102. assert_true("test_partitions" in response.content)
  103. assert_true("partitions" in response.content)
  104. # Not partitioned
  105. response = self.client.get("/metastore/table/default/test/partitions", follow=True)
  106. assert_true("is not partitioned." in response.content)
  107. def test_browse_partitioned_table_with_limit(self):
  108. # Limit to 90
  109. finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
  110. try:
  111. response = self.client.get("/metastore/table/default/test_partitions")
  112. assert_true("0x%x" % 89 in response.content, response.content)
  113. assert_false("0x%x" % 90 in response.content, response.content)
  114. finally:
  115. finish()
  116. def test_browse_partitions(self):
  117. response = self.client.get("/metastore/table/default/test_partitions/partitions/0", follow=True)
  118. response = wait_for_query_to_finish(self.client, response, max=30.0)
  119. page_context = [context for context in response.context if 'results' in context][0]
  120. assert_true(len(page_context['results']) > 10)
  121. def test_drop_multi_tables(self):
  122. hql = """
  123. CREATE TABLE test_drop_1 (a int);
  124. CREATE TABLE test_drop_2 (a int);
  125. CREATE TABLE test_drop_3 (a int);
  126. """
  127. resp = _make_query(self.client, hql)
  128. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  129. # Drop them
  130. resp = self.client.get('/metastore/tables/drop/default', follow=True)
  131. assert_true('want to delete' in resp.content, resp.content)
  132. resp = self.client.post('/metastore/tables/drop/default', {u'table_selection': [u'test_drop_1', u'test_drop_2', u'test_drop_3']})
  133. assert_equal(resp.status_code, 302)
  134. def test_drop_multi_databases(self):
  135. hql = """
  136. CREATE DATABASE test_drop_1;
  137. CREATE DATABASE test_drop_2;
  138. CREATE DATABASE test_drop_3;
  139. """
  140. resp = _make_query(self.client, hql)
  141. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  142. # Drop them
  143. resp = self.client.get('/metastore/databases/drop', follow=True)
  144. assert_true('want to delete' in resp.content, resp.content)
  145. resp = self.client.post('/metastore/databases/drop', {u'database_selection': [u'test_drop_1', u'test_drop_2', u'test_drop_3']})
  146. assert_equal(resp.status_code, 302)
  147. def test_load_data(self):
  148. """
  149. Test load data queries.
  150. These require Hadoop, because they ask the metastore
  151. about whether a table is partitioned.
  152. """
  153. # Check that view works
  154. resp = self.client.get("/metastore/table/default/test/load", follow=True)
  155. assert_true('Path' in resp.content)
  156. # Try the submission
  157. self.client.post("/metastore/table/default/test/load", dict(path="/tmp/foo", overwrite=True), follow=True)
  158. query = QueryHistory.objects.latest('id')
  159. assert_equal_mod_whitespace("LOAD DATA INPATH '/tmp/foo' OVERWRITE INTO TABLE `default.test`", query.query)
  160. resp = self.client.post("/metastore/table/default/test/load", dict(path="/tmp/foo", overwrite=False), follow=True)
  161. query = QueryHistory.objects.latest('id')
  162. assert_equal_mod_whitespace("LOAD DATA INPATH '/tmp/foo' INTO TABLE `default.test`", query.query)
  163. # Try it with partitions
  164. resp = self.client.post("/metastore/table/default/test_partitions/load", dict(path="/tmp/foo", partition_0="alpha", partition_1="beta"), follow=True)
  165. query = QueryHistory.objects.latest('id')
  166. assert_equal_mod_whitespace(query.query, "LOAD DATA INPATH '/tmp/foo' INTO TABLE `default.test_partitions` PARTITION (baz='alpha', boom='beta')")
  167. def test_has_write_access_frontend(self):
  168. # HS2 bug: Proxy user substitution is not supported for unsecure hadoop
  169. raise SkipTest
  170. client = make_logged_in_client(username='write_access_frontend', groupname='write_access_frontend', is_superuser=False)
  171. grant_access("write_access_frontend", "write_access_frontend", "metastore")
  172. user = User.objects.get(username='write_access_frontend')
  173. def check(client, assertz):
  174. response = client.get("/metastore/databases")
  175. assertz("Drop</button>" in response.content, response.content)
  176. assertz("Create a new database" in response.content, response.content)
  177. response = client.get("/metastore/tables/")
  178. assertz("Drop</button>" in response.content, response.content)
  179. assertz("Create a new table" in response.content, response.content)
  180. check(client, assert_true)
  181. # Remove access
  182. group, created = Group.objects.get_or_create(name='write_access_frontend')
  183. perm, created = HuePermission.objects.get_or_create(app='metastore', action='read_only_access')
  184. GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
  185. check(client, assert_false)
  186. def test_has_write_access_backend(self):
  187. # HS2 bug: Proxy user substitution is not supported for unsecure hadoop
  188. raise SkipTest
  189. client = make_logged_in_client(username='write_access_backend', groupname='write_access_backend', is_superuser=False)
  190. grant_access("write_access_backend", "write_access_backend", "metastore")
  191. grant_access("write_access_backend", "write_access_backend", "beeswax")
  192. user = User.objects.get(username='write_access_backend')
  193. def check(client, http_code):
  194. resp = _make_query(client, 'CREATE TABLE test_perm_1 (a int);')
  195. resp = wait_for_query_to_finish(client, resp, max=30.0)
  196. resp = client.get('/metastore/tables/drop/default', follow=True)
  197. #assert_true('want to delete' in resp.content, resp.content)
  198. assert_equal(resp.status_code, http_code, resp.content)
  199. resp = client.post('/metastore/tables/drop/default', {u'table_selection': [u'test_perm_1']}, follow=True)
  200. assert_equal(resp.status_code, http_code, resp.content)
  201. check(client, 200)
  202. # Remove access
  203. group, created = Group.objects.get_or_create(name='write_access_backend')
  204. perm, created = HuePermission.objects.get_or_create(app='metastore', action='read_only_access')
  205. GroupPermission.objects.get_or_create(group=group, hue_permission=perm)
  206. check(client, 500)