tests.py 140 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # Licensed to Cloudera, Inc. under one
  4. # or more contributor license agreements. See the NOTICE file
  5. # distributed with this work for additional information
  6. # regarding copyright ownership. Cloudera, Inc. licenses this file
  7. # to you under the Apache License, Version 2.0 (the
  8. # "License"); you may not use this file except in compliance
  9. # with the License. You may obtain a copy of the License at
  10. #
  11. # http://www.apache.org/licenses/LICENSE-2.0
  12. #
  13. # Unless required by applicable law or agreed to in writing, software
  14. # distributed under the License is distributed on an "AS IS" BASIS,
  15. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16. # See the License for the specific language governing permissions and
  17. # limitations under the License.
  18. import cStringIO
  19. import gzip
  20. import json
  21. import logging
  22. import os
  23. import re
  24. import shutil
  25. import socket
  26. import tempfile
  27. import threading
  28. import hadoop
  29. from nose.tools import assert_true, assert_equal, assert_false, assert_not_equal, assert_raises
  30. from nose.plugins.skip import SkipTest
  31. from django.utils.encoding import smart_str
  32. from django.utils.html import escape
  33. from django.contrib.auth.models import User
  34. from django.core.urlresolvers import reverse
  35. from django.db import transaction
  36. from desktop.lib.exceptions_renderable import PopupException
  37. from desktop.conf import \
  38. AUTH_USERNAME as DEFAULT_AUTH_USERNAME, \
  39. AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD, \
  40. AUTH_PASSWORD_SCRIPT as DEFAULT_AUTH_PASSWORD_SCRIPT, \
  41. LDAP_USERNAME, \
  42. LDAP_PASSWORD, \
  43. USE_NEW_EDITOR
  44. from desktop import redaction
  45. from desktop.redaction import logfilter
  46. from desktop.redaction.engine import RedactionPolicy, RedactionRule
  47. from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod_whitespace
  48. from desktop.lib.parameterization import substitute_variables
  49. from desktop.lib.test_utils import grant_access, add_to_group
  50. from desktop.lib.security_util import get_localhost_name
  51. from desktop.lib.test_export_csvxls import _read_xls_sheet_data
  52. from hadoop.fs.hadoopfs import Hdfs
  53. from hadoop.pseudo_hdfs4 import is_live_cluster
  54. import desktop.conf as desktop_conf
  55. import beeswax.create_table
  56. import beeswax.hive_site
  57. import beeswax.models
  58. import beeswax.views
  59. from beeswax import conf, hive_site
  60. from beeswax.common import apply_natural_sort
  61. from beeswax.conf import HIVE_SERVER_HOST, AUTH_USERNAME, AUTH_PASSWORD, AUTH_PASSWORD_SCRIPT
  62. from beeswax.views import collapse_whitespace, _save_design
  63. from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config,\
  64. fetch_query_result_data
  65. from beeswax.design import hql_query, strip_trailing_semicolon
  66. from beeswax.data_export import upload, download
  67. from beeswax.models import SavedQuery, QueryHistory, HQL, HIVE_SERVER2
  68. from beeswax.server import dbms
  69. from beeswax.server.dbms import QueryServerException
  70. from beeswax.server.hive_server2_lib import HiveServerClient,\
  71. PartitionKeyCompatible, PartitionValueCompatible, HiveServerTable,\
  72. HiveServerTColumnValue2
  73. from beeswax.test_base import BeeswaxSampleProvider, is_hive_on_spark
  74. from beeswax.hive_site import get_metastore, hiveserver2_jdbc_url
  75. LOG = logging.getLogger(__name__)
  76. def _list_dir_without_temp_files(fs, target_dir):
  77. return [f for f in fs.listdir(target_dir) if not f.startswith('.')]
  78. def _make_query(client, query, submission_type="Execute",
  79. udfs=None, settings=None, resources=[],
  80. wait=False, name=None, desc=None, local=True,
  81. is_parameterized=True, max=30.0, database='default', email_notify=False, params=None, server_name='beeswax', **kwargs):
  82. res = make_query(client, query, submission_type,
  83. udfs, settings, resources,
  84. wait, name, desc, local, is_parameterized, max, database, email_notify, params, server_name, **kwargs)
  85. # Should be in the history if it's submitted.
  86. if submission_type == 'Execute':
  87. if is_parameterized and params:
  88. query = substitute_variables(query, dict(params))
  89. fragment = collapse_whitespace(smart_str(escape(query[:20])))
  90. verify_history(client, fragment=fragment, server_name=server_name)
  91. return res
  92. def get_csv(client, result_response):
  93. """Get the csv for a query result"""
  94. content = json.loads(result_response.content)
  95. assert_true(content['isSuccess'])
  96. csv_link = '/beeswax/download/%s/csv' % content['id']
  97. csv_resp = client.get(csv_link)
  98. return ''.join(csv_resp.streaming_content)
  99. class TestBeeswaxWithHadoop(BeeswaxSampleProvider):
  100. requires_hadoop = True
  101. def setUp(self):
  102. self.user = User.objects.get(username='test')
  103. add_to_group('test')
  104. self.db = dbms.get(self.user, get_query_server_config())
  105. self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir, '/user/test')
  106. def _verify_query_state(self, state):
  107. """
  108. Verify the state of the latest query.
  109. Return the id of that query
  110. """
  111. resp = self.client.get('/beeswax/query_history')
  112. history = resp.context['page'].object_list[0]
  113. last_state = history.last_state
  114. assert_equal(beeswax.models.QueryHistory.STATE[last_state], state)
  115. return history.id
  116. def test_query_with_error(self):
  117. # Creating a table "again" should not work; error should be displayed.
  118. response = _make_query(self.client, "CREATE TABLE test (foo INT)", database=self.db_name, wait=True)
  119. content = json.loads(response.content)
  120. assert_true("AlreadyExistsException" in content.get('message'), content)
  121. def test_query_with_resource(self):
  122. udf = self.cluster.fs_prefix + "/square.py"
  123. script = self.cluster.fs.open(udf, "w")
  124. script.write(
  125. """#!/usr/bin/python
  126. import sys
  127. for x in sys.stdin:
  128. val = int(x)
  129. print val*val
  130. """)
  131. script.close()
  132. response = _make_query(self.client,
  133. "SELECT TRANSFORM (foo) USING 'python square.py' AS b FROM test",
  134. resources=[("FILE", udf)], local=False, database=self.db_name)
  135. response = wait_for_query_to_finish(self.client, response, max=180.0)
  136. content = fetch_query_result_data(self.client, response)
  137. assert_equal([['0'], ['1'], ['4'], ['9']], content["results"][0:4])
  138. def test_query_with_setting(self):
  139. table_name = 'test_query_with_setting'
  140. response = _make_query(self.client, "CREATE TABLE `%(db)s`.`%(table_name)s` AS SELECT foo+1 FROM test WHERE foo=4" % {'db': self.db_name, 'table_name': table_name},
  141. settings=[("mapred.job.name", "test_query_with_setting"),
  142. ("hive.exec.compress.output", "true")], local=False, database=self.db_name) # Run on MR, because that's how we check it worked.
  143. response = wait_for_query_to_finish(self.client, response, max=180.0)
  144. # Check that we actually got a compressed output
  145. table = self.db.get_table(database=self.db_name, table_name=table_name)
  146. hdfs_loc = Hdfs.urlsplit(table.path_location)
  147. files = self.cluster.fs.listdir(hdfs_loc[2])
  148. assert_true(len(files) >= 1, files)
  149. assert_true(files[0].endswith(".deflate"), files[0])
  150. raise SkipTest
  151. # And check that the name is right...
  152. assert_true("test_query_with_setting" in [ x.profile.name for x in self.cluster.jt.all_jobs().jobs ])
  153. # While we're at it, check that we're running jobs as the correct user on MR.
  154. assert_equal("test",
  155. [ x.profile for x in self.cluster.jt.all_jobs().jobs
  156. if x.profile.name == "test_query_with_setting" ][0].user)
  157. def test_lazy_query_status_update(self):
  158. QUERY = """
  159. SELECT MIN(foo), MAX(foo), SUM(foo) FROM test;
  160. """
  161. wait_for_query_to_finish(self.client, _make_query(self.client, QUERY, local=False, database=self.db_name), max=180.0)
  162. self._verify_query_state(beeswax.models.QueryHistory.STATE.available)
  163. # Make sure expired query states are lazily updated.
  164. resp = self.client.get('/beeswax/query_history')
  165. history = resp.context['page'].object_list[0]
  166. self.db.close_operation(history.get_full_object().get_handle())
  167. resp = self.client.get("/beeswax/execute/query/%s" % history.id)
  168. assert_true(resp.status_code, 302)
  169. resp = self.client.get('/beeswax/query_history')
  170. history = resp.context['page'].object_list[0]
  171. assert_equal(beeswax.models.QueryHistory.STATE[history.last_state], beeswax.models.QueryHistory.STATE.expired)
  172. def test_basic_flow(self):
  173. # Minimal server operation
  174. databases = self.db.get_databases()
  175. assert_true('default' in databases, databases)
  176. assert_true(self.db_name in databases, databases)
  177. assert_true('%s_other' % self.db_name in databases, databases)
  178. # Use GROUP BY to trigger MR job
  179. QUERY = """
  180. SELECT MIN(foo), MAX(foo), SUM(foo) FROM test;
  181. """
  182. response = _make_query(self.client, QUERY, local=False, database=self.db_name)
  183. content = json.loads(response.content)
  184. assert_true('watch_url' in content)
  185. # Check that we report this query as "running" (this query should take a little while).
  186. if not is_hive_on_spark():
  187. self._verify_query_state(beeswax.models.QueryHistory.STATE.running)
  188. response = wait_for_query_to_finish(self.client, response, max=180.0)
  189. content = fetch_query_result_data(self.client, response)
  190. assert_equal([0, 255, 32640], content["results"][0], content["results"][0])
  191. assert_equal(['INT_TYPE', 'INT_TYPE', 'BIGINT_TYPE'], [col['type'] for col in content["columns"]])
  192. self._verify_query_state(beeswax.models.QueryHistory.STATE.available)
  193. # Query multi-page request
  194. QUERY = """
  195. SELECT * FROM test
  196. """
  197. response = _make_query(self.client, QUERY, name='select star', local=False, database=self.db_name)
  198. response = wait_for_query_to_finish(self.client, response)
  199. content = fetch_query_result_data(self.client, response)
  200. assert_true([99, u'0x63'] in content['results'], content['results'])
  201. assert_true(content["has_more"])
  202. response = self.client.get("/beeswax/results/%s/%s?format=json" % (content["id"], content["next_row"]))
  203. content = json.loads(response.content)
  204. assert_true([199, u'0xc7'] in content['results'], content['results'])
  205. response = self.client.get("/beeswax/results/%s/0?format=json" % (content["id"]))
  206. content = json.loads(response.content)
  207. assert_true([99, u'0x63'] in content['results'])
  208. assert_equal(0, len(content["hadoop_jobs"]), "SELECT * shouldn't have started jobs.")
  209. # Download the data
  210. response = self.client.get(content["download_urls"]["csv"])
  211. # Header line plus data lines...
  212. assert_equal(257, ''.join(response.streaming_content).count("\n"))
  213. def test_api_get_session(self):
  214. session = None
  215. try:
  216. # Create open session
  217. session = self.db.open_session(self.user)
  218. resp = self.client.get(reverse("beeswax:api_get_session"))
  219. data = json.loads(resp.content)
  220. assert_true('properties' in data, data)
  221. assert_true('session' in data, data)
  222. assert_true('id' in data['session'], data['session'])
  223. finally:
  224. if session is not None:
  225. try:
  226. self.db.close_session(session)
  227. except Exception:
  228. pass
  229. def test_api_close_session(self):
  230. session = None
  231. try:
  232. # Create open session
  233. session = self.db.open_session(self.user)
  234. resp = self.client.post(reverse("beeswax:api_close_session", kwargs={'session_id': session.id}))
  235. data = json.loads(resp.content)
  236. assert_equal(0, data['status'])
  237. assert_true('session' in data)
  238. assert_equal(4, data['session']['status'])
  239. # Closed sessions will return error response
  240. resp = self.client.post(reverse("beeswax:api_close_session", kwargs={'session_id': session.id}))
  241. data = json.loads(resp.content)
  242. assert_equal(-1, data['status'])
  243. finally:
  244. if session is not None:
  245. try:
  246. self.db.close_session(session)
  247. except Exception:
  248. pass
  249. def test_result_escaping(self):
  250. # Check for XSS and NULL display
  251. QUERY = """
  252. SELECT 'abc', 1.0, 1=1, 1, 1/0, '<a>lala</a>lulu', 'some spaces' from test LIMIT 3;
  253. """
  254. response = _make_query(self.client, QUERY, local=False, database=self.db_name)
  255. content = json.loads(response.content)
  256. assert_true('watch_url' in content)
  257. response = wait_for_query_to_finish(self.client, response, max=180.0)
  258. content = fetch_query_result_data(self.client, response)
  259. assert_equal([
  260. [u'abc', 1.0, True, 1, u'NULL', u'&lt;a&gt;lala&lt;/a&gt;lulu', 'some&nbsp;&nbsp;&nbsp;spaces'],
  261. [u'abc', 1.0, True, 1, u'NULL', u'&lt;a&gt;lala&lt;/a&gt;lulu', 'some&nbsp;&nbsp;&nbsp;spaces'],
  262. [u'abc', 1.0, True, 1, u'NULL', u'&lt;a&gt;lala&lt;/a&gt;lulu', 'some&nbsp;&nbsp;&nbsp;spaces'],
  263. ], content["results"], content)
  264. def test_result_nullification(self):
  265. QUERY = """
  266. CREATE TABLE test_result_nullification (a int);
  267. INSERT INTO TABLE test_result_nullification
  268. VALUES
  269. (1), (1), (1), (1), (1), (1), (1), (1),
  270. (2), (2), (2), (2), (2), (2), (2), (2),
  271. (NULL), (3), (3), (3), (3), (3), (3), (3),
  272. (4), (4), (4), (4), (4), (4), (4), (4),
  273. (5), (5), (5), (5), (5), (5), (5), (5),
  274. (6), (6), (6), (6), (6), (6), (6), (6);
  275. """
  276. response = _make_query(self.client, QUERY, local=False, database=self.db_name)
  277. content = json.loads(response.content)
  278. assert_true('watch_url' in content)
  279. response = wait_for_query_to_finish(self.client, response, max=180.0)
  280. content = fetch_query_result_data(self.client, response)
  281. QUERY = """
  282. SELECT * FROM test_result_nullification;
  283. """
  284. response = _make_query(self.client, QUERY, local=False, database=self.db_name)
  285. content = json.loads(response.content)
  286. assert_true('watch_url' in content)
  287. response = wait_for_query_to_finish(self.client, response, max=180.0)
  288. content = fetch_query_result_data(self.client, response)
  289. assert_equal([
  290. [1], [1], [1], [1], [1], [1], [1], [1],
  291. [2], [2], [2], [2], [2], [2], [2], [2],
  292. [u'NULL'], [3], [3], [3], [3], [3], [3], [3],
  293. [4], [4], [4], [4], [4], [4], [4], [4],
  294. [5], [5], [5], [5], [5], [5], [5], [5],
  295. [6], [6], [6], [6], [6], [6], [6], [6]
  296. ], content["results"], content)
  297. def test_query_with_udf(self):
  298. """
  299. Testing query with udf
  300. """
  301. response = _make_query(self.client, "SELECT my_sqrt(foo), my_float(foo) FROM test where foo=4 GROUP BY foo", # Force MR job with GROUP BY
  302. udfs=[('my_sqrt', 'org.apache.hadoop.hive.ql.udf.UDFSqrt'),
  303. ('my_float', 'org.apache.hadoop.hive.ql.udf.UDFToFloat')], local=False, database=self.db_name)
  304. response = wait_for_query_to_finish(self.client, response, max=60.0)
  305. content = fetch_query_result_data(self.client, response)
  306. assert_equal([2.0, 4.0], content["results"][0])
  307. log = content['log']
  308. if not is_hive_on_spark():
  309. assert_true(search_log_line('map = 100%', log), log)
  310. assert_true(search_log_line('reduce = 100%', log), log)
  311. # Test job extraction while we're at it
  312. assert_equal(1, len(content["hadoop_jobs"]), "Should have started 1 job and extracted it.")
  313. def test_query_with_remote_udf(self):
  314. """
  315. UDF is on HDFS. This was implemented as part of HIVE-1157.
  316. """
  317. # BeeswaxTest.jar is gone
  318. raise SkipTest
  319. src = file(os.path.join(os.path.dirname(__file__), "..", "..", "java-lib", "BeeswaxTest.jar"))
  320. udf = self.cluster.fs_prefix + "hive1157.jar"
  321. dest = self.cluster.fs.open(udf, "w")
  322. shutil.copyfileobj(src, dest)
  323. dest.close()
  324. src.close()
  325. # Beware that this doesn't work with mapred.job.tracker=local :/
  326. response = _make_query(self.client, "SELECT cube(foo) FROM test WHERE foo=4",
  327. udfs=[('cube', 'com.cloudera.beeswax.CubeSampleUDF')],
  328. resources=[('JAR', udf)], local=False, database=self.db_name)
  329. response = wait_for_query_to_finish(self.client, response, max=60.0)
  330. assert_equal(["64"], response.context["results"][0])
  331. def test_query_with_simple_errors(self):
  332. hql = "SELECT KITTENS ARE TASTY"
  333. resp = _make_query(self.client, hql, name='tasty kittens', wait=False, local=False, database=self.db_name)
  334. assert_true("ParseException line" in json.loads(resp.content)["message"])
  335. # Watch page will fail as operationHandle=None
  336. self._verify_query_state(beeswax.models.QueryHistory.STATE.failed)
  337. def test_sync_query_exec(self):
  338. # Execute Query Synchronously, set fetch size and fetch results
  339. # verify the size of resultset,
  340. hql = """
  341. SELECT foo FROM `%(db)s`.`test`;
  342. """ % {'db': self.db_name}
  343. query = hql_query(hql)
  344. handle = self.db.execute_and_wait(query)
  345. results = self.db.fetch(handle, True, 5)
  346. row_list = list(results.rows())
  347. assert_equal(len(row_list), 5)
  348. self.db.close(handle)
  349. def test_sync_query_error(self):
  350. # We don't use synchronous queries anywhere.
  351. # It used to call BeeswaxService.executeAndWait()
  352. raise SkipTest
  353. # Execute incorrect Query , verify the error code and sqlstate
  354. hql = """
  355. SELECT FROM `%(db)s`.`zzzzz`
  356. """ % {'db': self.db_name}
  357. query = hql_query(hql)
  358. try:
  359. self.db.execute_and_wait(query)
  360. except QueryServerException, bex:
  361. assert_equal(bex.errorCode, 40000)
  362. assert_equal(bex.SQLState, "42000")
  363. def test_fetch_configuration(self):
  364. class MockClient:
  365. """Check if sent fetch correctly supports start_over."""
  366. def __init__(self, support_start_over):
  367. self.support_start_over = support_start_over
  368. def fetch(self, query_id, start_over, fetch_size):
  369. assert_equal(self.support_start_over, start_over)
  370. class Result: pass
  371. res = Result()
  372. res.ready = False
  373. return res
  374. class ConfigVariable:
  375. def __init__(self, **entries):
  376. self.__dict__.update(entries)
  377. client = self.db
  378. prev_get_default_configuration = client.get_default_configuration
  379. prev_client = client.client
  380. try:
  381. client.client = MockClient(True)
  382. client.get_default_configuration = lambda a: []
  383. client.fetch(None, True, 5)
  384. client.client = MockClient(False)
  385. client.get_default_configuration = lambda a: []
  386. client.fetch(None, False, 5)
  387. client.client = MockClient(True)
  388. client.get_default_configuration = lambda a: [ConfigVariable(key='support_start_over', value='true')]
  389. client.fetch(None, True, 5)
  390. client.client = MockClient(False)
  391. client.get_default_configuration = lambda a: [ConfigVariable(key='support_start_over', value='false')]
  392. client.fetch(None, True, 5)
  393. finally:
  394. client.get_default_configuration = prev_get_default_configuration
  395. client.client = prev_client
  396. def test_parameterization(self):
  397. response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", is_parameterized=False, database=self.db_name)
  398. content = json.loads(response.content)
  399. # Assert no parameterization was offered
  400. assert_true('watch_url' in content, content)
  401. data = {
  402. 'query-query': "SELECT foo FROM test WHERE foo='$x' and bar='$y'",
  403. 'query-database': self.db_name
  404. }
  405. response = self.client.post(reverse('beeswax:api_parameters'), data)
  406. content = json.loads(response.content)
  407. assert_equal([
  408. {'parameter': 'parameterization-x', 'name': 'x'},
  409. {'parameter': 'parameterization-y', 'name': 'y'}
  410. ], content['parameters'], content)
  411. # Now fill it out
  412. response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", params=[('x', '1'), ('y', '2')], database=self.db_name)
  413. content = json.loads(response.content)
  414. assert_true('watch_url' in content, content)
  415. query_history = QueryHistory.get(content['id'])
  416. # Check that substitution happened!
  417. assert_equal("SELECT foo FROM test WHERE foo='1' and bar='2'", query_history.query)
  418. # Check that error handling is reasonable
  419. response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", params=[('x', "'_this_is_not SQL "), ('y', '2')], database=self.db_name)
  420. content = json.loads(response.content)
  421. assert_true("FAILED: ParseException" in content.get('message'), content)
  422. # Check multi DB with a non default DB
  423. other_db = '%s_other' % self.db_name
  424. response = _make_query(self.client, "CREATE TABLE test (foo INT, bar STRING)", database=other_db)
  425. response = wait_for_query_to_finish(self.client, response)
  426. response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", database=other_db)
  427. response = _make_query(self.client, "SELECT foo FROM test WHERE foo='$x' and bar='$y'", database=other_db,
  428. params=[('x', '1'), ('y', '2')])
  429. content = json.loads(response.content)
  430. assert_true('watch_url' in content, content)
  431. query_history = QueryHistory.get(content['id'])
  432. assert_equal(other_db, query_history.design.get_design().query['database'])
  433. def test_explain_query(self):
  434. c = self.client
  435. response = _make_query(c, "SELECT KITTENS ARE TASTY", submission_type="Explain", database=self.db_name)
  436. assert_true("ParseException line" in json.loads(response.content)["message"])
  437. CREATE_TABLE = "CREATE TABLE `%(db)s`.`test_explain` (foo INT, bar STRING);" % {'db': self.db_name}
  438. response = _make_query(c, CREATE_TABLE, database=self.db_name)
  439. wait_for_query_to_finish(c, response)
  440. response = _make_query(c, "SELECT SUM(foo) FROM `%(db)s`.`test_explain`" % {'db': self.db_name}, submission_type="Explain") # Need to prefix database in Explain
  441. explanation = json.loads(response.content)['explanation']
  442. assert_true('STAGE DEPENDENCIES:' in explanation, explanation)
  443. assert_true('STAGE PLANS:' in explanation, explanation)
  444. def test_explain_query_i18n(self):
  445. if is_live_cluster():
  446. raise SkipTest('HUE-2884: Skipping test because we cannot guarantee live cluster supports utf8')
  447. query = u"SELECT foo FROM `%(db)s`.`test_utf8` WHERE bar='%(val)s'" % {'val': unichr(200), 'db': self.db_name}
  448. response = _make_query(self.client, query, submission_type="Explain")
  449. explanation = json.loads(response.content)['explanation']
  450. assert_true('STAGE DEPENDENCIES:' in explanation, explanation)
  451. assert_true('STAGE PLANS:' in explanation, explanation)
  452. def test_query_i18n(self):
  453. # Test fails because HIVE_PLAN cannot be found and raises FileNotFoundException
  454. # because of a Hive bug.
  455. raise SkipTest
  456. # Selecting from utf-8 table should get correct result
  457. query = u"SELECT * FROM `%(db)s`.`test_utf8` WHERE bar='%(val)s'" % {'val': unichr(200), 'db': self.db_name}
  458. response = _make_query(self.client, query, wait=True, database=self.db_name)
  459. assert_equal(["200", unichr(200)], response.context["results"][0], "selecting from utf-8 table should get correct result")
  460. csv = get_csv(self.client, response)
  461. assert_equal('"200","%s"' % (unichr(200).encode('utf-8'),), csv.split()[1])
  462. # Selecting from latin1 table should not blow up
  463. query = u"SELECT * FROM `%(db)s`.`test_latin1` WHERE bar='%(val)s'" % {'val': unichr(200), 'db': self.db_name}
  464. response = _make_query(self.client, query, wait=True, database=self.db_name)
  465. assert_true('results' in response.context, "selecting from latin1 table should not blow up")
  466. # Describe table should be fine with non-ascii comment
  467. response = self.client.get('/beeswax/table/%(db)s/test_utf8' % {'db': self.db_name})
  468. assert_equal(response.context['table'].parameters['comment'], self.get_i18n_table_comment())
  469. def _parallel_query_helper(self, i, result_holder, lock, num_tasks):
  470. client = make_logged_in_client()
  471. try:
  472. q = "SELECT foo+" + str(i + 1) + " FROM test WHERE foo < 2"
  473. LOG.info("Starting " + str(i) + ": " + q)
  474. response = _make_query(client, q, local=False)
  475. response = wait_for_query_to_finish(client, response, max=(240.0 * num_tasks))
  476. lock.acquire()
  477. result_holder[i] = response
  478. lock.release()
  479. LOG.info("Finished: " + str(i))
  480. except Exception, e:
  481. LOG.exception("Saw exception in child thread: %s" % e)
  482. def test_multiple_statements_no_result_set(self):
  483. hql = """
  484. CREATE TABLE test_multiple_statements_1 (a int);
  485. CREATE TABLE test_multiple_statements_2 (a int);
  486. DROP TABLE test_multiple_statements_1;
  487. DROP TABLE test_multiple_statements_2;
  488. """
  489. resp = _make_query(self.client, hql, database=self.db_name)
  490. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  491. content = json.loads(resp.content)
  492. history_id = content['id']
  493. query_history = QueryHistory.get(id=history_id)
  494. resp = self.client.get("/beeswax/results/%s/0?format=json" % history_id)
  495. content = json.loads(resp.content)
  496. assert_equal('DROP TABLE test_multiple_statements_2', query_history.get_current_statement(), content)
  497. def test_multiple_statements_with_result_set(self):
  498. hql = """
  499. SELECT foo FROM test;
  500. SELECT count(*) FROM test;
  501. """
  502. resp = _make_query(self.client, hql, database=self.db_name)
  503. content = json.loads(resp.content)
  504. assert_true('watch_url' in content, content)
  505. watch_url = content['watch_url']
  506. assert_equal('SELECT foo FROM test', content.get('statement'), content)
  507. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  508. content = fetch_query_result_data(self.client, resp)
  509. assert_false(content.get('is_finished'), content)
  510. resp = self.client.post(watch_url, {'next': True})
  511. content = json.loads(resp.content)
  512. assert_equal('SELECT count(*) FROM test', content.get('statement'), content)
  513. def test_multiple_statements_various_queries(self):
  514. hql = """
  515. CREATE TABLE test_multiple_statements_2 (a int);
  516. DROP TABLE test_multiple_statements_2;
  517. SELECT foo FROM test;
  518. """
  519. resp = _make_query(self.client, hql, database=self.db_name)
  520. content = json.loads(resp.content)
  521. assert_equal('CREATE TABLE test_multiple_statements_2 (a int)', content.get('statement'), content)
  522. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  523. content = json.loads(resp.content)
  524. assert_equal('SELECT foo FROM test', content.get('statement'), content)
  525. content = fetch_query_result_data(self.client, resp)
  526. assert_true(content.get('is_finished'), content)
  527. def test_multiple_statements_with_next_button(self):
  528. hql = """
  529. show tables;
  530. select * from test
  531. """
  532. resp = _make_query(self.client, hql, database=self.db_name)
  533. # First statement
  534. content = json.loads(resp.content)
  535. watch_url = content['watch_url']
  536. assert_equal('show tables', content.get('statement'), content)
  537. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  538. content = fetch_query_result_data(self.client, resp)
  539. assert_true([u'test'] in content.get('results'), content)
  540. # Next statement
  541. resp = self.client.post(watch_url, {'next': True, 'query-query': hql})
  542. content = json.loads(resp.content)
  543. assert_equal('select * from test', content.get('statement'), content)
  544. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  545. content = fetch_query_result_data(self.client, resp)
  546. assert_true([0, u'0x0'] in content.get('results'), content)
  547. def test_multiple_statements_with_params(self):
  548. hql = """
  549. select ${x} from test;
  550. select ${y} from test;
  551. """
  552. resp = _make_query(self.client, hql, params=[('x', '1'), ('y', '2')], database=self.db_name)
  553. # First statement
  554. content = json.loads(resp.content)
  555. watch_url = content['watch_url']
  556. assert_equal('select ${x} from test', content.get('statement'), content)
  557. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  558. content = fetch_query_result_data(self.client, resp)
  559. # Next statement
  560. resp = self.client.post(watch_url, {'next': True, 'query-query': hql})
  561. content = json.loads(resp.content)
  562. assert_equal('select ${y} from test', content.get('statement'), content)
  563. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  564. def test_multiple_statements_with_error(self):
  565. hql = """
  566. show tables;
  567. select * from
  568. """
  569. resp = _make_query(self.client, hql, database=self.db_name)
  570. content = json.loads(resp.content)
  571. watch_url = content['watch_url']
  572. assert_equal('show tables', content.get('statement'), content)
  573. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  574. resp = self.client.post(watch_url, {'next': True, 'query-query': hql})
  575. content = json.loads(resp.content)
  576. assert_true('Error while compiling statement' in content.get('message'), content)
  577. hql = """
  578. show tables;
  579. select * from test
  580. """
  581. # Retry where we were with the statement fixed
  582. resp = self.client.post(watch_url, {'next': True, 'query-query': hql})
  583. content = json.loads(resp.content)
  584. assert_equal('select * from test', content.get('statement'), content)
  585. def test_parallel_queries(self):
  586. """
  587. Test that we can issue two queries to the BeeswaxServer in parallel.
  588. However, the test assertion has been plagued by the django test framework,
  589. which does not set request.context in a thread safe manner.
  590. So we check the results by looking at the csv files.
  591. """
  592. raise SkipTest # sqlite does not support concurrent transaction
  593. PARALLEL_TASKS = 2
  594. responses = [ None ] * PARALLEL_TASKS
  595. threads = []
  596. # Protects responses
  597. lock = threading.Lock()
  598. for i in range(PARALLEL_TASKS):
  599. t = threading.Thread(target=self._parallel_query_helper,
  600. args=(i, responses, lock, PARALLEL_TASKS))
  601. t.start()
  602. threads.append(t)
  603. for t in threads:
  604. t.join()
  605. # Commit transactions to be sure that QueryHistory up to date
  606. transaction.commit()
  607. for i in range(PARALLEL_TASKS):
  608. csv = get_csv(self.client, responses[i])
  609. # We get 3 rows: Column header, and 2 rows of results in double quotes
  610. answer = [ int(data.strip('"')) for data in csv.split()[1:] ]
  611. assert_equal( [ i + 1, i + 2 ], answer)
  612. def test_data_export_limit_clause(self):
  613. limit = 3
  614. hql = 'SELECT foo FROM `%(db)s`.`test` limit %(limit)d' % {'limit': limit, 'db': self.db_name}
  615. query = hql_query(hql)
  616. handle = self.db.execute_and_wait(query)
  617. # Get the result in csv. Should have 3 + 1 header row.
  618. csv_resp = download(handle, 'csv', self.db)
  619. csv_content = ''.join(csv_resp.streaming_content)
  620. assert_equal(len(csv_content.strip().split('\n')), limit + 1)
  621. def test_query_done_cb(self):
  622. hql = 'SELECT * FROM `%(db)s`.`test`' % {'db': self.db_name}
  623. query = hql_query(hql)
  624. query._data_dict['query']['email_notify'] = False
  625. query_history = self.db.execute_and_watch(query)
  626. response = self.client.get('/beeswax/query_cb/done/%s' % query_history.server_id)
  627. assert_true('email_notify is false' in response.content, response.content)
  628. query = hql_query(hql)
  629. query._data_dict['query']['email_notify'] = True
  630. query_history = self.db.execute_and_watch(query)
  631. response = self.client.get('/beeswax/query_cb/done/%s' % query_history.server_id,)
  632. assert_true('sent' in response.content, response.content)
  633. response = self.client.get('/beeswax/query_cb/done/blahblahblah')
  634. assert_true('QueryHistory matching query does not exist' in response.content, response.content)
  635. def test_data_export(self):
  636. hql = 'SELECT * FROM `%(db)s`.`test`' % {'db': self.db_name}
  637. query = hql_query(hql)
  638. # Get the result in xls.
  639. handle = self.db.execute_and_wait(query)
  640. resp = download(handle, 'xls', self.db)
  641. sheet_data = _read_xls_sheet_data(resp)
  642. num_cols = len(sheet_data[0])
  643. # It should have 257 lines (256 + header)
  644. assert_equal(len(sheet_data), 257, sheet_data)
  645. # Get the result in csv.
  646. query = hql_query(hql)
  647. handle = self.db.execute_and_wait(query)
  648. resp = download(handle, 'csv', self.db)
  649. csv_resp = ''.join(resp.streaming_content)
  650. csv_data = [[int(col) if col.isdigit() else col for col in row.split(',')] for row in csv_resp.strip().split('\r\n')]
  651. assert_equal(sheet_data, csv_data)
  652. # Test max cell limit truncation
  653. finish = conf.DOWNLOAD_CELL_LIMIT.set_for_testing(num_cols*5)
  654. try:
  655. hql = 'SELECT * FROM `%(db)s`.`test`' % {'db': self.db_name}
  656. query = hql_query(hql)
  657. handle = self.db.execute_and_wait(query)
  658. resp = download(handle, 'xls', self.db)
  659. sheet_data = _read_xls_sheet_data(resp)
  660. # It should have 5 lines
  661. assert_equal(len(sheet_data), 5, sheet_data)
  662. finally:
  663. finish()
  664. def test_data_upload(self):
  665. hql = 'SELECT * FROM `%(db)s`.`test`' % {'db': self.db_name}
  666. query = hql_query(hql)
  667. handle = self.db.execute_and_wait(query)
  668. csv_file = self.cluster.fs_prefix + '/test_data_upload.csv'
  669. upload(csv_file, handle, self.user, self.db, self.cluster.fs)
  670. assert_true(self.cluster.fs.exists(csv_file))
  671. def test_designs(self):
  672. if is_live_cluster():
  673. raise SkipTest('HUE-2902: Skipping because test is not reentrant')
  674. cli = self.client
  675. # An auto hql design should be created, and it should ignore the given name and desc
  676. _make_query(self.client, 'SELECT bogus FROM test', name='mydesign', desc='hyatt', database=self.db_name)
  677. resp = cli.get('/beeswax/list_designs')
  678. n_designs = len(resp.context['page'].object_list)
  679. # Retrieve that design. It's the first one since it's most recent
  680. design = beeswax.models.SavedQuery.objects.all()[0]
  681. resp = cli.get('/beeswax/execute/design/%s' % design.id)
  682. assert_true('query' in resp.context, resp.context)
  683. assert_true(resp.context['query'] is None, resp.context)
  684. assert_equal(design, resp.context['design'], resp.context)
  685. # Retrieve that query history. It's the first one since it's most recent
  686. query_history = beeswax.models.QueryHistory.objects.all()[0]
  687. resp = cli.get('/beeswax/execute/query/%s' % query_history.id)
  688. assert_true('query' in resp.context, resp.context)
  689. assert_true(resp.context['query'] is not None, resp.context)
  690. assert_true('design' in resp.context, resp.context)
  691. assert_true(resp.context['design'] is not None, resp.context)
  692. resp = cli.get(reverse('beeswax:api_fetch_saved_design', kwargs={'design_id': design.id}))
  693. content = json.loads(resp.content)
  694. assert_true('SELECT bogus FROM test' in content['design']['query'], content)
  695. # Make a valid auto hql design
  696. resp = _make_query(self.client, 'SELECT * FROM test', database=self.db_name)
  697. wait_for_query_to_finish(self.client, resp, max=60.0)
  698. resp = cli.get('/beeswax/list_designs')
  699. nplus_designs = len(resp.context['page'].object_list)
  700. assert_true(nplus_designs == n_designs, 'Auto design should not show up in list_designs')
  701. # Test explicit save and use another DB
  702. query = 'MORE BOGUS JUNKS FROM test'
  703. other_db = '%s_other' % self.db_name
  704. resp = _make_query(self.client, query, name='rubbish', submission_type='Save', database=other_db)
  705. content = json.loads(resp.content)
  706. assert_equal(0, content['status'])
  707. assert_true('design_id' in content, content)
  708. resp = cli.get('/beeswax/list_designs')
  709. assert_true('rubbish' in resp.content, resp.content)
  710. nplusplus_designs = len(resp.context['page'].object_list)
  711. assert_true(nplusplus_designs > nplus_designs)
  712. # Retrieve that design and check correct DB is selected
  713. design = beeswax.models.SavedQuery.objects.filter(name='rubbish')[0]
  714. resp = cli.get(reverse('beeswax:api_fetch_saved_design', kwargs={'design_id': design.id}))
  715. content = json.loads(resp.content)
  716. assert_true(query in content['design']['query'], content)
  717. assert_equal('', content['design']['desc'], content)
  718. assert_equal(other_db, content['design']['database'], content)
  719. # Clone the rubbish design
  720. len_before = len(beeswax.models.SavedQuery.objects.filter(name__contains='rubbish'))
  721. resp = cli.get('/beeswax/clone_design/%s' % (design.id,))
  722. len_after = len(beeswax.models.SavedQuery.objects.filter(name__contains='rubbish'))
  723. assert_equal(len_before + 1, len_after)
  724. # Make 3 more designs
  725. resp = cli.get('/beeswax/clone_design/%s' % (design.id,))
  726. resp = cli.get('/beeswax/clone_design/%s' % (design.id,))
  727. resp = cli.get('/beeswax/clone_design/%s' % (design.id,))
  728. designs = beeswax.models.SavedQuery.objects.filter(name__contains='rubbish')[:3]
  729. # Delete a design
  730. resp = cli.get('/beeswax/delete_designs')
  731. assert_true('Delete design(s)' in resp.content, resp.content)
  732. resp = cli.post('/beeswax/delete_designs', {u'designs_selection': [design.id]})
  733. assert_equal(resp.status_code, 302)
  734. # Delete designs
  735. design_ids = map(str, designs.values_list('id', flat=True))
  736. resp = cli.get('/beeswax/delete_designs', {u'designs_selection': design_ids})
  737. assert_true('Delete design(s)' in resp.content, resp.content)
  738. resp = cli.post('/beeswax/delete_designs', {u'designs_selection': design_ids})
  739. assert_equal(resp.status_code, 302)
  740. # Helper to test the view, filtering, etc
  741. def do_view(param):
  742. resp = cli.get('/beeswax/list_designs?' + param)
  743. assert_true(len(resp.context['page'].object_list) >= 0) # Make the query run
  744. return resp
  745. do_view('user=test')
  746. do_view('text=whatever')
  747. do_view('type=hql')
  748. do_view('sort=date')
  749. do_view('sort=-date')
  750. do_view('sort=name')
  751. do_view('sort=-name')
  752. do_view('sort=desc')
  753. do_view('sort=-desc')
  754. do_view('sort=type')
  755. do_view('sort=-type')
  756. do_view('sort=name&user=bogus')
  757. resp = do_view('sort=-type&user=test&type=hql&text=Rubbish')
  758. assert_true('rubbish' in resp.content)
  759. # Test personal saved queries permissions
  760. client_me = make_logged_in_client(username='its_me', is_superuser=False, groupname='test')
  761. grant_access("its_me", "test", "beeswax")
  762. _make_query(client_me, "select one", name='client query 1', submission_type='Save')
  763. _make_query(client_me, "select two", name='client query 2', submission_type='Save')
  764. # TODO in HUE-1589
  765. raise SkipTest
  766. finish = conf.SHARE_SAVED_QUERIES.set_for_testing(True)
  767. try:
  768. resp = client_me.get('/beeswax/list_designs')
  769. assert_true('client query 1' in resp.content, resp.content)
  770. assert_true('client query 2' in resp.content, resp.content)
  771. finally:
  772. finish()
  773. finish = conf.SHARE_SAVED_QUERIES.set_for_testing(False)
  774. try:
  775. resp = client_me.get('/beeswax/list_designs')
  776. assert_true('client query 1' in resp.content)
  777. assert_true('client query 2' in resp.content)
  778. finally:
  779. finish()
  780. client_me.logout()
  781. # Login as someone else
  782. client_not_me = make_logged_in_client(username='not_me', is_superuser=False, groupname='test')
  783. grant_access("not_me", "test", "beeswax")
  784. finish = conf.SHARE_SAVED_QUERIES.set_for_testing(True)
  785. try:
  786. resp = client_not_me.get('/beeswax/list_designs')
  787. assert_true('client query 1' in resp.content)
  788. assert_true('client query 2' in resp.content)
  789. finally:
  790. finish()
  791. finish = conf.SHARE_SAVED_QUERIES.set_for_testing(False)
  792. try:
  793. resp = client_not_me.get('/beeswax/list_designs')
  794. assert_true('client query 1' not in resp.content)
  795. assert_true('client query 2' not in resp.content)
  796. finally:
  797. finish()
  798. client_not_me.logout()
  799. # Login as super user
  800. client_admin = make_logged_in_client('admin', is_superuser=True)
  801. finish = conf.SHARE_SAVED_QUERIES.set_for_testing(True)
  802. try:
  803. resp = client_admin.get('/beeswax/list_designs')
  804. assert_true('client query 1' in resp.content)
  805. assert_true('client query 2' in resp.content)
  806. finally:
  807. finish()
  808. finish = conf.SHARE_SAVED_QUERIES.set_for_testing(False)
  809. try:
  810. resp = client_admin.get('/beeswax/list_designs')
  811. assert_true('client query 1' in resp.content)
  812. assert_true('client query 2' in resp.content)
  813. finally:
  814. finish()
  815. client_admin.logout()
  816. def test_my_queries(self):
  817. # Explicit save a design
  818. _make_query(self.client, "select noHQL", name='my rubbish kuery', submission_type='Save', database=self.db_name)
  819. # Run something
  820. _make_query(self.client, "Even More Bogus Junk", database=self.db_name)
  821. resp = self.client.get('/beeswax/my_queries')
  822. assert_true('my rubbish kuery' in resp.content, resp.content)
  823. assert_true('Even More Bogus Junk' in resp.content)
  824. # Login as someone else
  825. client_not_me = make_logged_in_client('not_me', groupname='test')
  826. grant_access("not_me", "test", "beeswax")
  827. resp = client_not_me.get('/beeswax/my_queries')
  828. assert_true('my rubbish kuery' not in resp.content)
  829. assert_true('Even More Bogus Junk' not in resp.content)
  830. client_not_me.logout()
  831. def test_save_results_to_dir(self):
  832. def save_and_verify(select_resp, target_dir, verify=True):
  833. content = json.loads(select_resp.content)
  834. qid = content['id']
  835. save_data = {
  836. 'type': 'hdfs-directory',
  837. 'path': target_dir
  838. }
  839. resp = self.client.post('/beeswax/api/query/%s/results/save/hdfs/directory' % qid, save_data, follow=True)
  840. content = json.loads(resp.content)
  841. if content['status'] == 0:
  842. success_url = content['success_url']
  843. resp = self.client.get(content['watch_url'], follow=True)
  844. resp = wait_for_query_to_finish(self.client, resp, max=120)
  845. resp.success_url = success_url # Hack until better API
  846. # Check that data is right
  847. if verify:
  848. target_ls = _list_dir_without_temp_files(self.cluster.fs, target_dir)
  849. assert_equal(len(target_ls), 1)
  850. data_buf = ""
  851. for target in target_ls:
  852. target_file = self.cluster.fs.open(target_dir + '/' + target)
  853. data_buf += target_file.read()
  854. target_file.close()
  855. assert_equal(256, len(data_buf.strip().split('\n')))
  856. assert_true('255' in data_buf)
  857. return resp
  858. TARGET_DIR_ROOT = self.cluster.fs_prefix + '/beeswax.test_save_directory_results'
  859. # Already existing dir
  860. if not self.cluster.fs.exists(TARGET_DIR_ROOT):
  861. self.cluster.fs.mkdir(TARGET_DIR_ROOT)
  862. self.cluster.fs.chown(TARGET_DIR_ROOT, user='test')
  863. hql = "SELECT * FROM test"
  864. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  865. resp = save_and_verify(resp, TARGET_DIR_ROOT, verify=False)
  866. assert_true('Directory already exists' in resp.content, resp.content)
  867. # SELECT *. (Result dir is same as table dir.)
  868. hql = "SELECT * FROM test"
  869. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  870. resp = save_and_verify(resp, TARGET_DIR_ROOT + '/1', verify=False)
  871. resp = self.client.get(resp.success_url)
  872. # Success and went to FB
  873. assert_true('File Browser' in resp.content, resp.content)
  874. # SELECT columns. (Result dir is in /tmp.)
  875. hql = "SELECT foo, bar FROM test"
  876. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  877. resp = save_and_verify(resp, TARGET_DIR_ROOT + '/2')
  878. resp = self.client.get(resp.success_url)
  879. assert_true('File Browser' in resp.content, resp.content)
  880. # Partition tables
  881. hql = "SELECT * FROM test_partitions"
  882. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  883. resp = save_and_verify(resp, TARGET_DIR_ROOT + '/3', verify=False)
  884. resp = self.client.get(resp.success_url)
  885. assert_true('File Browser' in resp.content, resp.content)
  886. def test_save_results_to_file(self):
  887. def save_and_verify(select_resp, target_file, overwrite=True, verify=True):
  888. content = json.loads(select_resp.content)
  889. qid = content['id']
  890. save_data = {
  891. 'type': 'hdfs',
  892. 'path': target_file,
  893. 'overwrite': overwrite
  894. }
  895. resp = self.client.post('/beeswax/api/query/%s/results/save/hdfs/file' % qid, save_data, follow=True)
  896. content = json.loads(resp.content)
  897. if content['status'] == 0:
  898. success_url = content['success_url']
  899. resp = self.client.get(content['watch_url'], follow=True)
  900. resp = wait_for_query_to_finish(self.client, resp, max=60)
  901. resp.success_url = success_url # Hack until better API
  902. # Check that data is right
  903. if verify:
  904. assert_true(self.cluster.fs.exists(target_file))
  905. assert_true(self.cluster.fs.isfile(target_file))
  906. data_buf = ""
  907. _file = self.cluster.fs.open(target_file)
  908. data_buf += _file.read()
  909. _file.close()
  910. assert_equal(256, len(data_buf.strip().split('\n')))
  911. assert_true('255' in data_buf)
  912. return resp
  913. TARGET_FILE = self.cluster.fs_prefix + '/beeswax.test_save_file_results'
  914. if self.cluster.fs.exists(TARGET_FILE):
  915. self.cluster.fs.rmtree(TARGET_FILE)
  916. # SELECT columns. (Result dir is in /tmp.)
  917. hql = "SELECT foo, bar FROM test"
  918. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  919. resp = save_and_verify(resp, TARGET_FILE)
  920. resp = self.client.get(resp.success_url)
  921. assert_true('File Browser' in resp.content, resp.content)
  922. # overwrite = false
  923. hql = "SELECT foo, bar FROM test"
  924. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  925. resp = save_and_verify(resp, TARGET_FILE, overwrite=False, verify=False)
  926. assert_true('-3' in resp.content, resp.content)
  927. assert_true('already exists' in resp.content)
  928. # Partition tables
  929. hql = "SELECT * FROM test_partitions"
  930. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  931. resp = save_and_verify(resp, TARGET_FILE, verify=False)
  932. resp = self.client.get(resp.success_url)
  933. assert_true('File Browser' in resp.content, resp.content)
  934. def test_save_results_to_tbl(self):
  935. def save_and_verify(select_resp, target_tbl):
  936. """Check that saving to table works"""
  937. content = json.loads(select_resp.content)
  938. qid = content['id']
  939. save_data = {
  940. 'type': 'hive-table',
  941. 'table': target_tbl
  942. }
  943. resp = self.client.post('/beeswax/api/query/%s/results/save/hive/table' % qid, save_data, follow=True)
  944. content = json.loads(resp.content)
  945. resp = self.client.get(content['watch_url'], follow=True)
  946. wait_for_query_to_finish(self.client, resp, max=120)
  947. # Check that data is right. The SELECT may not give us the whole table.
  948. resp = _make_query(self.client, 'SELECT * FROM %s' % target_tbl, wait=True, local=False, database=self.db_name)
  949. content = fetch_query_result_data(self.client, resp)
  950. for i in xrange(90):
  951. assert_equal([i, '0x%x' % (i,)], content['results'][i])
  952. TARGET_TBL_ROOT = 'test_copy'
  953. # SELECT *. (Result dir is same as table dir.)
  954. hql = "SELECT * FROM test"
  955. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  956. save_and_verify(resp, TARGET_TBL_ROOT + '_1')
  957. # SELECT columns. (Result dir is in /tmp.)
  958. hql = "SELECT foo, bar FROM test"
  959. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  960. save_and_verify(resp, TARGET_TBL_ROOT + '_2')
  961. # Save to another DB
  962. hql = "SELECT * FROM test"
  963. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  964. other_db = '%s_other' % self.db_name
  965. save_and_verify(resp, other_db + '.' + TARGET_TBL_ROOT)
  966. def test_install_examples(self):
  967. assert_true(not beeswax.models.MetaInstall.get().installed_example)
  968. # Check popup
  969. resp = self.client.get('/beeswax/install_examples')
  970. assert_true('POST request is required.' in json.loads(resp.content)['message'])
  971. self.client.post('/beeswax/install_examples')
  972. # New tables exists
  973. resp = self.client.get('/metastore/tables/?format=json')
  974. data = json.loads(resp.content)
  975. assert_true('sample_08' in data['table_names'])
  976. assert_true('sample_07' in data['table_names'])
  977. assert_true('customers' in data['table_names'])
  978. # Sample tables contain data (examples are installed in default DB)
  979. resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': 'default', 'table': 'sample_07'}))
  980. data = json.loads(resp.content)
  981. assert_true(data['rows'], data)
  982. resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': 'default', 'table': 'sample_08'}))
  983. data = json.loads(resp.content)
  984. assert_true(data['rows'], data)
  985. resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': 'default', 'table': 'customers'}))
  986. if USE_NEW_EDITOR.get():
  987. # New queries exist
  988. resp = self.client.get('/desktop/api2/docs/')
  989. data = json.loads(resp.content)
  990. doc_names = [doc['name'] for doc in data['documents']]
  991. assert_true('examples' in doc_names, data)
  992. uuid = next((doc['uuid'] for doc in data['documents'] if doc['name'] == 'examples'), None)
  993. resp = self.client.get('/desktop/api2/doc/', {'uuid': uuid})
  994. data = json.loads(resp.content)
  995. doc_names = [doc['name'] for doc in data['children']]
  996. assert_true('Sample: Job loss' in doc_names, data)
  997. assert_true('Sample: Salary growth' in doc_names, data)
  998. assert_true('Sample: Top salary' in doc_names, data)
  999. assert_true('Sample: Customers' in doc_names, data)
  1000. else:
  1001. # New designs exists
  1002. resp = self.client.get('/beeswax/list_designs')
  1003. assert_true('Sample: Job loss' in resp.content, resp.content)
  1004. assert_true('Sample: Salary growth' in resp.content)
  1005. assert_true('Sample: Top salary' in resp.content)
  1006. assert_true('Sample: Customers' in resp.content)
  1007. # Now install it a second time, and no error
  1008. resp = self.client.post('/beeswax/install_examples')
  1009. assert_equal(0, json.loads(resp.content)['status'])
  1010. assert_equal('', json.loads(resp.content)['message'])
  1011. def test_create_table_generation(self):
  1012. """
  1013. Checks HQL generation for create table.
  1014. NOT TESTED/DONE: Validation checks for the inputs.
  1015. """
  1016. # Make sure we get a form
  1017. resp = self.client.get("/beeswax/create/create_table/%s" % self.db_name)
  1018. assert_true("Field terminator" in resp.content)
  1019. # Make a submission
  1020. resp = self.client.post("/beeswax/create/create_table/%s" % self.db_name, {
  1021. 'table-name': 'my_table',
  1022. 'table-comment': 'Yo>>>>dude', # Make sure escaping is sort of ok.
  1023. 'table-row_format': 'Delimited',
  1024. 'table-field_terminator_0': r',',
  1025. 'table-collection_terminator_0': r'\002',
  1026. 'table-map_key_terminator_0': r'\003',
  1027. 'table-file_format': 'TextFile',
  1028. 'table-use_default_location': 'False',
  1029. 'table-external_location': '/tmp/foo',
  1030. 'columns-0-column_name': 'my_col',
  1031. 'columns-0-column_type': 'string',
  1032. 'columns-0-_exists': 'True',
  1033. 'columns-next_form_id': '1',
  1034. 'partitions-next_form_id': '0',
  1035. 'create': 'Create table',
  1036. }, follow=True)
  1037. # Ensure we can see table.
  1038. response = self.client.get("/metastore/table/%s/my_table?format=json" % self.db_name)
  1039. data = json.loads(response.content)
  1040. assert_true("my_col" in [col['name'] for col in data['cols']], data)
  1041. def test_create_table_timestamp(self):
  1042. # Check form
  1043. response = self.client.get('/beeswax/create/create_table/%s' % self.db_name)
  1044. assert_true('<option value="timestamp">timestamp</option>' in response.content, response.content)
  1045. # Check creation
  1046. filename = self.cluster.fs_prefix + '/timestamp_data'
  1047. # Bad format
  1048. self._make_custom_data_file(filename, [0, 0, 0])
  1049. self._make_table('timestamp_invalid_data', 'CREATE TABLE timestamp_invalid_data (timestamp1 TIMESTAMP)', filename)
  1050. resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': self.db_name, 'table': 'timestamp_invalid_data'}))
  1051. rows = json.loads(resp.content)['rows']
  1052. flat_rows = sum(rows, [])
  1053. assert_true("NULL" in flat_rows, flat_rows)
  1054. # Good format
  1055. self._make_custom_data_file(filename, ['2012-01-01 10:11:30', '2012-01-01 10:11:31'])
  1056. self._make_table('timestamp_valid_data', 'CREATE TABLE timestamp_valid_data (timestamp1 TIMESTAMP)', filename)
  1057. resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': self.db_name, 'table': 'timestamp_valid_data'}))
  1058. rows = json.loads(resp.content)['rows']
  1059. flat_rows = sum(rows, [])
  1060. assert_true("2012-01-01 10:11:30.0" in flat_rows, flat_rows)
  1061. def test_partitioned_create_table(self):
  1062. # Make sure we get a form
  1063. resp = self.client.get("/beeswax/create/create_table/%s" % self.db_name)
  1064. assert_true("Field terminator" in resp.content)
  1065. # Make a submission
  1066. resp = self.client.post("/beeswax/create/create_table/%s" % self.db_name, {
  1067. 'table-name': 'my_table2',
  1068. 'table-row_format': 'Delimited',
  1069. 'table-field_terminator_0': r'\001',
  1070. 'table-collection_terminator_0': r'\002',
  1071. 'table-map_key_terminator_0': r'\003',
  1072. 'table-file_format': 'TextFile',
  1073. 'table-use_default_location': 'True',
  1074. 'columns-0-column_name': 'my_col',
  1075. 'columns-0-column_type': 'string',
  1076. 'columns-0-_exists': 'True',
  1077. 'columns-next_form_id': '1',
  1078. 'partitions-0-column_name': 'my_partition',
  1079. 'partitions-0-column_type': 'string',
  1080. 'partitions-0-_exists': 'True',
  1081. 'partitions-next_form_id': '1',
  1082. 'create': 'Create table',
  1083. }, follow=True)
  1084. history = QueryHistory.objects.latest('id')
  1085. assert_equal_mod_whitespace("""
  1086. CREATE TABLE `%s.my_table2`
  1087. (
  1088. `my_col` string
  1089. )
  1090. PARTITIONED BY
  1091. (
  1092. `my_partition` string
  1093. )
  1094. ROW FORMAT DELIMITED
  1095. FIELDS TERMINATED BY '\\001'
  1096. COLLECTION ITEMS TERMINATED BY '\\002'
  1097. MAP KEYS TERMINATED BY '\\003'
  1098. STORED AS TextFile
  1099. """ % self.db_name, history.query)
  1100. def test_create_table_dependencies(self):
  1101. """
  1102. Test field dependency in the create table form
  1103. """
  1104. resp = self.client.post("/beeswax/create/create_table/%s" % self.db_name, {
  1105. 'table-name': 'my_table',
  1106. 'table-row_format': 'SerDe',
  1107. # Missing SerDe fields!
  1108. 'table-file_format': 'InputFormat',
  1109. # Input format fields missing!
  1110. 'columns-0-column_name': 'my_col',
  1111. 'columns-0-column_type': 'map',
  1112. # Map key value missing!
  1113. 'columns-0-_exists': 'True',
  1114. 'columns-1-column_name': 'my_partition',
  1115. 'columns-1-column_type': 'string',
  1116. 'columns-1-is_partition_key': 'on',
  1117. 'columns-1-_exists': 'True',
  1118. 'columns-next_form_id': '2',
  1119. 'partitions-next_form_id': '0',
  1120. 'create': 'Create table',
  1121. })
  1122. # All of these errors should have been triggered!
  1123. assert_true(resp.context["table_form"].errors["input_format_class"])
  1124. assert_true(resp.context["table_form"].errors["output_format_class"])
  1125. assert_true(resp.context["table_form"].errors["serde_name"])
  1126. assert_true(resp.context["table_form"].errors["serde_properties"])
  1127. assert_true(resp.context["table_form"].errors["serde_properties"])
  1128. assert_true(resp.context["columns_form"].forms[0].errors["map_key_type"])
  1129. assert_true(resp.context["columns_form"].forms[0].errors["map_value_type"])
  1130. def test_create_table_import(self):
  1131. RAW_FIELDS = [
  1132. ['ta\tb', 'nada', 'sp ace'],
  1133. ['f\too', 'bar', 'fred'],
  1134. ['a\ta', 'bb', 'cc'],
  1135. ]
  1136. CSV_FIELDS = [
  1137. ['a', 'b', 'c'],
  1138. ['"a,a"', '"b,b"', '"c,c"'],
  1139. ['"a,\"\"a"', '"b,\"\"b"', '"c,\"\"c"'],
  1140. ]
  1141. def write_file(filename, raw_fields, delim, do_gzip=False):
  1142. lines = [ delim.join(row) for row in raw_fields ]
  1143. data = '\n'.join(lines)
  1144. if do_gzip:
  1145. sio = cStringIO.StringIO()
  1146. gzdat = gzip.GzipFile(fileobj=sio, mode='wb')
  1147. gzdat.write(data)
  1148. gzdat.close()
  1149. data = sio.getvalue()
  1150. f = self.cluster.fs.open(filename, "w")
  1151. f.write(data)
  1152. f.close()
  1153. #self.cluster.fs.do_as_superuser(self.cluster.fs.chown, filename, 'test', 'test')
  1154. #self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir, '/user/test')
  1155. write_file(self.cluster.fs_prefix + '/spacé.dat'.decode('utf-8'), RAW_FIELDS, ' ')
  1156. write_file(self.cluster.fs_prefix + '/tab.dat', RAW_FIELDS, '\t')
  1157. write_file(self.cluster.fs_prefix + '/comma.dat', RAW_FIELDS, ',')
  1158. write_file(self.cluster.fs_prefix + '/pipes.dat', RAW_FIELDS, '|')
  1159. write_file(self.cluster.fs_prefix + '/comma.dat.gz', RAW_FIELDS, ',', do_gzip=True)
  1160. write_file(self.cluster.fs_prefix + '/comma.csv', CSV_FIELDS, ',')
  1161. # Test auto delim selection
  1162. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1163. 'submit_file': 'on',
  1164. 'path': self.cluster.fs_prefix + '/comma.dat',
  1165. 'name': 'test_create_import',
  1166. })
  1167. assert_equal(resp.context['fields_list'], RAW_FIELDS)
  1168. # Test same with gzip
  1169. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1170. 'submit_file': 'on',
  1171. 'path': self.cluster.fs_prefix + '/comma.dat.gz',
  1172. 'name': 'test_create_import',
  1173. })
  1174. assert_equal(resp.context['fields_list'], RAW_FIELDS)
  1175. # Make sure space works
  1176. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1177. 'submit_preview': 'on',
  1178. 'path': self.cluster.fs_prefix + '/spacé.dat',
  1179. 'name': 'test_create_import',
  1180. 'delimiter_0': ' ',
  1181. 'delimiter_1': '',
  1182. 'file_type': 'text',
  1183. })
  1184. assert_equal(len(resp.context['fields_list'][0]), 4)
  1185. # Make sure custom delimiters work
  1186. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1187. 'submit_preview': 'on',
  1188. 'path': self.cluster.fs_prefix + '/pipes.dat',
  1189. 'name': 'test_create_import',
  1190. 'delimiter_0': '__other__',
  1191. 'delimiter_1': '|',
  1192. 'file_type': 'text',
  1193. })
  1194. assert_equal(len(resp.context['fields_list'][0]), 3)
  1195. # Make sure quoted CSV works
  1196. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1197. 'submit_preview': 'on',
  1198. 'path': self.cluster.fs_prefix + '/comma.csv',
  1199. 'name': 'test_create_import_csv',
  1200. 'delimiter_0': '__other__',
  1201. 'delimiter_1': ',',
  1202. 'file_type': 'text',
  1203. })
  1204. assert_equal(resp.context['fields_list'], [
  1205. ['a', 'b', 'c'],
  1206. ['a,a', 'b,b', 'c,c'],
  1207. ['a,"a', 'b,"b', 'c,"c'],
  1208. ] )
  1209. # Test column definition
  1210. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1211. 'submit_delim': 'on',
  1212. 'path': self.cluster.fs_prefix + '/comma.dat.gz',
  1213. 'name': 'test_create_import',
  1214. 'delimiter_0': ',',
  1215. 'delimiter_1': '',
  1216. 'file_type': 'gzip',
  1217. })
  1218. # Should have 3 columns available
  1219. assert_equal(len(resp.context['column_formset'].forms), 3)
  1220. # Test table creation and data loading
  1221. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1222. 'submit_create': 'on',
  1223. 'path': self.cluster.fs_prefix + '/comma.dat.gz',
  1224. 'name': 'test_create_import',
  1225. 'delimiter_0': ',',
  1226. 'delimiter_1': '',
  1227. 'file_type': 'gzip',
  1228. 'do_import': 'True',
  1229. 'cols-0-_exists': 'True',
  1230. 'cols-0-column_name': 'col_a',
  1231. 'cols-0-column_type': 'string',
  1232. 'cols-1-_exists': 'True',
  1233. 'cols-1-column_name': 'col_b',
  1234. 'cols-1-column_type': 'string',
  1235. 'cols-2-_exists': 'True',
  1236. 'cols-2-column_name': 'col_c',
  1237. 'cols-2-column_type': 'string',
  1238. 'cols-next_form_id': '3',
  1239. }, follow=True)
  1240. #
  1241. # Little nightmare here:
  1242. # We have a POST (create table) with a redirect (load data) of redirect (show table)
  1243. #
  1244. assert_equal(resp.context['action'], 'watch-redirect')
  1245. on_success_url_load_data = resp.context['on_success_url']
  1246. assert_true('auto_load' in on_success_url_load_data, on_success_url_load_data)
  1247. query_history = resp.context['query_history']
  1248. resp = self.client.get(reverse('beeswax:api_fetch_query_history', kwargs={'query_history_id': query_history.id}), follow=True)
  1249. content = json.loads(resp.content)
  1250. watch_url = content['query_history']['watch_url']
  1251. class MockResponse():
  1252. def __init__(self, content):
  1253. self.content = json.dumps(content)
  1254. # Wait for CREATE TABLE to finis
  1255. resp = wait_for_query_to_finish(self.client, MockResponse({'status': 'ok', 'watch_url': watch_url}), max=180.0)
  1256. # Get URL that will load the data into the table. Also get the URL that will show the table in metastore app.
  1257. resp = self.client.get(on_success_url_load_data, follow=True)
  1258. assert_equal(resp.context['action'], 'watch-redirect')
  1259. on_success_url_show_table = resp.context['on_success_url']
  1260. assert_true('/metastore/table/' in on_success_url_show_table, on_success_url_show_table)
  1261. query_history = resp.context['query_history']
  1262. # Wait for load data to finish
  1263. resp = wait_for_query_to_finish(self.client, MockResponse({'status': 'ok', 'watch_url': watch_url}), max=180.0)
  1264. # Check data is in the table (by describing it)
  1265. resp = self.client.get(on_success_url_show_table)
  1266. cols = resp.context['table'].cols
  1267. assert_equal(len(cols), 3)
  1268. assert_equal([ col.name for col in cols ], [ 'col_a', 'col_b', 'col_c' ])
  1269. resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': self.db_name, 'table': 'test_create_import'}))
  1270. rows = json.loads(resp.content)['rows']
  1271. flat_rows = sum(rows, [])
  1272. assert_true("nada" in flat_rows, flat_rows)
  1273. assert_true("sp ace" in flat_rows, flat_rows)
  1274. # Test table creation and data loading and removing header
  1275. resp = self.client.post('/beeswax/create/import_wizard/%s' % self.db_name, {
  1276. 'submit_create': 'on',
  1277. 'path': self.cluster.fs_prefix + '/comma.csv',
  1278. 'name': 'test_create_import_with_header',
  1279. 'delimiter_0': ',',
  1280. 'delimiter_1': '',
  1281. 'file_type': 'text',
  1282. 'do_import': 'True',
  1283. 'cols-0-_exists': 'True',
  1284. 'cols-0-column_name': 'col_a',
  1285. 'cols-0-column_type': 'string',
  1286. 'cols-1-_exists': 'True',
  1287. 'cols-1-column_name': 'col_b',
  1288. 'cols-1-column_type': 'string',
  1289. 'cols-2-_exists': 'True',
  1290. 'cols-2-column_name': 'col_c',
  1291. 'cols-2-column_type': 'string',
  1292. 'cols-next_form_id': '3',
  1293. 'removeHeader': 'on'
  1294. }, follow=True)
  1295. # We have a POST (create table) with a redirect (load data) of redirect (show table)
  1296. assert_equal(resp.context['action'], 'watch-redirect')
  1297. on_success_url_load_data = resp.context['on_success_url']
  1298. assert_true('auto_load' in on_success_url_load_data, on_success_url_load_data)
  1299. query_history = resp.context['query_history']
  1300. resp = self.client.get(reverse('beeswax:api_fetch_query_history', kwargs={'query_history_id': query_history.id}), follow=True)
  1301. content = json.loads(resp.content)
  1302. watch_url = content['query_history']['watch_url']
  1303. # Wait for CREATE TABLE to finis
  1304. resp = wait_for_query_to_finish(self.client, MockResponse({'status': 'ok', 'watch_url': watch_url}), max=180.0)
  1305. # Get URL that will load the data into the table. Also get the URL that will show the table in metastore app.
  1306. resp = self.client.get(on_success_url_load_data, follow=True)
  1307. assert_equal(resp.context['action'], 'watch-redirect')
  1308. on_success_url_show_table = resp.context['on_success_url']
  1309. assert_true('/metastore/table/' in on_success_url_show_table, on_success_url_show_table)
  1310. query_history = resp.context['query_history']
  1311. # Wait for load data to finish
  1312. resp = wait_for_query_to_finish(self.client, MockResponse({'status': 'ok', 'watch_url': watch_url}), max=180.0)
  1313. # Check data is in the table (by describing it)
  1314. resp = self.client.get(on_success_url_show_table)
  1315. # Check data is in the table (by describing it)
  1316. cols = resp.context['table'].cols
  1317. assert_equal(len(cols), 3)
  1318. assert_equal([col.name for col in cols], ['col_a', 'col_b', 'col_c'])
  1319. resp = self.client.get(reverse('beeswax:get_sample_data', kwargs={'database': self.db_name, 'table': 'test_create_import_with_header'}))
  1320. rows = json.loads(resp.content)['rows']
  1321. assert_equal([
  1322. #['a', 'b', 'c'], # Gone as told to be header
  1323. ['"a', 'a"', '"b'], # Hive does not support natively quoted CSV
  1324. ['"a', '""a"', '"b']
  1325. ], rows)
  1326. def test_select_invalid_data(self):
  1327. filename = self.cluster.fs_prefix + '/test_select_invalid_data'
  1328. self._make_custom_data_file(filename, [1, 2, 3, 'NaN', 'INF', '-INF', 'BAD']) # Infinity not supported yet
  1329. self._make_table('test_select_invalid_data', 'CREATE TABLE test_select_invalid_data (timestamp1 DOUBLE)', filename)
  1330. hql = """
  1331. SELECT * FROM test_select_invalid_data;
  1332. """
  1333. resp = _make_query(self.client, hql, database=self.db_name)
  1334. resp = wait_for_query_to_finish(self.client, resp, max=30.0)
  1335. content = json.loads(resp.content)
  1336. history_id = content['id']
  1337. query_history = QueryHistory.get(id=history_id)
  1338. resp = self.client.get("/beeswax/results/%s/0?format=json" % history_id)
  1339. content = json.loads(resp.content)
  1340. assert_equal([[1.0], [2.0], [3.0], [u'NaN'], [u'NULL'], [u'NULL'], [u'NULL']], content['results'])
  1341. def test_create_database(self):
  1342. db_name = self.db_name + '_my_db'
  1343. db_name_accent = self.db_name + '_credito'
  1344. try:
  1345. resp = self.client.post("/beeswax/create/database", {
  1346. 'name': db_name,
  1347. 'comment': 'foo',
  1348. 'create': 'Create database',
  1349. 'use_default_location': True,
  1350. }, follow=True)
  1351. resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
  1352. resp = wait_for_query_to_finish(self.client, resp, max=180.0)
  1353. resp = self.client.get("/metastore/databases/")
  1354. assert_true(db_name in resp.context["databases"], resp)
  1355. # Test for accented characters in 'comment'
  1356. resp = self.client.post("/beeswax/create/database", {
  1357. 'name': db_name_accent,
  1358. 'comment': 'crédito',
  1359. 'create': 'Create database',
  1360. 'use_default_location': True,
  1361. }, follow=True)
  1362. resp = self.client.get(reverse("beeswax:api_watch_query_refresh_json", kwargs={'id': resp.context['query'].id}), follow=True)
  1363. resp = wait_for_query_to_finish(self.client, resp, max=180.0)
  1364. resp = self.client.get("/metastore/databases/")
  1365. assert_true(db_name_accent in resp.context['databases'], resp)
  1366. finally:
  1367. make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db_name}, wait=True)
  1368. make_query(self.client, 'DROP DATABASE IF EXISTS %(db)s' % {'db': db_name_accent}, wait=True)
  1369. def test_select_query_server(self):
  1370. c = make_logged_in_client()
  1371. _make_query(c, 'SELECT bogus FROM test', database=self.db_name) # Improvement: mock another server
  1372. history = beeswax.models.QueryHistory.objects.latest('id')
  1373. assert_equal('beeswax', history.server_name)
  1374. assert_equal(HIVE_SERVER_HOST.get(), history.server_host)
  1375. query_server = history.get_query_server_config()
  1376. assert_equal('beeswax', query_server['server_name'])
  1377. # NOTE: The history server is typically on a different server when live
  1378. # cluster testing.
  1379. if not is_live_cluster():
  1380. assert_equal(get_localhost_name(), query_server['server_host'])
  1381. assert_equal('hiveserver2', query_server['server_type'])
  1382. assert_true(query_server['principal'] is None, query_server['principal']) # No default hive/HOST_@TEST.COM so far
  1383. def test_select_multi_db(self):
  1384. response = _make_query(self.client, 'SELECT * FROM test LIMIT 5', local=False, database=self.db_name)
  1385. response = wait_for_query_to_finish(self.client, response)
  1386. content = fetch_query_result_data(self.client, response)
  1387. assert_true([0, u'0x0'] in content['results'], content)
  1388. response = _make_query(self.client, 'SHOW TABLES', local=False, database='%s_other' % self.db_name)
  1389. response = wait_for_query_to_finish(self.client, response)
  1390. content = fetch_query_result_data(self.client, response)
  1391. assert_true('tab_name' in content['columns'][0]['name'], content)
  1392. response = _make_query(self.client, 'SELECT * FROM test LIMIT 5', local=False, database='not_there')
  1393. content = json.loads(response.content)
  1394. assert_equal(-1, content.get('status'), content)
  1395. def test_list_design_pagination(self):
  1396. client = make_logged_in_client()
  1397. _make_query(client, 'SELECT', name='my query history', submission_type='Save', database=self.db_name)
  1398. design = SavedQuery.objects.get(name='my query history')
  1399. for i in range(25):
  1400. client.get('/beeswax/clone_design/%s' % (design.id,))
  1401. resp = client.get('/beeswax/list_designs')
  1402. ids_page_1 = set([query.id for query in resp.context['page'].object_list])
  1403. resp = client.get('/beeswax/list_designs?q-page=2')
  1404. ids_page_2 = set([query.id for query in resp.context['page'].object_list])
  1405. for id in ids_page_2:
  1406. assert_true(id not in ids_page_1)
  1407. SavedQuery.objects.filter(name='my query history').delete()
  1408. def test_get_table_sample(self):
  1409. client = make_logged_in_client()
  1410. resp = client.get(reverse('beeswax:get_sample_data', kwargs={'database': self.db_name, 'table': 'test'}))
  1411. json_resp = json.loads(resp.content)
  1412. assert_equal(0, json_resp['status'], json_resp)
  1413. assert_true('test.foo' in json_resp['headers'], json_resp)
  1414. assert_true([0, '0x0'] in json_resp['rows'], json_resp)
  1415. def test_get_sample_partitioned(self):
  1416. # Test limit of one partition
  1417. finish = conf.QUERY_PARTITIONS_LIMIT.set_for_testing(1)
  1418. try:
  1419. table_name = 'test_partitions'
  1420. partition_spec = "(`baz`='baz_one' AND `boom`='boom_two')"
  1421. table = self.db.get_table(database=self.db_name, table_name=table_name)
  1422. hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
  1423. assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec))
  1424. finally:
  1425. finish()
  1426. # Test limit of more than one partition
  1427. finish = conf.QUERY_PARTITIONS_LIMIT.set_for_testing(2)
  1428. try:
  1429. table_name = 'test_partitions'
  1430. partition_spec = "(`baz`='baz_one' AND `boom`='boom_two') OR (`baz`='baz_foo' AND `boom`='boom_bar')"
  1431. table = self.db.get_table(database=self.db_name, table_name=table_name)
  1432. hql = self.db._get_sample_partition_query(self.db_name, table, limit=10)
  1433. assert_equal(hql, 'SELECT * FROM `%s`.`%s` WHERE %s LIMIT 10' % (self.db_name, table_name, partition_spec))
  1434. finally:
  1435. finish()
  1436. # Test table with non-STRING (INT) partition columns
  1437. # Also tests for single partition column case
  1438. hql = """
  1439. CREATE TABLE test_partitions_int (a INT) PARTITIONED BY (b INT);
  1440. INSERT OVERWRITE TABLE test_partitions_int PARTITION (b=100)
  1441. SELECT 101 AS a FROM test_partitions LIMIT 1;
  1442. INSERT OVERWRITE TABLE test_partitions_int PARTITION (b=200)
  1443. SELECT 201 AS a FROM test_partitions LIMIT 1;
  1444. INSERT OVERWRITE TABLE test_partitions_int PARTITION (b=300)
  1445. SELECT 301 AS a FROM test_partitions LIMIT 1;
  1446. """
  1447. resp = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  1448. finish = conf.QUERY_PARTITIONS_LIMIT.set_for_testing(2)
  1449. try:
  1450. table_name = 'test_partitions_int'
  1451. table = self.db.get_table(database=self.db_name, table_name=table_name)
  1452. result = self.db.get_sample(self.db_name, table)
  1453. sample = list(result.rows())
  1454. assert_equal(len(sample), 2, sample)
  1455. finally:
  1456. finish()
  1457. # Test table that is partitioned but empty
  1458. hql = """
  1459. CREATE TABLE test_partitions_empty (a STRING) PARTITIONED BY (b STRING);
  1460. """
  1461. resp = _make_query(self.client, hql, wait=True, local=False, max=60.0, database=self.db_name)
  1462. finish = conf.QUERY_PARTITIONS_LIMIT.set_for_testing(2)
  1463. try:
  1464. table_name = 'test_partitions_empty'
  1465. table = self.db.get_table(database=self.db_name, table_name=table_name)
  1466. result = self.db.get_sample(self.db_name, table)
  1467. sample = list(result.rows())
  1468. assert_equal(len(sample), 0, sample)
  1469. finally:
  1470. finish()
  1471. def test_redacting_queries(self):
  1472. c = make_logged_in_client()
  1473. old_policies = redaction.global_redaction_engine.policies
  1474. redaction.global_redaction_engine.policies = [
  1475. RedactionPolicy([
  1476. RedactionRule('', 'ssn=\d{3}-\d{2}-\d{4}', 'ssn=XXX-XX-XXXX'),
  1477. ])
  1478. ]
  1479. logfilter.add_log_redaction_filter_to_logger(redaction.global_redaction_engine, logging.root)
  1480. try:
  1481. # Make sure redacted queries are redacted.
  1482. query = 'SELECT "ssn=123-45-6789"'
  1483. expected_query = 'SELECT "ssn=XXX-XX-XXXX"'
  1484. resp = make_query(c, query, database=self.db_name)
  1485. content = json.loads(resp.content)
  1486. assert_true('id' in content, 'Query failed: %s' % (content,))
  1487. query_id = content['id']
  1488. history = beeswax.models.QueryHistory.objects.get(pk=query_id)
  1489. assert_equal(history.query, expected_query)
  1490. assert_true(history.is_redacted)
  1491. # Make sure unredacted queries are not redacted.
  1492. query = 'SELECT "hello"'
  1493. expected_query = 'SELECT "hello"'
  1494. resp = make_query(c, query, database=self.db_name)
  1495. content = json.loads(resp.content)
  1496. query_id = content['id']
  1497. history = beeswax.models.QueryHistory.objects.get(pk=query_id)
  1498. assert_equal(history.query, expected_query)
  1499. assert_false(history.is_redacted)
  1500. finally:
  1501. redaction.global_redaction_engine.policies = old_policies
  1502. def test_analyze_table_and_read_statistics(self):
  1503. _make_query(self.client, "USE %s" % self.db_name, wait=True) # We need this until Hive 1.2
  1504. try:
  1505. # No stats
  1506. resp = self.client.get(reverse('beeswax:get_table_stats', kwargs={'database': self.db_name, 'table': 'test'}))
  1507. stats = json.loads(resp.content)['stats']
  1508. assert_equal('COLUMN_STATS_ACCURATE', stats[0]['data_type'], resp.content)
  1509. resp = self.client.get(reverse('beeswax:get_table_stats', kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo'}))
  1510. stats = json.loads(resp.content)['stats']
  1511. assert_equal([
  1512. {u'col_name': u'foo'},
  1513. {u'data_type': u'int'},
  1514. {u'min': u''},
  1515. {u'max': u''},
  1516. {u'num_nulls': u''},
  1517. {u'distinct_count': u''},
  1518. {u'avg_col_len': u''},
  1519. {u'max_col_len': u''},
  1520. {u'num_trues': u''},
  1521. {u'num_falses': u''}
  1522. ],
  1523. stats
  1524. )
  1525. # Compute stats
  1526. response = self.client.post(reverse("beeswax:analyze_table", kwargs={'database': self.db_name, 'table': 'test'}), follow=True)
  1527. response = wait_for_query_to_finish(self.client, response, max=120.0)
  1528. assert_true(response, response)
  1529. response = self.client.post(reverse("beeswax:analyze_table", kwargs={'database': self.db_name, 'table': 'test', 'columns': True}), follow=True)
  1530. response = wait_for_query_to_finish(self.client, response, max=120.0)
  1531. assert_true(response, response)
  1532. # Retrieve stats
  1533. resp = self.client.get(reverse('beeswax:get_table_stats', kwargs={'database': self.db_name, 'table': 'test'}))
  1534. stats = json.loads(resp.content)['stats']
  1535. assert_true(any([stat for stat in stats if stat['data_type'] == 'numRows']), resp.content)
  1536. assert_true(any([stat for stat in stats if stat['comment'] == '256']), resp.content)
  1537. resp = self.client.get(reverse('beeswax:get_table_stats', kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo'}))
  1538. stats = json.loads(resp.content)['stats']
  1539. assert_equal([
  1540. {u'col_name': u'foo'},
  1541. {u'data_type': u'int'},
  1542. {u'min': u'0'},
  1543. {u'max': u'255'},
  1544. {u'num_nulls': u'0'},
  1545. {u'distinct_count': u'180'},
  1546. {u'avg_col_len': u''},
  1547. {u'max_col_len': u''},
  1548. {u'num_trues': u''},
  1549. {u'num_falses': u''}
  1550. ],
  1551. stats
  1552. )
  1553. finally:
  1554. _make_query(self.client, "USE default", wait=True)
  1555. def test_get_top_terms(self):
  1556. if is_live_cluster():
  1557. raise SkipTest('HUE-2902: Skipping because test is not reentrant')
  1558. resp = self.client.get(reverse("beeswax:get_top_terms", kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo'}))
  1559. content = json.loads(resp.content)
  1560. assert_true('terms' in content, 'Failed to get terms: %s' % (content,))
  1561. terms = content['terms']
  1562. assert_equal([[255, 1], [254, 1], [253, 1], [252, 1]], terms[:4])
  1563. resp = self.client.get(reverse("beeswax:get_top_terms", kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo', 'prefix': '10'}))
  1564. content = json.loads(resp.content)
  1565. assert_true('terms' in content, 'Failed to get terms: %s' % (content,))
  1566. terms = content['terms']
  1567. assert_equal([[109, 1], [108, 1], [107, 1], [106, 1]], terms[:4])
  1568. resp = self.client.get(reverse("beeswax:get_top_terms", kwargs={'database': self.db_name, 'table': 'test', 'column': 'foo', 'prefix': '10'}) + '?limit=2')
  1569. content = json.loads(resp.content)
  1570. assert_true('terms' in content, 'Failed to get terms: %s' % (content,))
  1571. terms = content['terms']
  1572. assert_equal([[109, 1], [108, 1]], terms)
  1573. def test_beeswax_api_autocomplete(self):
  1574. CREATE_TABLE = "CREATE TABLE `%(db)s`.`nested_table` (foo ARRAY<STRUCT<bar:INT, baz:STRING>>);" % {'db': self.db_name}
  1575. _make_query(self.client, CREATE_TABLE, wait=True)
  1576. resp = self.client.get(reverse("beeswax:api_autocomplete_databases", kwargs={}))
  1577. databases = json.loads(resp.content)['databases']
  1578. assert_true(self.db_name in databases)
  1579. # Autocomplete tables for a given database
  1580. resp = self.client.get(reverse("beeswax:api_autocomplete_tables", kwargs={'database': self.db_name}))
  1581. tables = json.loads(resp.content)['tables_meta']
  1582. assert_true("nested_table" in [table['name'] for table in tables])
  1583. # Autocomplete columns for a given table
  1584. resp = self.client.get(reverse("beeswax:api_autocomplete_columns", kwargs={'database': self.db_name, 'table': 'nested_table'}))
  1585. columns = json.loads(resp.content)['columns']
  1586. assert_true("foo" in columns)
  1587. extended_columns = json.loads(resp.content)['extended_columns']
  1588. assert_equal({'comment': '', 'type': 'array<struct<bar:int,baz:string>>', 'name': 'foo'}, extended_columns[0])
  1589. # Autocomplete nested fields for a given column
  1590. resp = self.client.get(reverse("beeswax:api_autocomplete_column", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo'}))
  1591. json_resp = json.loads(resp.content)
  1592. assert_false('error' in json_resp, 'Failed to autocomplete nested type: %s' % json_resp.get('error'))
  1593. assert_equal("array", json_resp['type'])
  1594. assert_true("item" in json_resp)
  1595. assert_equal("struct", json_resp["item"]["type"])
  1596. # Autocomplete nested fields for a given nested type
  1597. resp = self.client.get(reverse("beeswax:api_autocomplete_nested", kwargs={'database': self.db_name, 'table': 'nested_table', 'column': 'foo', 'nested': 'item'}))
  1598. json_resp = json.loads(resp.content)
  1599. assert_false('error' in json_resp, 'Failed to autocomplete nested type: %s' % json_resp.get('error'))
  1600. assert_equal("struct", json_resp['type'])
  1601. assert_true("fields" in json_resp)
  1602. def test_get_indexes(self):
  1603. table_name = 'indexed_table'
  1604. hql = """
  1605. CREATE TABLE `%(db)s`.`%(table)s` (id INT, name STRING, age INT, state STRING);
  1606. CREATE INDEX `id_idx` ON TABLE `%(db)s`.`%(table)s` (`id`) AS 'COMPACT' WITH DEFERRED REBUILD;
  1607. CREATE INDEX `state_idx` ON TABLE `%(db)s`.`%(table)s` (`state`) AS 'COMPACT' WITH DEFERRED REBUILD;
  1608. """ % {'db': self.db_name, 'table': table_name}
  1609. _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  1610. resp = self.client.get(reverse("beeswax:get_indexes", kwargs={'database': self.db_name, 'table': table_name}))
  1611. json_resp = json.loads(resp.content)
  1612. assert_true('headers' in json_resp, json_resp)
  1613. assert_true('rows' in json_resp, json_resp)
  1614. assert_equal(2, len(json_resp['rows']), json_resp['rows'])
  1615. def test_get_settings(self):
  1616. resets = [
  1617. beeswax.conf.CONFIG_WHITELIST.set_for_testing('hive.execution.engine,mapreduce.job.queuename'),
  1618. ]
  1619. try:
  1620. resp = self.client.get(reverse("beeswax:get_settings"))
  1621. json_resp = json.loads(resp.content)
  1622. assert_equal(0, json_resp['status'])
  1623. assert_equal(2, len(json_resp['settings'].items()), json_resp)
  1624. assert_true('hive.execution.engine' in json_resp['settings'])
  1625. assert_true('mapreduce.job.queuename' in json_resp['settings'])
  1626. finally:
  1627. for reset in resets:
  1628. reset()
  1629. def test_get_functions(self):
  1630. resp = self.client.get(reverse("beeswax:get_functions"))
  1631. json_resp = json.loads(resp.content)
  1632. assert_true('functions' in json_resp, json_resp)
  1633. assert_true('coalesce' in json_resp['functions'], json_resp['functions'])
  1634. resp = self.client.get(reverse("beeswax:get_functions"), {'prefix': 'a'})
  1635. json_resp = json.loads(resp.content)
  1636. assert_true('functions' in json_resp, json_resp)
  1637. assert_true('avg' in json_resp['functions'], json_resp['functions'])
  1638. assert_false('coalesce' in json_resp['functions'], json_resp['functions'])
  1639. def test_databases_quote(self):
  1640. c = self.client
  1641. db_name = '__%s' % self.db_name
  1642. _make_query(c, "CREATE DATABASE IF NOT EXISTS `%s`" % db_name, database=self.db_name)
  1643. try:
  1644. self.db.use(db_name)
  1645. self.db.get_tables(db_name)
  1646. finally:
  1647. self.db.use(self.db_name)
  1648. _make_query(c, "DROP DATABASE IF EXISTS `%s`" % db_name, database=self.db_name)
  1649. def test_hs2_log_verbose(self):
  1650. """
  1651. Test that the HS2 logs send back the ql.Driver log output with JobID
  1652. """
  1653. if is_hive_on_spark():
  1654. raise SkipTest
  1655. hql = "SELECT foo FROM `%(db)s`.`test` GROUP BY foo" % {'db': self.db_name} # GROUP BY forces the MR job
  1656. response = _make_query(self.client, hql, wait=True, local=False, max=180.0, database=self.db_name)
  1657. content = fetch_query_result_data(self.client, response)
  1658. log = content['log']
  1659. assert_true(search_log_line('Starting Job = ', log), log)
  1660. assert_true(search_log_line('Ended Job = ', log), log)
  1661. # Test job extraction while we're at it
  1662. assert_equal(1, len(content["hadoop_jobs"]), "Should have started 1 job and extracted it.")
  1663. def test_import_gzip_reader():
  1664. """Test the gzip reader in create table"""
  1665. # Make gzipped data
  1666. data = file(__file__).read()
  1667. data_gz_sio = cStringIO.StringIO()
  1668. gz = gzip.GzipFile(fileobj=data_gz_sio, mode='wb')
  1669. gz.write(data)
  1670. gz.close()
  1671. data_gz = data_gz_sio.getvalue()
  1672. # Make sure we can't look at the whole gzipped data
  1673. old_peek_size = beeswax.create_table.IMPORT_PEEK_SIZE
  1674. beeswax.create_table.IMPORT_PEEK_SIZE = len(data_gz) - 1024
  1675. try:
  1676. reader = beeswax.create_table.GzipFileReader
  1677. lines = reader.readlines(data_gz_sio, 'utf-8')
  1678. assert_true(lines is not None)
  1679. lines_joined = '\n'.join(lines)
  1680. assert_equal(data[:len(lines_joined)], lines_joined)
  1681. finally:
  1682. beeswax.create_table.IMPORT_PEEK_SIZE = old_peek_size
  1683. def test_index_page():
  1684. """Minimal test that index page renders."""
  1685. c = make_logged_in_client()
  1686. c.get("/beeswax")
  1687. def test_history_page():
  1688. client = make_logged_in_client()
  1689. test_user = User.objects.get(username='test')
  1690. query, created = SavedQuery.objects.get_or_create(
  1691. type=HQL,
  1692. owner=test_user,
  1693. data='HQL query...',
  1694. name='Test query',
  1695. desc='Description',
  1696. )
  1697. QueryHistory.objects.get_or_create(
  1698. owner=test_user,
  1699. query='SELECT',
  1700. design=query,
  1701. last_state=0,
  1702. has_results=True,
  1703. query_type=HQL
  1704. )
  1705. def do_view(param, n=1):
  1706. resp = client.get('/beeswax/query_history?' + param)
  1707. if n == 0:
  1708. assert_equal(len(resp.context['page'].object_list), 0)
  1709. else:
  1710. assert_true(len(resp.context['page'].object_list) >= n) # Make the query run
  1711. return resp
  1712. do_view('')
  1713. do_view('q-user=test')
  1714. do_view('q-user=test_who', 0)
  1715. do_view('q-user=:all')
  1716. do_view('q-design_id=%s' % query.id)
  1717. do_view('q-design_id=9999999', 0)
  1718. do_view('q-auto_query=0')
  1719. do_view('q-auto_query=1')
  1720. do_view('sort=date')
  1721. do_view('sort=-date')
  1722. do_view('sort=state')
  1723. do_view('sort=-state')
  1724. do_view('sort=name')
  1725. do_view('sort=-name')
  1726. do_view('sort=type')
  1727. do_view('sort=-type')
  1728. do_view('sort=name&user=bogus&design_id=1&auto_query=1')
  1729. do_view('sort=-type&user=:all&type=hql&auto_query=0')
  1730. # Only show Beeswax queries
  1731. response = do_view('')
  1732. assert_equal({u'q-type': [u'beeswax']}, response.context['filter_params'])
  1733. # Test pagination
  1734. response = do_view('q-page=100', 0)
  1735. assert_equal(0, len(response.context['page'].object_list))
  1736. client = make_logged_in_client(username='test_who')
  1737. grant_access('test_who', 'test_who', 'test_who')
  1738. do_view('q-user=test_who', 0)
  1739. do_view('q-user=:all')
  1740. def teststrip_trailing_semicolon():
  1741. # Note that there are two queries (both an execute and an explain) scattered
  1742. # in this file that use semicolons all the way through.
  1743. # Single semicolon
  1744. assert_equal("foo", strip_trailing_semicolon("foo;\n"))
  1745. assert_equal("foo\n", strip_trailing_semicolon("foo\n;\n\n\n"))
  1746. # Multiple semicolons: strip only last one
  1747. assert_equal("fo;o;", strip_trailing_semicolon("fo;o;; "))
  1748. # No semicolons
  1749. assert_equal("foo", strip_trailing_semicolon("foo"))
  1750. def test_hadoop_extraction():
  1751. sample_log = """
  1752. Starting Job = job_201003191517_0002, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201003191517_0002
  1753. --- we should be ignoring duplicates ---
  1754. Starting Job = job_201003191517_0002, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201003191517_0002
  1755. Starting Job = job_201003191517_0003, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201003191517_0003
  1756. 14/06/10 14:30:55 INFO exec.Task: Starting Job = job_1402420825148_0001, Tracking URL = http://localhost:8088/proxy/application_1402420825148_0001/
  1757. """
  1758. assert_equal(
  1759. ["job_201003191517_0002", "job_201003191517_0003", "job_1402420825148_0001"],
  1760. beeswax.views._parse_out_hadoop_jobs(sample_log))
  1761. assert_equal([], beeswax.views._parse_out_hadoop_jobs("nothing to see here"))
  1762. sample_log_no_direct_url = """
  1763. 14/06/09 08:40:38 INFO impl.YarnClientImpl: Submitted application application_1402269517321_0003
  1764. 14/06/09 08:40:38 INFO mapreduce.Job: The url to track the job: N/A
  1765. 14/06/09 08:40:38 INFO exec.Task: Starting Job = job_1402269517321_0003, Tracking URL = N/A
  1766. 14/06/09 08:40:38 INFO exec.Task: Kill Command = /usr/lib/hadoop/bin/hadoop job -kill job_1402269517321_0003
  1767. 14/06/09 08:40:38 INFO cli.CLIService: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=2168d15e-96d2-415a-8d49-e2535e82c2a4]: getOperationStatus()
  1768. """
  1769. assert_equal(
  1770. ["job_1402269517321_0003"],
  1771. beeswax.views._parse_out_hadoop_jobs(sample_log_no_direct_url))
  1772. def test_hive_site():
  1773. tmpdir = tempfile.mkdtemp()
  1774. saved = None
  1775. try:
  1776. # We just replace the Beeswax conf variable
  1777. class Getter(object):
  1778. def get(self):
  1779. return tmpdir
  1780. xml = hive_site_xml(is_local=True, use_sasl=False)
  1781. file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
  1782. beeswax.hive_site.reset()
  1783. saved = beeswax.conf.HIVE_CONF_DIR
  1784. beeswax.conf.HIVE_CONF_DIR = Getter()
  1785. assert_equal(beeswax.hive_site.get_conf()['hive.metastore.warehouse.dir'], u'/abc')
  1786. assert_equal(beeswax.hive_site.get_hiveserver2_kerberos_principal('localhost'), 'hs2test/test.com@TEST.COM')
  1787. assert_equal(beeswax.hive_site.get_hiveserver2_authentication(), 'NOSASL')
  1788. finally:
  1789. beeswax.hive_site.reset()
  1790. if saved is not None:
  1791. beeswax.conf.HIVE_CONF_DIR = saved
  1792. shutil.rmtree(tmpdir)
  1793. def test_hive_site_host_pattern_local_host():
  1794. hostname = socket.getfqdn()
  1795. tmpdir = tempfile.mkdtemp()
  1796. saved = None
  1797. try:
  1798. # We just replace the Beeswax conf variable
  1799. class Getter(object):
  1800. def get(self):
  1801. return tmpdir
  1802. thrift_uris = 'thrift://%s:9999' % hostname
  1803. xml = hive_site_xml(is_local=False, use_sasl=False, thrift_uris=thrift_uris, kerberos_principal='test/_HOST@TEST.COM', hs2_kerberos_principal='test/_HOST@TEST.COM')
  1804. file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
  1805. beeswax.hive_site.reset()
  1806. saved = beeswax.conf.HIVE_CONF_DIR
  1807. beeswax.conf.HIVE_CONF_DIR = Getter()
  1808. reset = []
  1809. reset.append(beeswax.conf.HIVE_SERVER_HOST.set_for_testing(hostname))
  1810. assert_equal(beeswax.hive_site.get_conf()['hive.metastore.warehouse.dir'], u'/abc')
  1811. assert_equal(beeswax.hive_site.get_hiveserver2_kerberos_principal(hostname), 'test/' + socket.getfqdn().lower() + '@TEST.COM')
  1812. finally:
  1813. for finish in reset:
  1814. finish()
  1815. beeswax.hive_site.reset()
  1816. if saved is not None:
  1817. beeswax.conf.HIVE_CONF_DIR = saved
  1818. shutil.rmtree(tmpdir)
  1819. def test_hive_site_null_hs2krb():
  1820. """Test hive-site parsing with null hs2 kerberos principal"""
  1821. tmpdir = tempfile.mkdtemp()
  1822. saved = None
  1823. try:
  1824. # We just replace the Beeswax conf variable
  1825. class Getter(object):
  1826. def get(self):
  1827. return tmpdir
  1828. xml = hive_site_xml(is_local=True, use_sasl=False, hs2_kerberos_principal=None)
  1829. file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
  1830. beeswax.hive_site.reset()
  1831. saved = beeswax.conf.HIVE_CONF_DIR
  1832. beeswax.conf.HIVE_CONF_DIR = Getter()
  1833. assert_equal(beeswax.hive_site.get_conf()['hive.metastore.warehouse.dir'], u'/abc')
  1834. assert_equal(beeswax.hive_site.get_hiveserver2_kerberos_principal('localhost'), None)
  1835. assert_equal(beeswax.hive_site.get_hiveserver2_authentication(), 'NOSASL')
  1836. finally:
  1837. beeswax.hive_site.reset()
  1838. if saved is not None:
  1839. beeswax.conf.HIVE_CONF_DIR = saved
  1840. shutil.rmtree(tmpdir)
  1841. def test_collapse_whitespace():
  1842. assert_equal("", collapse_whitespace("\t\n\n \n\t \n"))
  1843. assert_equal("x", collapse_whitespace("\t\nx\n \n\t \n"))
  1844. assert_equal("x y", collapse_whitespace("\t\nx\n \ny\t \n"))
  1845. def test_search_log_line():
  1846. logs = """
  1847. FAILED: Parse Error: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant
  1848. 2012-08-18 12:23:15,648 ERROR [pool-1-thread-2] ql.Driver (SessionState.java:printError(380)) - FAILED: Parse Error: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant
  1849. org.apache.hadoop.hive.ql.parse.ParseException: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant
  1850. """
  1851. assert_true(search_log_line('FAILED: Parse Error', logs))
  1852. logs = "12/08/22 20:50:14 ERROR ql.Driver: FAILED: Parse Error: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant'"
  1853. assert_true(search_log_line('FAILED: Parse Error', logs))
  1854. logs = """
  1855. FAILED: Parse Error: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant
  1856. 2012-08-18 12:23:15,648 ERROR [pool-1-thread-2] ql.Driver (SessionState.java:printError(380)) - FAILED: Parse XXXX Error: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant
  1857. org.apache.hadoop.hive.ql.parse.ParseException: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant
  1858. """
  1859. assert_false(search_log_line('FAILED: Undefined', logs))
  1860. logs = """
  1861. 2012-08-18 12:23:15,648 ERROR [pool-1-thread-2] ql.Driver (SessionState.java:printError(380)) - FAILED: Parse
  1862. Error: line 1:31 cannot recognize input near '''' '_this_is_not' 'SQL' in constant
  1863. """
  1864. assert_false(search_log_line('FAILED: Parse Error', logs))
  1865. def test_split_statements():
  1866. assert_equal([''], hql_query(";;;").statements)
  1867. assert_equal(["select * where id == '10'"], hql_query("select * where id == '10'").statements)
  1868. assert_equal(["select * where id == '10'"], hql_query("select * where id == '10';").statements)
  1869. assert_equal(['select', "select * where id == '10;' limit 100"], hql_query("select; select * where id == '10;' limit 100;").statements)
  1870. assert_equal(['select', "select * where id == \"10;\" limit 100"], hql_query("select; select * where id == \"10;\" limit 100;").statements)
  1871. assert_equal(['select', "select * where id == '\"10;\"\"\"' limit 100"], hql_query("select; select * where id == '\"10;\"\"\"' limit 100;").statements)
  1872. query_with_comments = """--First query;
  1873. select concat('--', name) -- The '--' in quotes is not a comment
  1874. where id = '10';
  1875. -- Second query
  1876. select * where id = '10';"""
  1877. assert_equal(["--First query;\nselect concat(\'--\', name) -- The \'--\' in quotes is not a comment\nwhere id = \'10\'",
  1878. "-- Second query\nselect * where id = \'10\'"], hql_query(query_with_comments).statements)
  1879. query = """CREATE DATABASE IF NOT EXISTS functional;
  1880. DROP TABLE IF EXISTS functional.alltypes;
  1881. CREATE EXTERNAL TABLE IF NOT EXISTS functional.alltypes (
  1882. id int COMMENT 'Add a comment',
  1883. bool_col boolean,
  1884. tinyint_col tinyint,
  1885. smallint_col smallint,
  1886. int_col int,
  1887. bigint_col bigint,
  1888. float_col float,
  1889. double_col double,
  1890. date_string_col string,
  1891. string_col string,
  1892. timestamp_col timestamp)
  1893. PARTITIONED BY (year int, month int)
  1894. ROW FORMAT delimited fields terminated by ',' escaped by '\\\\'
  1895. STORED AS TEXTFILE
  1896. LOCATION '/user/admin/alltypes/alltypes';
  1897. USE functional;
  1898. ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=1);
  1899. ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=2);"""
  1900. assert_equal(['CREATE DATABASE IF NOT EXISTS functional',
  1901. 'DROP TABLE IF EXISTS functional.alltypes',
  1902. "CREATE EXTERNAL TABLE IF NOT EXISTS functional.alltypes (\nid int COMMENT 'Add a comment',\nbool_col boolean,\ntinyint_col tinyint,\nsmallint_col smallint,\nint_col int,\nbigint_col bigint,\nfloat_col float,\ndouble_col double,\ndate_string_col string,\nstring_col string,\ntimestamp_col timestamp)\nPARTITIONED BY (year int, month int)\nROW FORMAT delimited fields terminated by ',' escaped by '\\\\'\nSTORED AS TEXTFILE\nLOCATION '/user/admin/alltypes/alltypes'",
  1903. 'USE functional',
  1904. 'ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=1)',
  1905. 'ALTER TABLE alltypes ADD IF NOT EXISTS PARTITION(year=2009, month=2)'
  1906. ],
  1907. hql_query(query).statements, hql_query(query).statements)
  1908. class MockHiveServerTable(HiveServerTable):
  1909. def __init__(self, describe=None):
  1910. if describe is not None:
  1911. self.describe = describe
  1912. else:
  1913. self.describe = [
  1914. {'comment': 'comment ', 'col_name': '# col_name ', 'data_type': 'data_type '},
  1915. {'comment': None, 'col_name': '', 'data_type': None},
  1916. {'comment': '', 'col_name': 'foo', 'data_type': 'int'},
  1917. {'comment': '', 'col_name': 'bar', 'data_type': 'string'},
  1918. {'comment': None, 'col_name': '', 'data_type': None},
  1919. {'comment': None, 'col_name': '# Partition Information', 'data_type': None},
  1920. {'comment': 'comment ', 'col_name': '# col_name ', 'data_type': 'data_type '},
  1921. {'comment': None, 'col_name': '', 'data_type': None},
  1922. {'comment': '', 'col_name': 'baz', 'data_type': 'string'},
  1923. {'comment': '', 'col_name': 'boom', 'data_type': 'string'},
  1924. {'comment': None, 'col_name': '', 'data_type': None},
  1925. {'comment': None, 'col_name': '# Detailed Table Information', 'data_type': None},
  1926. {'comment': None, 'col_name': 'Database: ', 'data_type': 'default '},
  1927. {'comment': None, 'col_name': 'Owner: ', 'data_type': 'romain '},
  1928. {'comment': None, 'col_name': 'CreateTime: ', 'data_type': 'Wed Aug 13 13:39:53 PDT 2014'},
  1929. {'comment': None, 'col_name': 'LastAccessTime: ', 'data_type': 'UNKNOWN '},
  1930. {'comment': None, 'col_name': 'Protect Mode: ', 'data_type': 'None '},
  1931. {'comment': None, 'col_name': 'Retention: ', 'data_type': '0 '},
  1932. {'comment': None, 'col_name': 'Location: ', 'data_type': 'hdfs://localhost:8020/user/hive/warehouse/test_partitions'},
  1933. {'comment': None, 'col_name': 'Table Type: ', 'data_type': 'MANAGED_TABLE '},
  1934. {'comment': None, 'col_name': 'Table Parameters:', 'data_type': None},
  1935. {'comment': '1407962393 ', 'col_name': '', 'data_type': 'transient_lastDdlTime'},
  1936. {'comment': None, 'col_name': '', 'data_type': None},
  1937. {'comment': None, 'col_name': '# Storage Information', 'data_type': None},
  1938. {'comment': None, 'col_name': 'SerDe Library: ', 'data_type': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'},
  1939. {'comment': None, 'col_name': 'InputFormat: ', 'data_type': 'org.apache.hadoop.mapred.TextInputFormat'},
  1940. {'comment': None, 'col_name': 'OutputFormat: ', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'},
  1941. {'comment': None, 'col_name': 'Compressed: ', 'data_type': 'No '},
  1942. {'comment': None, 'col_name': 'Num Buckets: ', 'data_type': '-1 '},
  1943. {'comment': None, 'col_name': 'Bucket Columns: ', 'data_type': '[] '},
  1944. {'comment': None, 'col_name': 'Sort Columns: ', 'data_type': '[] '},
  1945. {'comment': None, 'col_name': 'Storage Desc Params:', 'data_type': None},
  1946. {'comment': '\\t ', 'col_name': '', 'data_type': 'field.delim '},
  1947. {'comment': '\\n ', 'col_name': '', 'data_type': 'line.delim '},
  1948. {'comment': '\\t ', 'col_name': '', 'data_type': 'serialization.format'}
  1949. ]
  1950. class MockHiveServerTableForPartitions(HiveServerTable):
  1951. def __init__(self, describe=None):
  1952. if describe is not None:
  1953. self.describe = describe
  1954. else:
  1955. self.describe = [
  1956. {'comment': 'comment ', 'col_name': '# col_name ', 'data_type': 'data_type '},
  1957. {'comment': 'NULL', 'col_name': '', 'data_type': 'NULL'},
  1958. {'comment': 'from deserializer', 'col_name': 'f1', 'data_type': 'string'},
  1959. {'comment': 'from deserializer', 'col_name': 'f2', 'data_type': 'string'},
  1960. {'comment': 'from deserializer', 'col_name': 'f3', 'data_type': 'string'},
  1961. {'comment': 'from deserializer', 'col_name': 'f4', 'data_type': 'string'},
  1962. {'comment': 'from deserializer', 'col_name': 'f5', 'data_type': 'string'},
  1963. {'comment': 'from deserializer', 'col_name': 'f6', 'data_type': 'string'},
  1964. {'comment': 'from deserializer', 'col_name': 'f7', 'data_type': 'string'},
  1965. {'comment': 'from deserializer', 'col_name': 'f8', 'data_type': 'string'},
  1966. {'comment': 'from deserializer', 'col_name': 'f9', 'data_type': 'string'},
  1967. {'comment': 'from deserializer', 'col_name': 'f10', 'data_type': 'string'},
  1968. {'comment': 'from deserializer', 'col_name': 'f11', 'data_type': 'string'},
  1969. {'comment': 'from deserializer', 'col_name': 'f12', 'data_type': 'string'},
  1970. {'comment': 'from deserializer', 'col_name': 'f13', 'data_type': 'bigint'},
  1971. {'comment': 'from deserializer', 'col_name': 'f14', 'data_type': 'int'},
  1972. {'comment': 'from deserializer', 'col_name': 'f15', 'data_type': 'string'},
  1973. {'comment': 'from deserializer', 'col_name': 'f16', 'data_type': 'string'},
  1974. {'comment': 'from deserializer', 'col_name': 'f17', 'data_type': 'string'},
  1975. {'comment': 'from deserializer', 'col_name': 'f18', 'data_type': 'timestamp'},
  1976. {'comment': 'from deserializer', 'col_name': 'f19', 'data_type': 'int'},
  1977. {'comment': 'from deserializer', 'col_name': 'f20', 'data_type': 'int'},
  1978. {'comment': 'from deserializer', 'col_name': 'f21', 'data_type': 'string'},
  1979. {'comment': 'from deserializer', 'col_name': 'f22', 'data_type': 'string'},
  1980. {'comment': 'from deserializer', 'col_name': 'f23', 'data_type': 'string'},
  1981. {'comment': 'from deserializer', 'col_name': 'f24', 'data_type': 'string'},
  1982. {'comment': 'from deserializer', 'col_name': 'f25', 'data_type': 'timestamp'},
  1983. {'comment': 'from deserializer', 'col_name': 'f26', 'data_type': 'int'},
  1984. {'comment': 'from deserializer', 'col_name': 'f27', 'data_type': 'binary'},
  1985. {'comment': 'NULL', 'col_name': '', 'data_type': 'NULL'},
  1986. {'comment': 'NULL', 'col_name': '# Partition Information', 'data_type': 'NULL'},
  1987. {'comment': 'comment ', 'col_name': '# col_name ', 'data_type': 'data_type '},
  1988. {'comment': 'NULL', 'col_name': '', 'data_type': 'NULL'},
  1989. {'comment': '', 'col_name': 'import_date', 'data_type': 'string'},
  1990. {'comment': '', 'col_name': 'import_id', 'data_type': 'int'},
  1991. {'comment': 'NULL', 'col_name': '', 'data_type': 'NULL'},
  1992. {'comment': 'NULL', 'col_name': '# Detailed Table Information', 'data_type': 'NULL'},
  1993. {'comment': 'NULL', 'col_name': 'Database: ', 'data_type': 'my_db '},
  1994. {'comment': 'NULL', 'col_name': 'Owner: ', 'data_type': 'hive '},
  1995. {'comment': 'NULL', 'col_name': 'CreateTime: ', 'data_type': 'Wed Feb 10 14:29:49 UTC 2016'},
  1996. {'comment': 'NULL', 'col_name': 'LastAccessTime: ', 'data_type': 'UNKNOWN '},
  1997. {'comment': 'NULL', 'col_name': 'Protect Mode: ', 'data_type': 'None '},
  1998. {'comment': 'NULL', 'col_name': 'Retention: ', 'data_type': '0 '},
  1999. {'comment': 'NULL', 'col_name': 'Location: ', 'data_type': 'hdfs://nameservice1/folder/folder'},
  2000. {'comment': 'NULL', 'col_name': 'Table Type: ', 'data_type': 'EXTERNAL_TABLE '},
  2001. {'comment': 'NULL', 'col_name': 'Table Parameters:', 'data_type': 'NULL'},
  2002. {'comment': 'TRUE ', 'col_name': '', 'data_type': 'EXTERNAL '},
  2003. {'comment': '1455114589 ', 'col_name': '', 'data_type': 'transient_lastDdlTime'},
  2004. {'comment': 'NULL', 'col_name': '', 'data_type': 'NULL'},
  2005. {'comment': 'NULL', 'col_name': '# Storage Information', 'data_type': 'NULL'},
  2006. {'comment': 'NULL', 'col_name': 'SerDe Library: ', 'data_type': 'com.x.y.z.a.MyDeserializer'},
  2007. {'comment': 'NULL', 'col_name': 'InputFormat: ', 'data_type': 'com.x.y.z.a.MyInputFormat'},
  2008. {'comment': 'NULL', 'col_name': 'OutputFormat: ', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'},
  2009. {'comment': 'NULL', 'col_name': 'Compressed: ', 'data_type': 'No '},
  2010. {'comment': 'NULL', 'col_name': 'Num Buckets: ', 'data_type': '-1 '},
  2011. {'comment': 'NULL', 'col_name': 'Bucket Columns: ', 'data_type': '[] '},
  2012. {'comment': 'NULL', 'col_name': 'Sort Columns: ', 'data_type': '[] '},
  2013. {'comment': 'NULL', 'col_name': 'Storage Desc Params:', 'data_type': 'NULL'},
  2014. {'comment': '1 ', 'col_name': '', 'data_type': 'serialization.format'},
  2015. ]
  2016. class TestHiveServer2API():
  2017. def test_parsing_partition_values(self):
  2018. table = MockHiveServerTable()
  2019. value = PartitionValueCompatible(['datehour=2013022516'], table)
  2020. assert_equal(['2013022516'], value.values)
  2021. value = PartitionValueCompatible(['month=2011-07/dt=2011-07-01/hr=12'], table)
  2022. assert_equal(['2011-07', '2011-07-01', '12'], value.values)
  2023. def test_hiveserver_table(self):
  2024. table = MockHiveServerTable()
  2025. assert_equal([
  2026. {'comment': None, 'col_name': '# Partition Information', 'data_type': None},
  2027. {'comment': 'comment', 'col_name': '# col_name', 'data_type': 'data_type'},
  2028. {'comment': None, 'col_name': '', 'data_type': None},
  2029. {'comment': '', 'col_name': 'baz', 'data_type': 'string'},
  2030. {'comment': '', 'col_name': 'boom', 'data_type': 'string'},
  2031. {'comment': None, 'col_name': '', 'data_type': None},
  2032. {'comment': None, 'col_name': '# Detailed Table Information', 'data_type': None},
  2033. {'comment': None, 'col_name': 'Database:', 'data_type': 'default'},
  2034. {'comment': None, 'col_name': 'Owner:', 'data_type': 'romain'},
  2035. {'comment': None, 'col_name': 'CreateTime:', 'data_type': 'Wed Aug 13 13:39:53 PDT 2014'},
  2036. {'comment': None, 'col_name': 'LastAccessTime:', 'data_type': 'UNKNOWN'},
  2037. {'comment': None, 'col_name': 'Protect Mode:', 'data_type': 'None'},
  2038. {'comment': None, 'col_name': 'Retention:', 'data_type': '0'},
  2039. {'comment': None, 'col_name': 'Location:', 'data_type': 'hdfs://localhost:8020/user/hive/warehouse/test_partitions'},
  2040. {'comment': None, 'col_name': 'Table Type:', 'data_type': 'MANAGED_TABLE'},
  2041. {'comment': None, 'col_name': 'Table Parameters:', 'data_type': None},
  2042. {'comment': '1407962393', 'col_name': '', 'data_type': 'transient_lastDdlTime'},
  2043. {'comment': None, 'col_name': '', 'data_type': None},
  2044. {'comment': None, 'col_name': '# Storage Information', 'data_type': None},
  2045. {'comment': None, 'col_name': 'SerDe Library:', 'data_type': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'},
  2046. {'comment': None, 'col_name': 'InputFormat:', 'data_type': 'org.apache.hadoop.mapred.TextInputFormat'},
  2047. {'comment': None, 'col_name': 'OutputFormat:', 'data_type': 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'},
  2048. {'comment': None, 'col_name': 'Compressed:', 'data_type': 'No'},
  2049. {'comment': None, 'col_name': 'Num Buckets:', 'data_type': '-1'},
  2050. {'comment': None, 'col_name': 'Bucket Columns:', 'data_type': '[]'},
  2051. {'comment': None, 'col_name': 'Sort Columns:', 'data_type': '[]'},
  2052. {'comment': None, 'col_name': 'Storage Desc Params:', 'data_type': None},
  2053. {'comment': '\\t', 'col_name': '', 'data_type': 'field.delim'},
  2054. {'comment': '\\n', 'col_name': '', 'data_type': 'line.delim'},
  2055. {'comment': '\\t', 'col_name': '', 'data_type': 'serialization.format'}
  2056. ],
  2057. table.properties
  2058. )
  2059. assert_equal('hdfs://localhost:8020/user/hive/warehouse/test_partitions', table.path_location)
  2060. assert_equal([
  2061. {'col_name': 'foo', 'comment': '', 'data_type': 'int'},
  2062. {'col_name': 'bar', 'comment': '', 'data_type': 'string'},
  2063. {'col_name': 'baz', 'comment': '', 'data_type': 'string'},
  2064. {'col_name': 'boom', 'comment': '', 'data_type': 'string'}], table.cols)
  2065. assert_equal([PartitionKeyCompatible('baz', 'string', ''),
  2066. PartitionKeyCompatible('boom', 'string', '')
  2067. ], table.partition_keys)
  2068. def test_hiveserver_table_for_partitions(self):
  2069. table = MockHiveServerTableForPartitions()
  2070. assert_equal([
  2071. PartitionKeyCompatible('import_date', 'string', ''),
  2072. PartitionKeyCompatible('import_id', 'int', '')
  2073. ], table.partition_keys
  2074. )
  2075. def test_hiveserver_has_complex(self):
  2076. # Test simple table with only scalars
  2077. table = MockHiveServerTable()
  2078. assert_false(table.has_complex, table.cols)
  2079. # Test complex table with array column
  2080. table.describe.insert(4, {'comment': '', 'col_name': 'fizz', 'data_type': 'array<string>'})
  2081. assert_true(table.has_complex, table.cols)
  2082. def test_hiveserver_table_partition_keys(self):
  2083. describe = [
  2084. {'comment': None, 'col_name': '# Partition Information', 'data_type': None},
  2085. {'comment': 'comment ', 'col_name': '# col_name ', 'data_type': 'data_type '},
  2086. {'comment': None, 'col_name': '', 'data_type': None},
  2087. {'comment': '', 'col_name': 'dt', 'data_type': 'string'},
  2088. {'comment': '', 'col_name': 'country', 'data_type': 'string'},
  2089. {'comment': 'this, has extra: sigils', 'col_name': 'decimal', 'data_type': 'decimal(9, 7)'},
  2090. {'comment': '', 'col_name': 'complex', 'data_type': 'UNIONTYPE<int, double, array<string>, struct<a:int,b:string>>'},
  2091. {'comment': None, 'col_name': '', 'data_type': None}
  2092. ]
  2093. table = MockHiveServerTable(describe)
  2094. assert_equal([PartitionKeyCompatible('dt', 'string', ''),
  2095. PartitionKeyCompatible('country', 'string', ''),
  2096. PartitionKeyCompatible('decimal', 'decimal(9, 7)', 'this, has extra: sigils'),
  2097. PartitionKeyCompatible('complex', 'UNIONTYPE<int, double, array<string>, struct<a:int,b:string>>', ''),
  2098. ], table.partition_keys)
  2099. def test_column_format_values_nulls(self):
  2100. data = [1, 1, 1]
  2101. nulls = '\x00'
  2102. assert_equal([1, 1, 1],
  2103. HiveServerTColumnValue2.set_nulls(data, nulls))
  2104. data = [1, 1, 1]
  2105. nulls = '\x03'
  2106. assert_equal([None, None, 1],
  2107. HiveServerTColumnValue2.set_nulls(data, nulls))
  2108. data = [1, 1, 1, 1, 1, 1, 1, 1]
  2109. nulls = 't' # 0b1110100
  2110. assert_equal([1, 1, None, 1, None, None, None, 1],
  2111. HiveServerTColumnValue2.set_nulls(data, nulls))
  2112. data = [1, 1, 'not_good', 'NaN', None, 'INF', 'INF', 3]
  2113. nulls = 't' # 0b1110100
  2114. assert_equal([1, 1, None, 'NaN', None, None, None, 3],
  2115. HiveServerTColumnValue2.set_nulls(data, nulls))
  2116. data = [1] * 18
  2117. nulls = '\xff\xee\x03'
  2118. assert_equal([None, None, None, None, None, None, None, None, 1, None, None, None, 1, None, None, None, None, None],
  2119. HiveServerTColumnValue2.set_nulls(data, nulls))
  2120. data = [1, 1, 1, 1, 1, 1, 1, 1]
  2121. nulls = '\x41'
  2122. assert_equal([None, 1, 1, 1, 1, 1, None, 1],
  2123. HiveServerTColumnValue2.set_nulls(data, nulls))
  2124. data = [1] * 8 * 8
  2125. nulls = '\x01\x23\x45\x67\x89\xab\xcd\xef'
  2126. assert_equal([None, 1, 1, 1, 1, 1, 1, 1, None, None, 1, 1, 1, None, 1, 1, None, 1, None, 1, 1, 1, None, 1, None, None, None, 1, 1, None, None, 1, None, 1, 1,
  2127. None, 1, 1, 1, None, None, None, 1, None, 1, None, 1, None, None, 1, None, None, 1, 1, None, None, None, None, None, None, 1, None, None, None],
  2128. HiveServerTColumnValue2.set_nulls(data, nulls))
  2129. def test_column_detect_if_values_nulls(self):
  2130. data = [1, 2, 3]
  2131. nulls = ''
  2132. assert_true(data is HiveServerTColumnValue2.set_nulls(data, nulls))
  2133. nulls = '\x00'
  2134. assert_true(data is HiveServerTColumnValue2.set_nulls(data, nulls))
  2135. nulls = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
  2136. assert_true(data is HiveServerTColumnValue2.set_nulls(data, nulls))
  2137. nulls = 'aaaa'
  2138. assert_false(data is HiveServerTColumnValue2.set_nulls(data, nulls))
  2139. nulls = '\x00\x01\x00'
  2140. assert_false(data is HiveServerTColumnValue2.set_nulls(data, nulls))
  2141. nulls = '\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00'
  2142. assert_false(data is HiveServerTColumnValue2.set_nulls(data, nulls))
  2143. class MockDbms:
  2144. def __init__(self, client, server_type):
  2145. pass
  2146. def get_databases(self):
  2147. return ['default', 'test']
  2148. def get_tables(self, database):
  2149. return ['table1', 'table2']
  2150. def get_state(self, handle):
  2151. return 0
  2152. class TestWithMockedServer(object):
  2153. def setUp(self):
  2154. # Beware: Monkey patch Beeswax/Hive server with Mock API
  2155. if not hasattr(dbms, 'OriginalBeeswaxApi'):
  2156. dbms.OriginalBeeswaxApi = dbms.HiveServer2Dbms
  2157. dbms.DBMS_CACHE = {}
  2158. dbms.HiveServer2Dbms = MockDbms
  2159. self.client = make_logged_in_client(is_superuser=False)
  2160. self.client_not_me = make_logged_in_client(username='not_me', is_superuser=False, groupname='test')
  2161. self.user = User.objects.get(username='test')
  2162. self.user_not_me = User.objects.get(username='not_me')
  2163. grant_access("test", "test", "beeswax")
  2164. def tearDown(self):
  2165. dbms.DBMS_CACHE = {}
  2166. dbms.HiveServer2Dbms = dbms.OriginalBeeswaxApi
  2167. def test_bulk_query_trash(self):
  2168. response = _make_query(self.client, 'SELECT', submission_type='Save', name='My Name 1', desc='My Description')
  2169. content = json.loads(response.content)
  2170. query = content['design_id']
  2171. response = _make_query(self.client, 'SELECT', submission_type='Save', name='My Name 2', desc='My Description')
  2172. content = json.loads(response.content)
  2173. query2 = content['design_id']
  2174. ids = [query, query2]
  2175. resp = self.client.get('/beeswax/list_designs')
  2176. ids_page_1 = set([query.id for query in resp.context['page'].object_list])
  2177. assert_equal(2, sum([query_id in ids_page_1 for query_id in ids]))
  2178. resp = self.client.post(reverse('beeswax:delete_design'), {u'skipTrash': [u'false'], u'designs_selection': ids})
  2179. queries = SavedQuery.objects.filter(id__in=ids)
  2180. assert_true(queries[0].doc.get().is_trashed())
  2181. assert_true(queries[1].doc.get().is_trashed())
  2182. resp = self.client.get('/beeswax/list_designs')
  2183. ids_page_1 = set([query.id for query in resp.context['page'].object_list])
  2184. assert_equal(0, sum([query_id in ids_page_1 for query_id in ids]))
  2185. resp = self.client.post(reverse('beeswax:restore_design'), {u'skipTrash': [u'false'], u'designs_selection': ids})
  2186. query = SavedQuery.objects.filter(id__in=ids)
  2187. assert_false(queries[0].doc.get().is_trashed())
  2188. assert_false(queries[1].doc.get().is_trashed())
  2189. resp = self.client.get('/beeswax/list_designs')
  2190. ids_page_1 = set([query.id for query in resp.context['page'].object_list])
  2191. assert_equal(2, sum([query_id in ids_page_1 for query_id in ids]))
  2192. resp = self.client.post(reverse('beeswax:delete_design'), {u'skipTrash': [u'false'], u'designs_selection': ids})
  2193. query = SavedQuery.objects.filter(id__in=ids)
  2194. assert_true(queries[0].doc.get().is_trashed())
  2195. assert_true(queries[1].doc.get().is_trashed())
  2196. resp = self.client.get('/beeswax/list_designs')
  2197. ids_page_1 = set([query.id for query in resp.context['page'].object_list])
  2198. assert_equal(0, sum([query_id in ids_page_1 for query_id in ids]))
  2199. resp = self.client.post(reverse('beeswax:delete_design'), {u'skipTrash': [u'true'], u'designs_selection': ids})
  2200. assert_false(SavedQuery.objects.filter(id__in=ids).exists())
  2201. resp = self.client.get('/beeswax/list_designs')
  2202. ids_page_1 = set([query.id for query in resp.context['page'].object_list])
  2203. assert_equal(0, sum([query_id in ids_page_1 for query_id in ids]))
  2204. def test_save_design(self):
  2205. response = _make_query(self.client, 'SELECT', submission_type='Save', name='My Name 1', desc='My Description')
  2206. content = json.loads(response.content)
  2207. design_id = content['design_id']
  2208. design = SavedQuery.objects.get(id=design_id)
  2209. design_obj = hql_query('SELECT')
  2210. # Save his own query
  2211. saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc')
  2212. assert_equal('test_save_design', saved_design.name)
  2213. assert_equal('test_save_design desc', saved_design.desc)
  2214. assert_equal('test_save_design', saved_design.doc.get().name)
  2215. assert_equal('test_save_design desc', saved_design.doc.get().description)
  2216. assert_false(saved_design.doc.get().is_historic())
  2217. # Execute it as auto
  2218. saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj, explicit_save=False, name='test_save_design', desc='test_save_design desc')
  2219. assert_equal('test_save_design (new)', saved_design.name)
  2220. assert_equal('test_save_design desc', saved_design.desc)
  2221. assert_equal('test_save_design (new)', saved_design.doc.get().name)
  2222. assert_equal('test_save_design desc', saved_design.doc.get().description)
  2223. assert_true(saved_design.doc.get().is_historic())
  2224. # not_me user can't modify it
  2225. try:
  2226. _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc')
  2227. assert_true(False, 'not_me is not allowed')
  2228. except PopupException:
  2229. pass
  2230. saved_design.doc.get().share_to_default()
  2231. try:
  2232. _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc')
  2233. assert_true(False, 'not_me is not allowed')
  2234. except PopupException:
  2235. pass
  2236. # not_me can execute it as auto
  2237. saved_design = _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=False, name='test_save_design', desc='test_save_design desc')
  2238. assert_equal('test_save_design (new)', saved_design.name)
  2239. assert_equal('test_save_design desc', saved_design.desc)
  2240. assert_equal('test_save_design (new)', saved_design.doc.get().name)
  2241. assert_equal('test_save_design desc', saved_design.doc.get().description)
  2242. assert_true(saved_design.doc.get().is_historic())
  2243. # not_me can save as a new design
  2244. design = SavedQuery(owner=self.user_not_me, type=HQL)
  2245. saved_design = _save_design(user=self.user_not_me, design=design, type_=HQL, design_obj=design_obj, explicit_save=True, name='test_save_design', desc='test_save_design desc')
  2246. assert_equal('test_save_design', saved_design.name)
  2247. assert_equal('test_save_design desc', saved_design.desc)
  2248. assert_equal('test_save_design', saved_design.doc.get().name)
  2249. assert_equal('test_save_design desc', saved_design.doc.get().description)
  2250. assert_false(saved_design.doc.get().is_historic())
  2251. # Save design with len(name) = 64
  2252. response = _make_query(self.client, 'SELECT', submission_type='Save',
  2253. name='test_character_limit', desc='test_character_limit desc')
  2254. content = json.loads(response.content)
  2255. design_id = content['design_id']
  2256. design = SavedQuery.objects.get(id=design_id)
  2257. design_obj = hql_query('SELECT')
  2258. # Save query
  2259. saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj,
  2260. explicit_save=True, name='This__design__name__contains___sixty__five___characters___exactly', desc='test_save_design desc')
  2261. len_after = len(saved_design.name)
  2262. assert_equal(len_after, 64)
  2263. saved_design = _save_design(user=self.user, design=design, type_=HQL, design_obj=design_obj,
  2264. explicit_save=False, name='This__design__name__contains___sixty__five___characters___exactly', desc='test_save_design desc')
  2265. # Above design name is already 64 characters, so saved_design name shouldn't exceed the limit
  2266. len_after = len(saved_design.name)
  2267. assert_equal(len_after, 64)
  2268. def test_get_history_xss(self):
  2269. sql = 'SELECT count(sample_07.salary) FROM sample_07;"><iFrAME>src="javascript:alert(\'Hue has an xss\');"></iFraME>'
  2270. sql_escaped = 'SELECT count(sample_07.salary) FROM sample_07;&quot;&gt;&lt;iFrAME&gt;src=&quot;javascript:alert(&#39;Hue has an xss&#39;);&quot;&gt;&lt;/iFraME&gt;'
  2271. response = _make_query(self.client, sql, submission_type='Save', name='My Name 1', desc='My Description')
  2272. content = json.loads(response.content)
  2273. design_id = content['design_id']
  2274. design = SavedQuery.objects.get(id=design_id)
  2275. query_history = QueryHistory.build(
  2276. owner=self.user,
  2277. query=sql,
  2278. server_host='server_host',
  2279. server_port=1,
  2280. server_name='server_name',
  2281. server_type=HIVE_SERVER2,
  2282. last_state=QueryHistory.STATE.submitted.index,
  2283. design=design,
  2284. notify=False,
  2285. query_type=HQL,
  2286. statement_number=0
  2287. )
  2288. query_history.save()
  2289. resp = self.client.get('/beeswax/query_history?format=json')
  2290. assert_true(sql_escaped in resp.content, resp.content)
  2291. assert_false(sql in resp.content, resp.content)
  2292. def test_redact_saved_design(self):
  2293. old_policies = redaction.global_redaction_engine.policies
  2294. redaction.global_redaction_engine.policies = [
  2295. RedactionPolicy([
  2296. RedactionRule('', 'ssn=\d{3}-\d{2}-\d{4}', 'ssn=XXX-XX-XXXX'),
  2297. ])
  2298. ]
  2299. logfilter.add_log_redaction_filter_to_logger(redaction.global_redaction_engine, logging.root)
  2300. try:
  2301. # Make sure redacted queries are redacted.
  2302. query = 'SELECT "ssn=123-45-6789"'
  2303. expected_query = 'SELECT "ssn=XXX-XX-XXXX"'
  2304. response = _make_query(self.client, query, submission_type='Save', name='My Name 1', desc='My Description')
  2305. content = json.loads(response.content)
  2306. design_id = content['design_id']
  2307. design = SavedQuery.get(id=design_id)
  2308. data = json.loads(design.data)
  2309. assert_equal(data['query']['query'], expected_query)
  2310. assert_true(design.is_redacted)
  2311. # Make sure unredacted queries are not redacted.
  2312. query = 'SELECT "hello"'
  2313. expected_query = 'SELECT "hello"'
  2314. response = _make_query(self.client, query, submission_type='Save', name='My Name 2', desc='My Description')
  2315. content = json.loads(response.content)
  2316. design_id = content['design_id']
  2317. design = SavedQuery.get(id=design_id)
  2318. data = json.loads(design.data)
  2319. assert_equal(data['query']['query'], expected_query)
  2320. assert_false(design.is_redacted)
  2321. finally:
  2322. redaction.global_redaction_engine.policies = old_policies
  2323. def test_search_designs(self):
  2324. # Create 20 (DEFAULT_PAGE_SIZE) queries to fill page 1, plus a target query for page 2
  2325. page_1 = []
  2326. for i in xrange(1, 21):
  2327. response = _make_query(self.client, 'SELECT', submission_type='Save', name='My Name %d' % i, desc='My Description')
  2328. content = json.loads(response.content)
  2329. query_id = content['design_id']
  2330. page_1.append(query_id)
  2331. response = _make_query(self.client, 'SELECT', submission_type='Save', name='Test Search Design', desc='My Test Search Design')
  2332. content = json.loads(response.content)
  2333. query_id = content['design_id']
  2334. page_2 = [query_id]
  2335. resp = self.client.get(reverse('beeswax:list_designs') + '?text=Test+Search+Design')
  2336. ids_page = set([query.id for query in resp.context['page'].object_list])
  2337. assert_equal(0, sum([query_id in ids_page for query_id in page_1]))
  2338. assert_equal(1, sum([query_id in ids_page for query_id in page_2]))
  2339. # Trash all designs and test search trashed designs
  2340. ids = page_1 + page_2
  2341. self.client.post(reverse('beeswax:delete_design'), {u'skipTrash': [u'false'], u'designs_selection': ids})
  2342. SavedQuery.objects.filter(id__in=ids)
  2343. resp = self.client.get(reverse('beeswax:list_trashed_designs') + '?text=Test+Search+Design')
  2344. ids_page = set([query.id for query in resp.context['page'].object_list])
  2345. assert_equal(0, sum([query_id in ids_page for query_id in page_1]))
  2346. assert_equal(1, sum([query_id in ids_page for query_id in page_2]))
  2347. def test_clear_history(self):
  2348. sql = 'SHOW TABLES'
  2349. response = _make_query(self.client, sql, submission_type='Save', name='My clear', desc='My Description')
  2350. content = json.loads(response.content)
  2351. design_id = content['design_id']
  2352. design = SavedQuery.objects.get(id=design_id)
  2353. query_history = QueryHistory.build(
  2354. owner=self.user,
  2355. query=sql,
  2356. server_host='server_host',
  2357. server_port=1,
  2358. server_name='server_name',
  2359. server_type=HIVE_SERVER2,
  2360. last_state=QueryHistory.STATE.submitted.index,
  2361. design=design,
  2362. notify=False,
  2363. query_type=HQL,
  2364. statement_number=0
  2365. )
  2366. query_history.save()
  2367. resp = self.client.get(reverse('beeswax:list_query_history') + '?q-design_id=%s&format=json' % design_id)
  2368. json_resp = json.loads(resp.content)
  2369. design_ids = [history['design_id'] for history in json_resp['queries']]
  2370. assert_true(design_id in design_ids, json_resp)
  2371. resp = self.client.get(reverse('beeswax:list_query_history') + '?q-design_id=%s&recent=true&format=json' % design_id)
  2372. json_resp = json.loads(resp.content)
  2373. design_ids = [history['design_id'] for history in json_resp['queries']]
  2374. assert_true(design_id in design_ids, json_resp)
  2375. self.client.post(reverse('beeswax:clear_history'))
  2376. resp = self.client.get(reverse('beeswax:list_query_history') + '?q-design_id=%s&format=json' % design_id)
  2377. json_resp = json.loads(resp.content)
  2378. design_ids = [history['design_id'] for history in json_resp['queries']]
  2379. assert_true(design_id in design_ids, json_resp)
  2380. resp = self.client.get(reverse('beeswax:list_query_history') + '?q-design_id=%s&recent=true&format=json' % design_id)
  2381. json_resp = json.loads(resp.content)
  2382. design_ids = [history['design_id'] for history in json_resp['queries']]
  2383. assert_false(design_id in design_ids, json_resp)
  2384. class TestDesign():
  2385. def test_hql_resource(self):
  2386. design = hql_query('SELECT')
  2387. design._data_dict['file_resources'] = [
  2388. {'type': 'FILE', 'path': 'my_file'},
  2389. {'type': 'FILE', 'path': '/my_path/my_file'},
  2390. {'type': 'FILE', 'path': 's3://host/my_s3_file'}
  2391. ]
  2392. statements = design.get_configuration_statements()
  2393. assert_true(re.match('ADD FILE hdfs://([^:]+):(\d+)my_file', statements[0]), statements[0])
  2394. assert_true(re.match('ADD FILE hdfs://([^:]+):(\d+)/my_path/my_file', statements[1]), statements[1])
  2395. assert_equal('ADD FILE s3://host/my_s3_file', statements[2])
  2396. def search_log_line(expected_log, all_logs):
  2397. return re.compile('%(expected_log)s' % {'expected_log': expected_log}).search(all_logs)
  2398. def test_hiveserver2_get_security():
  2399. make_logged_in_client()
  2400. user = User.objects.get(username='test')
  2401. # Bad but easy mocking
  2402. hive_site.get_conf()
  2403. prev = hive_site._HIVE_SITE_DICT.get(hive_site._CNF_HIVESERVER2_AUTHENTICATION)
  2404. try:
  2405. hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_KERBEROS_PRINCIPAL] = 'hive/hive@test.com'
  2406. principal = get_query_server_config('beeswax')['principal']
  2407. assert_true(principal.startswith('hive/'), principal)
  2408. principal = get_query_server_config('impala')['principal']
  2409. assert_true(principal.startswith('impala/'), principal)
  2410. default_query_server = {'server_host': 'my_host', 'server_port': 12345}
  2411. # Beeswax
  2412. beeswax_query_server = {'server_name': 'beeswax', 'principal': 'hive', 'auth_username': 'hue', 'auth_password': None}
  2413. beeswax_query_server.update(default_query_server)
  2414. assert_equal((True, 'PLAIN', 'hive', True, 'hue', None), HiveServerClient(beeswax_query_server, user).get_security())
  2415. # HiveServer2 LDAP passthrough
  2416. beeswax_query_server.update({'auth_username': 'hueabcd', 'auth_password': 'abcd'})
  2417. assert_equal((True, 'PLAIN', 'hive', True, 'hueabcd', 'abcd'), HiveServerClient(beeswax_query_server, user).get_security())
  2418. beeswax_query_server.update({'auth_username': 'hue', 'auth_password': None})
  2419. hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_AUTHENTICATION] = 'NOSASL'
  2420. hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_IMPERSONATION] = 'false'
  2421. assert_equal((False, 'NOSASL', 'hive', False, 'hue', None), HiveServerClient(beeswax_query_server, user).get_security())
  2422. hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_AUTHENTICATION] = 'KERBEROS'
  2423. assert_equal((True, 'GSSAPI', 'hive', False, 'hue', None), HiveServerClient(beeswax_query_server, user).get_security())
  2424. # Impala
  2425. cluster_conf = hadoop.cluster.get_cluster_conf_for_job_submission()
  2426. finish = cluster_conf.SECURITY_ENABLED.set_for_testing(False)
  2427. try:
  2428. impala_query_server = {'server_name': 'impala', 'principal': 'impala', 'impersonation_enabled': False, 'auth_username': 'hue', 'auth_password': None}
  2429. impala_query_server.update(default_query_server)
  2430. assert_equal((False, 'GSSAPI', 'impala', False, 'hue', None), HiveServerClient(impala_query_server, user).get_security())
  2431. impala_query_server = {'server_name': 'impala', 'principal': 'impala', 'impersonation_enabled': True, 'auth_username': 'hue', 'auth_password': None}
  2432. impala_query_server.update(default_query_server)
  2433. assert_equal((False, 'GSSAPI', 'impala', True, 'hue', None), HiveServerClient(impala_query_server, user).get_security())
  2434. finally:
  2435. finish()
  2436. finish = cluster_conf.SECURITY_ENABLED.set_for_testing(True)
  2437. try:
  2438. assert_equal((True, 'GSSAPI', 'impala', True, 'hue', None), HiveServerClient(impala_query_server, user).get_security())
  2439. finally:
  2440. finish()
  2441. finally:
  2442. if prev is not None:
  2443. hive_site._HIVE_SITE_DICT[hive_site._CNF_HIVESERVER2_AUTHENTICATION] = prev
  2444. else:
  2445. hive_site._HIVE_SITE_DICT.pop(hive_site._CNF_HIVESERVER2_AUTHENTICATION, None)
  2446. class MockClient():
  2447. def __init__(self):
  2448. self.open_session_args = None
  2449. def OpenSession(self, args):
  2450. self.open_session_args = args
  2451. #def test_hive_server2_open_session():
  2452. # make_logged_in_client()
  2453. # user = User.objects.get(username='test')
  2454. #
  2455. # query_server = get_query_server_config()
  2456. #
  2457. # db_client = HiveServerClient(query_server, user)
  2458. # mock_hs2_client = MockClient()
  2459. # setattr(db_client, '_client', mock_hs2_client)
  2460. #
  2461. # # Regular session
  2462. # finish = desktop_conf.LDAP_PASSWORD.set_for_testing('')
  2463. # try:
  2464. # db_client.open_session(user)
  2465. # except:
  2466. # pass
  2467. # finally:
  2468. # finish()
  2469. # req = mock_hs2_client.open_session_args
  2470. # assert_equal('test', req.username)
  2471. # assert_equal(None, req.password)
  2472. # assert_equal('test', req.configuration['hive.server2.proxy.user'])
  2473. #
  2474. # # LDAP credentials
  2475. # finish = desktop_conf.LDAP_PASSWORD.set_for_testing('I_love_Hue')
  2476. # try:
  2477. # db_client.open_session(user)
  2478. # except:
  2479. # pass
  2480. # finally:
  2481. # finish()
  2482. # req = mock_hs2_client.open_session_args
  2483. # assert_equal('test', req.username) # Same as kerberos, real username is picked from Thrift authentication, this one does not matter
  2484. # assert_equal(None, req.password)
  2485. def test_metastore_security():
  2486. tmpdir = tempfile.mkdtemp()
  2487. saved = None
  2488. try:
  2489. # We just replace the Beeswax conf variable
  2490. class Getter(object):
  2491. def get(self):
  2492. return tmpdir
  2493. xml = hive_site_xml(is_local=False, use_sasl=True, kerberos_principal='hive/_HOST@test.com')
  2494. file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
  2495. beeswax.hive_site.reset()
  2496. saved = beeswax.conf.HIVE_CONF_DIR
  2497. beeswax.conf.HIVE_CONF_DIR = Getter()
  2498. metastore = get_metastore()
  2499. assert_true(metastore['use_sasl'])
  2500. assert_equal('thrift://darkside-1234:9999', metastore['thrift_uri'])
  2501. assert_equal('hive/darkside-1234@test.com', metastore['kerberos_principal'])
  2502. finally:
  2503. beeswax.hive_site.reset()
  2504. if saved is not None:
  2505. beeswax.conf.HIVE_CONF_DIR = saved
  2506. shutil.rmtree(tmpdir)
  2507. def test_close_queries_flag():
  2508. c = make_logged_in_client()
  2509. finish = conf.CLOSE_QUERIES.set_for_testing(False)
  2510. try:
  2511. resp = c.get('/beeswax/execute')
  2512. assert_false('closeQuery()' in resp.content, resp.content)
  2513. finally:
  2514. finish()
  2515. finish = conf.CLOSE_QUERIES.set_for_testing(True)
  2516. try:
  2517. resp = c.get('/beeswax/execute')
  2518. assert_true('closeQuery()' in resp.content, resp.content)
  2519. finally:
  2520. finish()
  2521. def test_auth_pass_through():
  2522. # Backward compatibility nothing set
  2523. finish = []
  2524. finish.append(LDAP_USERNAME.set_for_testing(present=False))
  2525. finish.append(LDAP_PASSWORD.set_for_testing(present=False))
  2526. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing(present=False))
  2527. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing(present=False))
  2528. finish.append(AUTH_USERNAME.set_for_testing(present=False))
  2529. finish.append(AUTH_PASSWORD.set_for_testing(present=False))
  2530. try:
  2531. assert_equal('hue', AUTH_USERNAME.get())
  2532. assert_equal(None, AUTH_PASSWORD.get())
  2533. finally:
  2534. for f in finish:
  2535. f()
  2536. # Backward compatibility
  2537. finish = []
  2538. finish.append(LDAP_USERNAME.set_for_testing('deprecated_default_username'))
  2539. finish.append(LDAP_PASSWORD.set_for_testing('deprecated_default_password'))
  2540. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing(present=False))
  2541. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing(present=False))
  2542. finish.append(AUTH_USERNAME.set_for_testing(present=False))
  2543. finish.append(AUTH_PASSWORD.set_for_testing(present=False))
  2544. try:
  2545. assert_equal('deprecated_default_username', AUTH_USERNAME.get())
  2546. assert_equal('deprecated_default_password', AUTH_PASSWORD.get())
  2547. finally:
  2548. for f in finish:
  2549. f()
  2550. # Backward compatibility override
  2551. finish = []
  2552. finish.append(LDAP_USERNAME.set_for_testing('deprecated_default_username'))
  2553. finish.append(LDAP_PASSWORD.set_for_testing('deprecated_default_password'))
  2554. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing('default_username'))
  2555. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing('default_password'))
  2556. try:
  2557. assert_equal('default_username', AUTH_USERNAME.get())
  2558. assert_equal('default_password', AUTH_PASSWORD.get())
  2559. finally:
  2560. for f in finish:
  2561. f()
  2562. # HiveServer2 specific
  2563. finish = []
  2564. finish.append(LDAP_USERNAME.set_for_testing('deprecated_default_username'))
  2565. finish.append(LDAP_PASSWORD.set_for_testing('deprecated_default_password'))
  2566. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing('default_username'))
  2567. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing('default_password'))
  2568. finish.append(AUTH_USERNAME.set_for_testing('hive_username'))
  2569. finish.append(AUTH_PASSWORD.set_for_testing('hive_password'))
  2570. try:
  2571. assert_equal('hive_username', AUTH_USERNAME.get())
  2572. assert_equal('hive_password', AUTH_PASSWORD.get())
  2573. finally:
  2574. for f in finish:
  2575. f()
  2576. # Common
  2577. finish = []
  2578. finish.append(LDAP_USERNAME.set_for_testing('deprecated_default_username'))
  2579. finish.append(LDAP_PASSWORD.set_for_testing('deprecated_default_password'))
  2580. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing('default_username'))
  2581. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing('default_password'))
  2582. finish.append(AUTH_USERNAME.set_for_testing(present=False))
  2583. finish.append(AUTH_PASSWORD.set_for_testing(present=False))
  2584. try:
  2585. assert_equal('default_username', AUTH_USERNAME.get())
  2586. assert_equal('default_password', AUTH_PASSWORD.get())
  2587. finally:
  2588. for f in finish:
  2589. f()
  2590. # Password file specific and use common username
  2591. finish = []
  2592. finish.append(LDAP_USERNAME.set_for_testing('deprecated_default_username'))
  2593. finish.append(LDAP_PASSWORD.set_for_testing('deprecated_default_password'))
  2594. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing('default_username'))
  2595. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing(present=False))
  2596. finish.append(AUTH_USERNAME.set_for_testing(present=False))
  2597. finish.append(AUTH_PASSWORD.set_for_testing(present=False))
  2598. finish.append(AUTH_PASSWORD_SCRIPT.set_for_testing('/bin/echo "my_hue_secret"'))
  2599. try:
  2600. assert_equal('default_username', AUTH_USERNAME.get())
  2601. assert_equal('my_hue_secret', AUTH_PASSWORD.get())
  2602. finally:
  2603. for f in finish:
  2604. f()
  2605. # Make sure global auth password script can be used.
  2606. finish = []
  2607. finish.append(LDAP_USERNAME.set_for_testing('deprecated_default_username'))
  2608. finish.append(LDAP_PASSWORD.set_for_testing('deprecated_default_password'))
  2609. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing('default_username'))
  2610. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing(present=False))
  2611. finish.append(DEFAULT_AUTH_PASSWORD_SCRIPT.set_for_testing('/bin/echo "my_hue_secret"'))
  2612. finish.append(AUTH_USERNAME.set_for_testing(present=False))
  2613. finish.append(AUTH_PASSWORD.set_for_testing(present=False))
  2614. finish.append(AUTH_PASSWORD_SCRIPT.set_for_testing(present=False))
  2615. try:
  2616. assert_equal('default_username', AUTH_USERNAME.get())
  2617. assert_equal('my_hue_secret', AUTH_PASSWORD.get())
  2618. finally:
  2619. for f in finish:
  2620. f()
  2621. # Make sure local auth password script overrides global password.
  2622. finish = []
  2623. finish.append(LDAP_USERNAME.set_for_testing('deprecated_default_username'))
  2624. finish.append(LDAP_PASSWORD.set_for_testing('deprecated_default_password'))
  2625. finish.append(DEFAULT_AUTH_USERNAME.set_for_testing('default_username'))
  2626. finish.append(DEFAULT_AUTH_PASSWORD.set_for_testing(present=False))
  2627. finish.append(DEFAULT_AUTH_PASSWORD_SCRIPT.set_for_testing('/bin/echo "not_my_secret"'))
  2628. finish.append(AUTH_USERNAME.set_for_testing(present=False))
  2629. finish.append(AUTH_PASSWORD.set_for_testing(present=False))
  2630. finish.append(AUTH_PASSWORD_SCRIPT.set_for_testing('/bin/echo "my_hue_secret"'))
  2631. try:
  2632. assert_equal('default_username', AUTH_USERNAME.get())
  2633. assert_equal('my_hue_secret', AUTH_PASSWORD.get())
  2634. finally:
  2635. for f in finish:
  2636. f()
  2637. def hive_site_xml(is_local=False, use_sasl=False, thrift_uris='thrift://darkside-1234:9999',
  2638. warehouse_dir='/abc', kerberos_principal='test/test.com@TEST.COM',
  2639. hs2_kerberos_principal='hs2test/test.com@TEST.COM',
  2640. hs2_authentication='NOSASL', hs2_impersonation='false'):
  2641. if not is_local:
  2642. uris = """
  2643. <property>
  2644. <name>hive.metastore.uris</name>
  2645. <value>%(thrift_uris)s</value>
  2646. </property>
  2647. """ % {'thrift_uris': thrift_uris}
  2648. else:
  2649. uris = ''
  2650. if hs2_kerberos_principal:
  2651. hs2_krb_princ = """
  2652. <property>
  2653. <name>hive.server2.authentication.kerberos.principal</name>
  2654. <value>%(hs2_kerberos_principal)s</value>
  2655. </property>
  2656. """ % {'hs2_kerberos_principal': hs2_kerberos_principal}
  2657. else:
  2658. hs2_krb_princ = ""
  2659. return """
  2660. <configuration>
  2661. %(uris)s
  2662. <property>
  2663. <name>hive.metastore.warehouse.dir</name>
  2664. <value>%(warehouse_dir)s</value>
  2665. </property>
  2666. <property>
  2667. <name>hive.metastore.kerberos.principal</name>
  2668. <value>%(kerberos_principal)s</value>
  2669. </property>
  2670. %(hs2_krb_princ)s
  2671. <property>
  2672. <name>hive.server2.enable.impersonation</name>
  2673. <value>%(hs2_impersonation)s</value>
  2674. </property>
  2675. <property>
  2676. <name>hive.server2.authentication</name>
  2677. <value>%(hs2_authentication)s</value>
  2678. </property>
  2679. <property>
  2680. <name>hive.metastore.sasl.enabled</name>
  2681. <value>%(use_sasl)s</value>
  2682. </property>
  2683. </configuration>
  2684. """ % {
  2685. 'uris': uris,
  2686. 'warehouse_dir': warehouse_dir,
  2687. 'kerberos_principal': kerberos_principal,
  2688. 'hs2_krb_princ': hs2_krb_princ,
  2689. 'hs2_authentication': hs2_authentication,
  2690. 'use_sasl': str(use_sasl).lower(),
  2691. 'hs2_impersonation': hs2_impersonation,
  2692. }
  2693. def test_ssl_cacerts():
  2694. for desktop_kwargs, conf_kwargs, expected in [
  2695. ({'present': False}, {'present': False}, ''),
  2696. ({'present': False}, {'data': 'local-cacerts.pem'}, 'local-cacerts.pem'),
  2697. ({'data': 'global-cacerts.pem'}, {'present': False}, 'global-cacerts.pem'),
  2698. ({'data': 'global-cacerts.pem'}, {'data': 'local-cacerts.pem'}, 'local-cacerts.pem'),
  2699. ]:
  2700. resets = [
  2701. desktop_conf.SSL_CACERTS.set_for_testing(**desktop_kwargs),
  2702. conf.SSL.CACERTS.set_for_testing(**conf_kwargs),
  2703. ]
  2704. try:
  2705. assert_equal(conf.SSL.CACERTS.get(), expected,
  2706. 'desktop:%s conf:%s expected:%s got:%s' % (desktop_kwargs, conf_kwargs, expected, conf.SSL.CACERTS.get()))
  2707. finally:
  2708. for reset in resets:
  2709. reset()
  2710. def test_ssl_validate():
  2711. for desktop_kwargs, conf_kwargs, expected in [
  2712. ({'present': False}, {'present': False}, True),
  2713. ({'present': False}, {'data': False}, False),
  2714. ({'present': False}, {'data': True}, True),
  2715. ({'data': False}, {'present': False}, False),
  2716. ({'data': False}, {'data': False}, False),
  2717. ({'data': False}, {'data': True}, True),
  2718. ({'data': True}, {'present': False}, True),
  2719. ({'data': True}, {'data': False}, False),
  2720. ({'data': True}, {'data': True}, True),
  2721. ]:
  2722. resets = [
  2723. desktop_conf.SSL_VALIDATE.set_for_testing(**desktop_kwargs),
  2724. conf.SSL.VALIDATE.set_for_testing(**conf_kwargs),
  2725. ]
  2726. try:
  2727. assert_equal(conf.SSL.VALIDATE.get(), expected,
  2728. 'desktop:%s conf:%s expected:%s got:%s' % (desktop_kwargs, conf_kwargs, expected, conf.SSL.VALIDATE.get()))
  2729. finally:
  2730. for reset in resets:
  2731. reset()
  2732. def test_to_matching_wildcard():
  2733. match_fn = dbms.HiveServer2Dbms.to_matching_wildcard
  2734. assert_equal(match_fn(None), '*')
  2735. assert_equal(match_fn(''), '*')
  2736. assert_equal(match_fn('*'), '*')
  2737. assert_equal(match_fn('test'), '*test*')
  2738. assert_equal(match_fn('test*'), '*test*')
  2739. def test_apply_natural_sort():
  2740. test_strings = ['test_1', 'test_100', 'test_2', 'test_200']
  2741. assert_equal(apply_natural_sort(test_strings), ['test_1', 'test_2', 'test_100', 'test_200'])
  2742. test_dicts = [{'name': 'test_1', 'comment': 'Test'},
  2743. {'name': 'test_100', 'comment': 'Test'},
  2744. {'name': 'test_2', 'comment': 'Test'},
  2745. {'name': 'test_200', 'comment': 'Test'}]
  2746. assert_equal(apply_natural_sort(test_dicts, key='name'), [{'name': 'test_1', 'comment': 'Test'},
  2747. {'name': 'test_2', 'comment': 'Test'},
  2748. {'name': 'test_100', 'comment': 'Test'},
  2749. {'name': 'test_200', 'comment': 'Test'}])
  2750. def test_hiveserver2_jdbc_url():
  2751. hostname = socket.getfqdn()
  2752. resets = [
  2753. beeswax.conf.HIVE_SERVER_HOST.set_for_testing(hostname),
  2754. beeswax.conf.HIVE_SERVER_PORT.set_for_testing('10000')
  2755. ]
  2756. try:
  2757. url = hiveserver2_jdbc_url()
  2758. assert_equal(url, 'jdbc:hive2://' + hostname + ':10000/default')
  2759. beeswax.conf.HIVE_SERVER_HOST.set_for_testing('server-with-ssl-enabled.com')
  2760. beeswax.conf.HIVE_SERVER_PORT.set_for_testing('10000')
  2761. url = hiveserver2_jdbc_url()
  2762. assert_equal(url, 'jdbc:hive2://server-with-ssl-enabled.com:10000/default')
  2763. beeswax.hive_site.reset()
  2764. beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_USE_SSL] = 'TRUE'
  2765. beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_TRUSTSTORE_PATH] = '/path/to/truststore.jks'
  2766. beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_TRUSTSTORE_PASSWORD] = 'password'
  2767. url = hiveserver2_jdbc_url()
  2768. assert_equal(url, 'jdbc:hive2://server-with-ssl-enabled.com:10000/default;ssl=true;sslTrustStore=/path/to/truststore.jks;trustStorePassword=password')
  2769. beeswax.hive_site.get_conf()[hive_site._CNF_HIVESERVER2_USE_SSL] = 'FALSE'
  2770. url = hiveserver2_jdbc_url()
  2771. assert_equal(url, 'jdbc:hive2://server-with-ssl-enabled.com:10000/default')
  2772. finally:
  2773. beeswax.hive_site.reset()
  2774. for reset in resets:
  2775. reset()