tests.py 104 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. from builtins import range
  18. from builtins import object
  19. import json
  20. import logging
  21. import re
  22. import sys
  23. import time
  24. import unittest
  25. import pytz
  26. from django.urls import reverse
  27. from nose.plugins.skip import SkipTest
  28. from nose.tools import assert_true, assert_false, assert_equal, assert_raises
  29. from desktop.lib.django_test_util import make_logged_in_client
  30. from desktop.lib.test_utils import grant_access, add_to_group
  31. from desktop.models import Document
  32. from hadoop import cluster
  33. from hadoop.conf import YARN_CLUSTERS
  34. from hadoop.pseudo_hdfs4 import is_live_cluster
  35. from hadoop.yarn import resource_manager_api, mapreduce_api, history_server_api, spark_history_server_api
  36. from hadoop.yarn.spark_history_server_api import SparkHistoryServerApi
  37. from liboozie.oozie_api_tests import OozieServerProvider
  38. from oozie.models import Workflow
  39. from useradmin.models import User
  40. from jobbrowser import views
  41. from jobbrowser.api import get_api
  42. from jobbrowser.apis.query_api import QueryApi
  43. from jobbrowser.apis import job_api
  44. from jobbrowser.conf import SHARE_JOBS
  45. from jobbrowser.models import can_view_job, can_modify_job, LinkJobLogs
  46. from jobbrowser.yarn_models import SparkJob
  47. from datetime import datetime
  48. from babel import localtime
  49. LOG = logging.getLogger()
  50. _INITIALIZED = False
  51. class TestBrowser(object):
  52. def test_format_counter_name(self):
  53. assert_equal("Foo Bar", views.format_counter_name("fooBar"))
  54. assert_equal("Foo Bar Baz", views.format_counter_name("fooBarBaz"))
  55. assert_equal("Foo", views.format_counter_name("foo"))
  56. assert_equal("Foo.", views.format_counter_name("foo."))
  57. assert_equal("A Bbb Ccc", views.format_counter_name("A_BBB_CCC"))
  58. def get_hadoop_job_id(oozie_api, oozie_jobid, action_index=1, timeout=60, step=5):
  59. hadoop_job_id = None
  60. start = time.time()
  61. while not hadoop_job_id and time.time() - start < timeout:
  62. time.sleep(step)
  63. hadoop_job_id = oozie_api.get_job(oozie_jobid).actions[action_index].externalId
  64. if not hadoop_job_id:
  65. logs = OozieServerProvider.oozie.get_job_log(oozie_jobid)
  66. msg = "[%d] %s took more than %d to create a job: %s" % (time.time(), oozie_jobid, timeout, logs)
  67. LOG.info(msg)
  68. raise Exception(msg)
  69. return hadoop_job_id
  70. class TestJobBrowserWithHadoop(unittest.TestCase, OozieServerProvider):
  71. requires_hadoop = True
  72. integration = True
  73. @classmethod
  74. def setup_class(cls):
  75. OozieServerProvider.setup_class()
  76. cls.username = 'hue_jobbrowser_test'
  77. cls.home_dir = '/user/%s' % cls.username
  78. cls.cluster.fs.do_as_user(cls.username, cls.cluster.fs.create_home_dir, cls.home_dir)
  79. cls.client = make_logged_in_client(username=cls.username, is_superuser=False, groupname='test')
  80. cls.user = User.objects.get(username=cls.username)
  81. grant_access(cls.username, 'test', 'jobsub')
  82. grant_access(cls.username, 'test', 'jobbrowser')
  83. grant_access(cls.username, 'test', 'oozie')
  84. add_to_group(cls.username)
  85. cls.prev_user = cls.cluster.fs.user
  86. cls.cluster.fs.setuser(cls.username)
  87. cls.install_examples()
  88. cls.design = cls.create_design()
  89. # Run the sleep example, since it doesn't require user home directory
  90. design_id = cls.design.id
  91. response = cls.client.post(reverse('oozie:submit_workflow',
  92. args=[design_id]),
  93. data={u'form-MAX_NUM_FORMS': [u''],
  94. u'form-INITIAL_FORMS': [u'1'],
  95. u'form-0-name': [u'REDUCER_SLEEP_TIME'],
  96. u'form-0-value': [u'1'],
  97. u'form-TOTAL_FORMS': [u'1']},
  98. follow=True)
  99. oozie_jobid = response.context[0]['oozie_workflow'].id
  100. OozieServerProvider.wait_until_completion(oozie_jobid)
  101. cls.hadoop_job_id = get_hadoop_job_id(cls.oozie, oozie_jobid, 1)
  102. cls.hadoop_job_id_short = views.get_shorter_id(cls.hadoop_job_id)
  103. @classmethod
  104. def teardown_class(cls):
  105. try:
  106. Document.objects.filter(name__contains=cls.username).delete()
  107. Workflow.objects.filter(name__contains=cls.username).delete()
  108. # Remove user home directories.
  109. cls.cluster.fs.do_as_superuser(cls.cluster.fs.rmtree, cls.home_dir)
  110. except:
  111. LOG.exception('failed to teardown %s' % cls.home_dir)
  112. cls.cluster.fs.setuser(cls.prev_user)
  113. @classmethod
  114. def create_design(cls):
  115. job_name = '%s_%s' % (cls.username, 'sleep_job')
  116. if not Document.objects.available_docs(Workflow, cls.user).filter(name=job_name).exists():
  117. response = cls.client.post(reverse('jobsub:jobsub.views.new_design',
  118. kwargs={'node_type': 'mapreduce'}),
  119. data={'name': job_name,
  120. 'description': '',
  121. 'node_type': 'mapreduce',
  122. 'jar_path': '/user/hue/oozie/workspaces/lib/hadoop-examples.jar',
  123. 'prepares': '[]',
  124. 'files': '[]',
  125. 'archives': '[]',
  126. 'job_properties': ('[{\"name\":\"mapred.reduce.tasks\",\"value\":\"1\"},'
  127. '{\"name\":\"mapred.mapper.class\",\"value\":\"org.apache.hadoop.examples.SleepJob\"},'
  128. '{\"name\":\"mapred.reducer.class\",\"value\":\"org.apache.hadoop.examples.SleepJob\"},'
  129. '{\"name\":\"mapred.mapoutput.key.class\",\"value\":\"org.apache.hadoop.io.IntWritable\"},'
  130. '{\"name\":\"mapred.mapoutput.value.class\",\"value\":\"org.apache.hadoop.io.NullWritable\"},'
  131. '{\"name\":\"mapred.output.format.class\",\"value\":\"org.apache.hadoop.mapred.lib.NullOutputFormat\"},'
  132. '{\"name\":\"mapred.input.format.class\",\"value\":\"org.apache.hadoop.examples.SleepJob$SleepInputFormat\"},'
  133. '{\"name\":\"mapred.partitioner.class\",\"value\":\"org.apache.hadoop.examples.SleepJob\"},'
  134. '{\"name\":\"mapred.speculative.execution\",\"value\":\"false\"},'
  135. '{\"name\":\"sleep.job.map.sleep.time\",\"value\":\"0\"},'
  136. '{\"name\":\"sleep.job.reduce.sleep.time\",\"value\":\"${REDUCER_SLEEP_TIME}\"}]')
  137. },
  138. HTTP_X_REQUESTED_WITH='XMLHttpRequest')
  139. assert_equal(response.status_code, 200)
  140. return Document.objects.available_docs(Workflow, cls.user).get(name=job_name).content_object
  141. @classmethod
  142. def install_examples(cls):
  143. global _INITIALIZED
  144. if _INITIALIZED:
  145. return
  146. cls.client.post(reverse('oozie:install_examples'))
  147. cls.cluster.fs.do_as_user(cls.username, cls.cluster.fs.create_home_dir, cls.home_dir)
  148. cls.cluster.fs.do_as_superuser(cls.cluster.fs.chmod, cls.home_dir, 0o777, True)
  149. _INITIALIZED = True
  150. def test_uncommon_views(self):
  151. """
  152. These views exist, but tend not to be ever called, because they're not in the normal UI.
  153. """
  154. raise SkipTest
  155. TestJobBrowserWithHadoop.client.get("/jobbrowser/clusterstatus")
  156. TestJobBrowserWithHadoop.client.get("/jobbrowser/queues")
  157. TestJobBrowserWithHadoop.client.get("/jobbrowser/jobbrowser")
  158. def test_failed_jobs(self):
  159. """
  160. Test jobs with genuine failure, not just killed
  161. """
  162. if is_live_cluster():
  163. raise SkipTest('HUE-2902: Skipping because test is not reentrant')
  164. # Create design that will fail because the script file isn't there
  165. INPUT_DIR = TestJobBrowserWithHadoop.home_dir + '/input'
  166. OUTPUT_DIR = TestJobBrowserWithHadoop.home_dir + '/output'
  167. try:
  168. TestJobBrowserWithHadoop.cluster.fs.mkdir(TestJobBrowserWithHadoop.home_dir + "/jt-test_failed_jobs")
  169. TestJobBrowserWithHadoop.cluster.fs.mkdir(INPUT_DIR)
  170. TestJobBrowserWithHadoop.cluster.fs.rmtree(OUTPUT_DIR)
  171. except:
  172. LOG.exception('failed to teardown tests')
  173. job_name = '%s_%s' % (TestJobBrowserWithHadoop.username, 'test_failed_jobs-1')
  174. response = TestJobBrowserWithHadoop.client.post(reverse('jobsub:jobsub.views.new_design', kwargs={'node_type': 'mapreduce'}), {
  175. 'name': [job_name],
  176. 'description': ['description test_failed_jobs-1'],
  177. 'args': '',
  178. 'jar_path': '/user/hue/oozie/workspaces/lib/hadoop-examples.jar',
  179. 'prepares': '[]',
  180. 'archives': '[]',
  181. 'files': '[]',
  182. 'job_properties': ['[{"name":"mapred.input.dir","value":"%s"},\
  183. {"name":"mapred.output.dir","value":"%s"},\
  184. {"name":"mapred.mapper.class","value":"org.apache.hadoop.mapred.lib.dne"},\
  185. {"name":"mapred.combiner.class","value":"org.apache.hadoop.mapred.lib.dne"},\
  186. {"name":"mapred.reducer.class","value":"org.apache.hadoop.mapred.lib.dne"}]' % (INPUT_DIR, OUTPUT_DIR)]
  187. }, HTTP_X_REQUESTED_WITH='XMLHttpRequest', follow=True)
  188. # Submit the job
  189. design_dict = json.loads(response.content)
  190. design_id = int(design_dict['id'])
  191. response = TestJobBrowserWithHadoop.client.post(reverse('oozie:submit_workflow',
  192. args=[design_id]),
  193. data={u'form-MAX_NUM_FORMS': [u''],
  194. u'form-INITIAL_FORMS': [u'1'],
  195. u'form-0-name': [u'REDUCER_SLEEP_TIME'],
  196. u'form-0-value': [u'1'],
  197. u'form-TOTAL_FORMS': [u'1']},
  198. follow=True)
  199. oozie_jobid = response.context[0]['oozie_workflow'].id
  200. job = OozieServerProvider.wait_until_completion(oozie_jobid)
  201. hadoop_job_id = get_hadoop_job_id(TestJobBrowserWithHadoop.oozie, oozie_jobid, 1)
  202. hadoop_job_id_short = views.get_shorter_id(hadoop_job_id)
  203. # Select only killed jobs (should be absent)
  204. # Taking advantage of the fact new jobs are at the top of the list!
  205. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'state': 'killed'})
  206. assert_false(hadoop_job_id_short in response.content)
  207. # Select only failed jobs (should be present)
  208. # Map job should succeed. Reduce job should fail.
  209. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'state': 'failed'})
  210. assert_true(hadoop_job_id_short in response.content)
  211. raise SkipTest # Not compatible with MR2
  212. # The single job view should have the failed task table
  213. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s' % (hadoop_job_id,))
  214. html = response.content.lower()
  215. assert_true('failed task' in html, html)
  216. # The map task should say success (empty input)
  217. map_task_id = TestJobBrowserWithHadoop.hadoop_job_id.replace('job', 'task') + '_m_000000'
  218. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/tasks/%s' % (hadoop_job_id, map_task_id))
  219. assert_true('succeed' in response.content)
  220. assert_true('failed' not in response.content)
  221. # The reduce task should say failed
  222. reduce_task_id = hadoop_job_id.replace('job', 'task') + '_r_000000'
  223. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/tasks/%s' % (hadoop_job_id, reduce_task_id))
  224. assert_true('succeed' not in response.content)
  225. assert_true('failed' in response.content)
  226. # Selecting by failed state should include the failed map
  227. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/tasks?taskstate=failed' % (hadoop_job_id,))
  228. assert_true('r_000000' in response.content)
  229. assert_true('m_000000' not in response.content)
  230. def test_jobs_page(self):
  231. # All jobs page and fetch job ID
  232. # Taking advantage of the fact new jobs are at the top of the list!
  233. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json'})
  234. assert_true(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content, response.content)
  235. # Make sure job succeeded
  236. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'state': 'completed'})
  237. assert_true(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  238. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'state': 'failed'})
  239. assert_false(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  240. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'state': 'running'})
  241. assert_false(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  242. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'state': 'killed'})
  243. assert_false(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  244. def test_tasks_page(self):
  245. raise SkipTest
  246. # Test tracker page
  247. early_task_id = TestJobBrowserWithHadoop.hadoop_job_id.replace('job', 'task') + '_m_000000'
  248. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/tasks/%s' % (TestJobBrowserWithHadoop.hadoop_job_id, early_task_id))
  249. tracker_url = re.search('<a href="(/jobbrowser/trackers/.+?)"', response.content).group(1)
  250. response = TestJobBrowserWithHadoop.client.get(tracker_url)
  251. assert_true('Tracker at' in response.content)
  252. def test_job_permissions(self):
  253. # Login as ourself
  254. finish = SHARE_JOBS.set_for_testing(True)
  255. try:
  256. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'user': ''})
  257. assert_true(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  258. finally:
  259. finish()
  260. finish = SHARE_JOBS.set_for_testing(False)
  261. try:
  262. response = TestJobBrowserWithHadoop.client.post('/jobbrowser/jobs/', {'format': 'json', 'user': ''})
  263. assert_true(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  264. finally:
  265. finish()
  266. # Login as someone else
  267. client_not_me = make_logged_in_client(username='not_me', is_superuser=False, groupname='test')
  268. grant_access("not_me", "test", "jobbrowser")
  269. finish = SHARE_JOBS.set_for_testing(True)
  270. try:
  271. response = client_not_me.post('/jobbrowser/jobs/', {'format': 'json', 'user': ''})
  272. assert_true(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  273. finally:
  274. finish()
  275. finish = SHARE_JOBS.set_for_testing(False)
  276. try:
  277. response = client_not_me.post('/jobbrowser/jobs/', {'format': 'json', 'user': ''})
  278. assert_false(TestJobBrowserWithHadoop.hadoop_job_id_short in response.content)
  279. finally:
  280. finish()
  281. def test_job_counter(self):
  282. raise SkipTest
  283. # Single job page
  284. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s' % TestJobBrowserWithHadoop.hadoop_job_id)
  285. # Check some counters for single job.
  286. counters = response.context[0]['job'].counters
  287. counters_file_bytes_written = counters['org.apache.hadoop.mapreduce.FileSystemCounter']['counters']['FILE_BYTES_WRITTEN']
  288. assert_true(counters_file_bytes_written['map'] > 0)
  289. assert_true(counters_file_bytes_written['reduce'] > 0)
  290. def test_task_page(self):
  291. raise SkipTest
  292. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/tasks' % (TestJobBrowserWithHadoop.hadoop_job_id,))
  293. assert_true(len(response.context[0]['page'].object_list), 4)
  294. # Select by tasktype
  295. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/tasks?tasktype=reduce' % (TestJobBrowserWithHadoop.hadoop_job_id,))
  296. assert_true(len(response.context[0]['page'].object_list), 1)
  297. # Select by taskstate
  298. response = TestJobBrowserWithHadoop.client.get(
  299. '/jobbrowser/jobs/%s/tasks?taskstate=succeeded' % (TestJobBrowserWithHadoop.hadoop_job_id,)
  300. )
  301. assert_true(len(response.context[0]['page'].object_list), 4)
  302. # Select by text
  303. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/tasks?tasktext=clean' % (TestJobBrowserWithHadoop.hadoop_job_id,))
  304. assert_true(len(response.context[0]['page'].object_list), 1)
  305. def test_job_single_logs(self):
  306. if not is_live_cluster():
  307. raise SkipTest
  308. response = TestJobBrowserWithHadoop.client.get('/jobbrowser/jobs/%s/single_logs?format=json' % (TestJobBrowserWithHadoop.hadoop_job_id))
  309. json_resp = json.loads(response.content)
  310. assert_true('logs' in json_resp)
  311. assert_true('Log Type: stdout' in json_resp['logs'][1])
  312. assert_true('Log Type: stderr' in json_resp['logs'][2])
  313. assert_true('Log Type: syslog' in json_resp['logs'][3])
  314. # Verify that syslog contains log information for a completed oozie job
  315. match = re.search(r"^Log Type: syslog(.+)Log Length: (?P<log_length>\d+)(.+)$", json_resp['logs'][3], re.DOTALL)
  316. assert_true(match and match.group(2), 'Failed to parse log length from syslog')
  317. log_length = match.group(2)
  318. assert_true(log_length > 0, 'Log Length is 0, expected content in syslog.')
  319. class TestMapReduce2NoHadoop(object):
  320. def setUp(self):
  321. # Beware: Monkey patching
  322. if not hasattr(resource_manager_api, 'old_get_resource_manager_api'):
  323. resource_manager_api.old_get_resource_manager = resource_manager_api.get_resource_manager
  324. if not hasattr(mapreduce_api, 'old_get_mapreduce_api'):
  325. mapreduce_api.old_get_mapreduce_api = mapreduce_api.get_mapreduce_api
  326. if not hasattr(history_server_api, 'old_get_history_server_api'):
  327. history_server_api.old_get_history_server_api = history_server_api.get_history_server_api
  328. if not hasattr(spark_history_server_api, 'old_get_spark_history_server_api'):
  329. spark_history_server_api.old_get_spark_history_server_api = spark_history_server_api.get_history_server_api
  330. self.c = make_logged_in_client(is_superuser=False)
  331. grant_access("test", "test", "jobbrowser")
  332. self.user = User.objects.get(username='test')
  333. self.c2 = make_logged_in_client(is_superuser=False, username="test2")
  334. grant_access("test2", "test2", "jobbrowser")
  335. self.user2 = User.objects.get(username='test2')
  336. self.c3 = make_logged_in_client(is_superuser=False, username="test3")
  337. grant_access("test3", "test3", "jobbrowser")
  338. self.user3 = User.objects.get(username='test3')
  339. resource_manager_api.get_resource_manager = lambda username: MockResourceManagerApi(username)
  340. mapreduce_api.get_mapreduce_api = lambda username: MockMapreduceApi(username)
  341. history_server_api.get_history_server_api = lambda username: HistoryServerApi(username)
  342. spark_history_server_api.get_history_server_api = lambda: MockSparkHistoryApi()
  343. self.finish = [
  344. YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True),
  345. SHARE_JOBS.set_for_testing(False)
  346. ]
  347. assert_true(cluster.is_yarn())
  348. def tearDown(self):
  349. resource_manager_api.get_resource_manager = getattr(resource_manager_api, 'old_get_resource_manager')
  350. mapreduce_api.get_mapreduce_api = getattr(mapreduce_api, 'old_get_mapreduce_api')
  351. history_server_api.get_history_server_api = getattr(history_server_api, 'old_get_history_server_api')
  352. spark_history_server_api.get_history_server_api = getattr(spark_history_server_api, 'old_get_spark_history_server_api')
  353. for f in self.finish:
  354. f()
  355. def test_jobs(self):
  356. response = self.c.post('/jobbrowser/', {'format': 'json'})
  357. response_content = json.loads(response.content)
  358. assert_equal(len(response_content['jobs']), 4)
  359. response = self.c.post('/jobbrowser/jobs/', {'format': 'json', 'text': 'W=MapReduce-copy2'})
  360. response_content = json.loads(response.content)
  361. assert_equal(len(response_content['jobs']), 1)
  362. def test_applications_no_start_time(self):
  363. response = self.c.post('/jobbrowser/', {'format': 'json'})
  364. data = json.loads(response.content)
  365. job = [j for j in data['jobs'] if j['id'] == 'application_1428442704693_0007']
  366. assert_true(job, job)
  367. job = job[0]
  368. assert_equal('', job['startTimeFormatted'], data)
  369. assert_equal('', job['durationFormatted'], data)
  370. def test_running_job(self):
  371. response = self.c.get('/jobbrowser/jobs/application_1356251510842_0054')
  372. assert_true(b'job_1356251510842_0054' in response.content, response.content)
  373. assert_true(b'RUNNING' in response.content)
  374. response = self.c.get('/jobbrowser/jobs/job_1356251510842_0054')
  375. assert_true(b'job_1356251510842_0054' in response.content)
  376. assert_true(b'RUNNING' in response.content)
  377. def test_application_no_start_time(self):
  378. response = self.c.get('/jobbrowser/jobs/application_1428442704693_0007?format=json')
  379. data = json.loads(response.content)
  380. assert_equal('', data['job']['startTimeFormatted'], data)
  381. assert_equal('', data['job']['durationFormatted'], data)
  382. def test_finished_job(self):
  383. response = self.c.get('/jobbrowser/jobs/application_1356251510842_0009')
  384. assert_equal(response.context[0]['job'].jobId, 'job_1356251510842_0009')
  385. response = self.c.get('/jobbrowser/jobs/job_1356251510842_0009')
  386. assert_equal(response.context[0]['job'].jobId, 'job_1356251510842_0009')
  387. def test_spark_job(self):
  388. response = self.c.get('/jobbrowser/jobs/application_1428442704693_0006')
  389. assert_equal(response.context[0]['job'].jobId, 'application_1428442704693_0006')
  390. def test_yarn_job(self):
  391. response = self.c.get('/jobbrowser/jobs/application_1428442704693_0007')
  392. assert_equal(response.context[0]['job'].jobId, 'job_1356251510842_0009')
  393. def job_not_assigned(self):
  394. response = self.c.get('/jobbrowser/jobs/job_1356251510842_0009/job_not_assigned//my_url')
  395. assert_equal(response.context[0]['jobid'], 'job_1356251510842_0009')
  396. assert_equal(response.context[0]['path'], '/my_url')
  397. response = self.c.get('/jobbrowser/jobs/job_1356251510842_0009/job_not_assigned//my_url?format=json')
  398. result = json.loads(response.content)
  399. assert_equal(result['status'], 0)
  400. def test_acls_job(self):
  401. response = self.c.get('/jobbrowser/jobs/job_1356251510842_0054') # Check in perm decorator
  402. assert_true(can_view_job('test', response.context[0]['job']))
  403. assert_true(can_modify_job('test', response.context[0]['job']))
  404. assert_true(can_view_job('test2', response.context[0]['job']))
  405. assert_false(can_modify_job('test2', response.context[0]['job']))
  406. assert_false(can_view_job('test3', response.context[0]['job']))
  407. assert_false(can_modify_job('test3', response.context[0]['job']))
  408. response2 = self.c3.get('/jobbrowser/jobs/job_1356251510842_0054')
  409. if sys.version_info[0] < 3:
  410. assert_true(b'don&#39;t have permission to access job' in response2.content, response2.content)
  411. else:
  412. assert_true(b'don&#x27;t have permission to access job' in response2.content, response2.content)
  413. def test_kill_job(self):
  414. job_id = 'application_1356251510842_0054'
  415. try:
  416. response = self.c.post('/jobbrowser/jobs/%s/kill?format=json' % job_id)
  417. assert_equal(json.loads(response.content), {"status": 0})
  418. finally:
  419. MockResourceManagerApi.APPS[job_id]['state'] = 'RUNNING'
  420. response = self.c2.post('/jobbrowser/jobs/%s/kill?format=json' % job_id)
  421. assert_true(b'Kill operation is forbidden.' in response.content, response.content)
  422. class TestResourceManagerHaNoHadoop(object):
  423. def setUp(self):
  424. # Beware: Monkey patching
  425. if not hasattr(resource_manager_api, 'old_get_resource_manager_api'):
  426. resource_manager_api.old_ResourceManagerApi = resource_manager_api.ResourceManagerApi
  427. if not hasattr(mapreduce_api, 'old_get_mapreduce_api'):
  428. mapreduce_api.old_get_mapreduce_api = mapreduce_api.get_mapreduce_api
  429. if not hasattr(history_server_api, 'old_get_history_server_api'):
  430. history_server_api.old_get_history_server_api = history_server_api.get_history_server_api
  431. self.c = make_logged_in_client(is_superuser=False)
  432. grant_access("test", "test", "jobbrowser")
  433. self.user = User.objects.get(username='test')
  434. resource_manager_api.ResourceManagerApi = MockResourceManagerHaApi
  435. mapreduce_api.get_mapreduce_api = lambda username: MockMapreduceHaApi(username)
  436. history_server_api.get_history_server_api = lambda username: HistoryServerHaApi(username)
  437. self.finish = []
  438. def tearDown(self):
  439. resource_manager_api.ResourceManagerApi = getattr(resource_manager_api, 'old_ResourceManagerApi')
  440. resource_manager_api.API_CACHE = None
  441. mapreduce_api.get_mapreduce_api = getattr(mapreduce_api, 'old_get_mapreduce_api')
  442. history_server_api.get_history_server_api = getattr(history_server_api, 'old_get_history_server_api')
  443. for f in self.finish:
  444. f()
  445. def test_failover_no_ha(self):
  446. self.finish = [
  447. YARN_CLUSTERS.set_for_testing({'default': {}}),
  448. YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True),
  449. YARN_CLUSTERS['default'].RESOURCE_MANAGER_API_URL.set_for_testing('rm_host_active'),
  450. YARN_CLUSTERS['default'].HISTORY_SERVER_API_URL.set_for_testing('jhs_host'),
  451. YARN_CLUSTERS['default'].SECURITY_ENABLED.set_for_testing(False),
  452. YARN_CLUSTERS['default'].SSL_CERT_CA_VERIFY.set_for_testing(False),
  453. ]
  454. resource_manager_api.API_CACHE = None
  455. api = get_api(self.user, jt=None)
  456. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  457. assert_false(api.resource_manager_api.from_failover)
  458. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  459. assert_false(api.resource_manager_api.from_failover)
  460. assert_raises(Exception, api.get_jobs, self.user, username=self.user.username, state='running', text='')
  461. def test_failover_ha(self):
  462. self.finish = [
  463. YARN_CLUSTERS.set_for_testing({'ha1': {}, 'ha2': {}}),
  464. YARN_CLUSTERS['ha1'].SUBMIT_TO.set_for_testing(True),
  465. YARN_CLUSTERS['ha1'].RESOURCE_MANAGER_API_URL.set_for_testing('rm_host_active'),
  466. YARN_CLUSTERS['ha1'].HISTORY_SERVER_API_URL.set_for_testing('jhs_host'),
  467. YARN_CLUSTERS['ha1'].SECURITY_ENABLED.set_for_testing(False),
  468. YARN_CLUSTERS['ha1'].SSL_CERT_CA_VERIFY.set_for_testing(False),
  469. YARN_CLUSTERS['ha2'].SUBMIT_TO.set_for_testing(True),
  470. YARN_CLUSTERS['ha2'].RESOURCE_MANAGER_API_URL.set_for_testing('rm_2_host'),
  471. YARN_CLUSTERS['ha2'].HISTORY_SERVER_API_URL.set_for_testing('jhs_host'),
  472. YARN_CLUSTERS['ha2'].SECURITY_ENABLED.set_for_testing(False),
  473. YARN_CLUSTERS['ha2'].SSL_CERT_CA_VERIFY.set_for_testing(False),
  474. ]
  475. resource_manager_api.API_CACHE = None
  476. api = get_api(self.user, jt=None)
  477. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  478. assert_false(api.resource_manager_api.from_failover)
  479. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  480. assert_false(api.resource_manager_api.from_failover)
  481. # rm1 is set to to fail the 3rd time
  482. YARN_CLUSTERS['ha1'].RESOURCE_MANAGER_API_URL.set_for_testing('rm_1_host')
  483. YARN_CLUSTERS['ha2'].RESOURCE_MANAGER_API_URL.set_for_testing('rm_2_host_active') # Just tells mocked RM that it should say it is active
  484. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  485. assert_true(api.resource_manager_api.from_failover)
  486. api.resource_manager_api.from_failover = False
  487. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  488. assert_false(api.resource_manager_api.from_failover)
  489. # rm2 is set to to fail the 3rd time
  490. YARN_CLUSTERS['ha1'].RESOURCE_MANAGER_API_URL.set_for_testing('rm_1_host_active')
  491. YARN_CLUSTERS['ha2'].RESOURCE_MANAGER_API_URL.set_for_testing('rm_2_host')
  492. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  493. assert_true(api.resource_manager_api.from_failover)
  494. api.resource_manager_api.from_failover = False
  495. api.get_jobs(self.user, username=self.user.username, state='running', text='')
  496. assert_false(api.resource_manager_api.from_failover)
  497. # if rm fails and no other active ones we fail
  498. assert_raises(Exception, api.get_jobs, self.user, username=self.user.username, state='running', text='')
  499. class TestImpalaApi(object):
  500. def setUp(self):
  501. api = MockImpalaQueryApi('http://url.com')
  502. self.api = QueryApi(None, impala_api=api)
  503. def handle_query_start_time(self, start_time):
  504. query_start_time = datetime.strptime(start_time[:-3], "%Y-%m-%d %H:%M:%S.%f"). \
  505. replace(tzinfo=pytz.utc).astimezone(localtime._get_localzone()).strftime("%Y-%m-%d %H:%M:%S.%f")
  506. return query_start_time
  507. def test_apps(self):
  508. response = self.api.apps({})
  509. target = [
  510. {
  511. 'status': u'FINISHED',
  512. 'rows_fetched': 28,
  513. 'user': u'admin',
  514. 'canWrite': False,
  515. 'duration': 3355000.0,
  516. 'id': u'8a46a8865624698f:b80b211500000000',
  517. 'apiStatus': 'SUCCEEDED',
  518. 'name': u'SELECT sample_07.description, sample_07.salary FROM sample...',
  519. 'submitted': self.handle_query_start_time('2017-10-25 15:38:26.637010000'),
  520. 'queue': u'root.admin',
  521. 'waiting': True, 'progress': u'1 / 1 ( 100%)',
  522. 'type': u'QUERY', 'waiting_time': u'52m8s'
  523. },
  524. {
  525. 'status': u'FINISHED',
  526. 'rows_fetched': 53, 'user': u'admin', 'canWrite': False,
  527. 'duration': 3369000.0, 'id': u'4d497267f34ff17d:817bdfb500000000',
  528. 'apiStatus': 'SUCCEEDED', 'name': u'select * from customers',
  529. 'submitted': self.handle_query_start_time('2017-10-25 15:38:12.872825000'), 'queue': u'root.admin',
  530. 'waiting': True, 'progress': u'2 / 3 (66.6667%)', 'type': u'QUERY', 'waiting_time': u'52m8s'
  531. }
  532. ]
  533. for i in range(0, len(target)):
  534. for key, value in target[i].items():
  535. assert_equal(response.get('apps')[i].get(key), value)
  536. def test_app(self):
  537. response = self.api.app('4d497267f34ff17d:817bdfb500000000')
  538. for key, value in {'status': u'FINISHED', 'name': u'select * from customers',
  539. 'duration': 3369000.0, 'progress': 66.6667, 'user': u'admin', 'type': 'queries',
  540. 'id': '4d497267f34ff17d:817bdfb500000000',
  541. 'submitted': self.handle_query_start_time('2017-10-25 15:38:12.872825000'),
  542. 'apiStatus': 'SUCCEEDED', 'doc_url': 'http://url.com/query_plan?query_id=4d497267f34ff17d:817bdfb500000000'}.items():
  543. assert_equal(response.get(key), value)
  544. response = self.api.app('8a46a8865624698f:b80b211500000000')
  545. for key, value in {'status': u'FINISHED', 'name': u'SELECT sample_07.description, sample_07.salary FROM sample...',
  546. 'duration': 3355000.0, 'progress': 100.0, 'user': u'admin', 'type': 'queries',
  547. 'id': '8a46a8865624698f:b80b211500000000',
  548. 'submitted': self.handle_query_start_time('2017-10-25 15:38:26.637010000'), 'apiStatus': 'SUCCEEDED',
  549. 'doc_url': 'http://url.com/query_plan?query_id=8a46a8865624698f:b80b211500000000'}.items():
  550. assert_equal(response.get(key), value)
  551. class TestSparkNoHadoop(object):
  552. def setUp(self):
  553. self.c = make_logged_in_client(is_superuser=False)
  554. grant_access("test", "test", "jobbrowser")
  555. self.user = User.objects.get(username='test')
  556. if not hasattr(job_api, 'old_NativeYarnApi'):
  557. job_api.old_NativeYarnApi = job_api.YarnApi
  558. if not hasattr(views, 'old_get_api'):
  559. views.old_get_api = views.get_api
  560. job_api.NativeYarnApi = MockYarnApi
  561. views.get_api = MockYarnApi
  562. def tearDown(self):
  563. job_api.NativeYarnApi = getattr(job_api, 'old_NativeYarnApi')
  564. views.get_api = getattr(views, 'old_get_api')
  565. def test_spark_executor_logs(self):
  566. # Spark job status is succeed
  567. query_executor_data = {u'interface': [u'"jobs"'], u'app_id': [u'"driver_executor_application_1513618343677_0018"']}
  568. resp_executor = self.c.post('/jobbrowser/api/job/jobs', query_executor_data)
  569. response_executor = json.loads(resp_executor.content)
  570. assert_equal(response_executor['status'], 0)
  571. assert_equal(response_executor['app']['executor_id'], 'driver')
  572. query_log_data = {u'interface': [u'"jobs"'], u'type': [u'"SPARK"'],
  573. u'app_id': [u'"application_1513618343677_0018"'], u'name': [u'"default"']}
  574. resp_log = self.c.post('/jobbrowser/api/job/logs', query_log_data)
  575. response_log = json.loads(resp_log.content)
  576. assert_equal(response_log['status'], 0)
  577. assert_equal(response_log['logs']['logs'], 'dummy_logs')
  578. # Spark job status is running
  579. query_executor_data = {u'interface': [u'"jobs"'], u'app_id': [u'"driver_executor_application_1513618343677_0020"']}
  580. resp_executor = self.c.post('/jobbrowser/api/job/jobs', query_executor_data)
  581. response_executor = json.loads(resp_executor.content)
  582. assert_equal(response_executor['status'], 0)
  583. assert_equal(response_executor['app']['executor_id'], 'driver')
  584. query_log_data = {u'interface': [u'"jobs"'], u'type': [u'"SPARK"'],
  585. u'app_id': [u'"application_1513618343677_0020"'], u'name': [u'"default"']}
  586. resp_log = self.c.post('/jobbrowser/api/job/logs', query_log_data)
  587. response_log = json.loads(resp_log.content)
  588. assert_equal(response_log['status'], 0)
  589. assert_equal(response_log['logs']['logs'], 'dummy_logs')
  590. class MockYarnApi(object):
  591. def __init__(self, user, jt=None):
  592. self.user = user
  593. def get_job(self, jobid):
  594. return MockSparkJob(app_id=jobid)
  595. class MockSparkJob(SparkJob):
  596. def __init__(self, app_id):
  597. self.history_server_api = MockSparkHistoryApi()
  598. self.jobId = app_id
  599. self.trackingUrl = 'http://localhost:8088/proxy/' + app_id
  600. if app_id == 'application_1513618343677_0018':
  601. self.status = 'SUCCEEDED'
  602. elif app_id == 'application_1513618343677_0020':
  603. self.status = 'RUNNING'
  604. self._get_metrics()
  605. class MockResourceManagerHaApi(object):
  606. """
  607. Mock the RM API.
  608. Raise a failover exception after 2 calls. Is active if name contains 'active'.
  609. """
  610. def __init__(self, rm_url, security_enabled=False, ssl_cert_ca_verify=False):
  611. self.rm_url = rm_url
  612. self.from_failover = False
  613. self.get_apps_count = 0
  614. def setuser(self, user):
  615. return user
  616. @property
  617. def user(self):
  618. return 'test'
  619. @property
  620. def username(self):
  621. return 'test'
  622. @property
  623. def url(self):
  624. return self.rm_url
  625. def apps(self, **kwargs):
  626. if self.get_apps_count >= 2:
  627. self.get_apps_count = 0
  628. raise Exception('standby RM after 2 tries')
  629. self.get_apps_count += 1
  630. return {
  631. 'apps': {
  632. 'app': []
  633. }
  634. }
  635. def cluster(self):
  636. return {'clusterInfo': {'haState': 'ACTIVE' if 'active' in self.rm_url else 'STANDBY'}}
  637. class MockMapreduceHaApi(object):
  638. def __init__(self, username): pass
  639. class HistoryServerHaApi(object):
  640. def __init__(self, username): pass
  641. class MockResourceManagerApi(object):
  642. APPS = {
  643. 'application_1356251510842_0054': {
  644. u'finishedTime': 1356961070119,
  645. u'name': u'oozie:launcher:T=map-reduce:W=MapReduce-copy:A=Sleep:ID=0000004-121223003201296-oozie-oozi-W',
  646. u'amContainerLogs': u'http://localhost:8042/node/containerlogs/container_1356251510842_0054_01_000001/romain',
  647. u'clusterId': 1356251510842,
  648. u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0054/jobhistory/job/job_1356251510842_0054',
  649. u'amHostHttpAddress': u'localhost:8042',
  650. u'startedTime': 1356961057225,
  651. u'queue': u'default',
  652. u'state': u'RUNNING',
  653. u'elapsedTime': 12894,
  654. u'finalStatus': u'UNDEFINED',
  655. u'diagnostics': u'',
  656. u'progress': 100.0,
  657. u'trackingUI': u'History',
  658. u'id': u'application_1356251510842_0054',
  659. u'user': u'test',
  660. # For when the job is KILLED
  661. u'startTime': 1356961057226,
  662. u'finishTime': 1356961057226,
  663. u'applicationType': 'MAPREDUCE'
  664. },
  665. 'application_1356251510842_0009': {
  666. u'finishedTime': 1356467118570,
  667. u'name': u'oozie:action:T=map-reduce:W=MapReduce-copy2:A=Sleep:ID=0000002-121223003201296-oozie-oozi-W',
  668. u'amContainerLogs': u'http://localhost:8042/node/containerlogs/container_1356251510842_0009_01_000001/romain',
  669. u'clusterId': 1356251510842,
  670. u'trackingUrl': u'http://localhost:8088/proxy/application_1356251510842_0009/jobhistory/job/job_1356251510842_0009',
  671. u'amHostHttpAddress': u'localhost:8042',
  672. u'startedTime': 1356467081121,
  673. u'queue': u'default',
  674. u'state': u'FINISHED',
  675. u'elapsedTime': 37449,
  676. u'finalStatus': u'SUCCEEDED',
  677. u'diagnostics': u'',
  678. u'progress': 100.0,
  679. u'trackingUI': u'History',
  680. u'id': u'application_1356251510842_0009',
  681. u'user': u'test',
  682. u'applicationType': 'MAPREDUCE'
  683. },
  684. 'application_1428442704693_0006': {
  685. u'allocatedMB': 4096,
  686. u'allocatedVCores': 3,
  687. u'amContainerLogs': u'http://localhost:8042/node/containerlogs/container_1428442704693_0006_01_000001/erickt',
  688. u'amHostHttpAddress': u'localhost:8042',
  689. u'applicationTags': u'',
  690. u'applicationType': u'SPARK',
  691. u'clusterId': 1428442704693,
  692. u'diagnostics': u'',
  693. u'elapsedTime': 529040,
  694. u'finalStatus': u'UNDEFINED',
  695. u'finishedTime': 0,
  696. u'id': u'application_1428442704693_0006',
  697. u'memorySeconds': 2138468,
  698. u'name': u'Spark shell',
  699. u'numAMContainerPreempted': 0,
  700. u'numNonAMContainerPreempted': 0,
  701. u'preemptedResourceMB': 0,
  702. u'preemptedResourceVCores': 0,
  703. u'progress': 10.0,
  704. u'queue': u'root.erickt',
  705. u'runningContainers': 3,
  706. u'startedTime': 1428443335161,
  707. u'state': u'RUNNING',
  708. u'trackingUI': u'ApplicationMaster',
  709. u'trackingUrl': u'http://localhost:8088/proxy/application_1428442704693_0006/',
  710. u'user': u'test',
  711. u'vcoreSeconds': 1567,
  712. },
  713. 'application_1428442704693_0007': {
  714. u'allocatedMB': -1,
  715. u'allocatedVCores': -1,
  716. u'applicationTags': u'',
  717. u'applicationType': u'YARN',
  718. u'clusterId': 1428442704693,
  719. u'diagnostics': u'',
  720. u'elapsedTime': 4056,
  721. u'finalStatus': u'SUCCEEDED',
  722. u'finishedTime': 1428454945371,
  723. u'id': u'application_1428442704693_0007',
  724. u'memorySeconds': 2290,
  725. u'name': u'UnmanagedAM',
  726. u'numAMContainerPreempted': 0,
  727. u'numNonAMContainerPreempted': 0,
  728. u'preemptedResourceMB': 0,
  729. u'preemptedResourceVCores': 0,
  730. u'progress': 100.0,
  731. u'queue': u'root.erickt',
  732. u'runningContainers': -1,
  733. u'startedTime': 0,
  734. u'state': u'FINISHED',
  735. u'trackingUI': u'History',
  736. u'trackingUrl': u'http://N/A',
  737. u'user': u'test',
  738. u'vcoreSeconds': 1,
  739. }
  740. }
  741. def __init__(self, user, rm_url=None): pass
  742. def apps(self, **kwargs):
  743. return {
  744. 'apps': {
  745. 'app': [
  746. # RUNNING
  747. MockResourceManagerApi.APPS['application_1356251510842_0054'],
  748. # FINISHED
  749. MockResourceManagerApi.APPS['application_1356251510842_0009'],
  750. # SPARK
  751. MockResourceManagerApi.APPS['application_1428442704693_0006'],
  752. # YARN
  753. MockResourceManagerApi.APPS['application_1428442704693_0007'],
  754. ]
  755. }
  756. }
  757. def app(self, job_id):
  758. return {
  759. u'app': MockResourceManagerApi.APPS[job_id]
  760. }
  761. class MockImpalaQueryApi(object):
  762. APPS = {
  763. '8a46a8865624698f:b80b211500000000': {
  764. u'stmt_type': u'QUERY', u'resource_pool': u'root.admin',
  765. u'waiting': True, u'last_event': u'Unregister query',
  766. u'start_time': u'2017-10-25 15:38:26.637010000',
  767. u'rows_fetched': 28,
  768. u'stmt': u'SELECT sample_07.description, sample_07.salary\r\nFROM\r\n '\
  769. 'sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000',
  770. u'executing': False, u'state': u'FINISHED', u'query_id': u'8a46a8865624698f:b80b211500000000',
  771. u'end_time': u'2017-10-25 16:34:22.592036000', u'duration': u'55m55s', u'progress': u'1 / 1 ( 100%)',
  772. u'effective_user': u'admin', u'default_db': u'default', u'waiting_time': u'52m8s'
  773. },
  774. '4d497267f34ff17d:817bdfb500000000': {
  775. u'stmt_type': u'QUERY', u'resource_pool': u'root.admin', u'waiting': True,
  776. u'last_event': u'Unregister query', u'start_time': u'2017-10-25 15:38:12.872825000',
  777. u'rows_fetched': 53, u'stmt': u'select * from customers', u'executing': False,
  778. u'state': u'FINISHED', u'query_id': u'4d497267f34ff17d:817bdfb500000000',
  779. u'end_time': u'2017-10-25 16:34:22.589811000', u'duration': u'56m9s', u'progress': u'2 / 3 (66.6667%)',
  780. u'effective_user': u'admin', u'default_db': u'default', u'waiting_time': u'52m8s'
  781. }
  782. }
  783. PLAN = {
  784. '4d497267f34ff17d:817bdfb500000000': {
  785. 'status': -1,
  786. u'plan': {
  787. u'status': u'OK',
  788. u'plan_json': {
  789. u'plan_nodes': [
  790. {
  791. u'num_instances': 1, u'output_card': 53, u'label_detail': u'UNPARTITIONED', u'label': u'01:EXCHANGE', u'is_broadcast': True,
  792. u'max_time': u'0.000ns', u'avg_time': u'0.000ns', u'children': [], u'max_time_val': 0
  793. },
  794. {
  795. u'num_instances': 1, u'output_card': 53, u'label_detail': u'default.customers', u'data_stream_target': u'01:EXCHANGE',
  796. u'label': u'00:SCAN HDFS', u'max_time': u'215.018ms', u'avg_time': u'215.018ms', u'children': [], u'max_time_val': 215018404
  797. }
  798. ]
  799. },
  800. u'__common__': {
  801. u'navbar': [
  802. {u'link': u'/backends', u'title': u'/backends'},
  803. {u'link': u'/catalog', u'title': u'/catalog'},
  804. {u'link': u'/hadoop-varz', u'title': u'/hadoop-varz'},
  805. {u'link': u'/log_level', u'title': u'/log_level'},
  806. {u'link': u'/logs', u'title': u'/logs'},
  807. {u'link': u'/memz', u'title': u'/memz'},
  808. {u'link': u'/metrics', u'title': u'/metrics'},
  809. {u'link': u'/queries', u'title': u'/queries'},
  810. {u'link': u'/rpcz', u'title': u'/rpcz'},
  811. {u'link': u'/sessions', u'title': u'/sessions'},
  812. {u'link': u'/threadz', u'title': u'/threadz'},
  813. {u'link': u'/varz', u'title': u'/varz'}
  814. ],
  815. u'process-name': u'impalad'
  816. },
  817. u'stmt': u'select * from customers',
  818. u'summary': u'\nOperator #Hosts Avg Time Max Time #Rows Est. #Rows Peak Mem Est. Peak Mem Detail '\
  819. ' \n-------------------------------------------------------------------------------------------------'\
  820. '----------\n01:EXCHANGE 1 0.000ns 0.000ns 53 0 0 0 UNPARTITIONED '\
  821. ' \n00:SCAN HDFS 1 215.018ms 215.018ms 53 0 45.02 KB 32.00 MB default.customers ',
  822. u'query_id': u'1a48b5796f8f07f5:49ba9e6b00000000',
  823. u'plan': u'\n----------------\nPer-Host Resource Reservation: Memory=0B\nPer-Host Resource Estimates: Memory=32.00MB\nWARNING: '\
  824. 'The following tables have potentially corrupt table statistics.\nDrop and re-compute statistics to resolve this '
  825. 'problem.\ndefault.customers\nWARNING: The following tables are missing relevant table and/or column statistics.'\
  826. '\ndefault.customers\n\nF01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1\nPLAN-ROOT SINK\n| mem-estimate=0B '\
  827. 'mem-reservation=0B\n|\n01:EXCHANGE [UNPARTITIONED]\n| mem-estimate=0B mem-reservation=0B\n| tuple-ids=0 row-size=19B '\
  828. 'cardinality=0\n|\nF00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1\n00:SCAN HDFS [default.customers, RANDOM]\n partitions=1/1 '\
  829. 'files=1 size=15.44KB\n table stats: 0 rows total\n column stats: unavailable\n mem-estimate=32.00MB mem-reservation=0B'\
  830. '\n tuple-ids=0 row-size=19B cardinality=0\n----------------'
  831. }
  832. },
  833. '8a46a8865624698f:b80b211500000000': {
  834. 'status': -1,
  835. u'plan': {
  836. u'status': u'OK',
  837. u'plan_json': {
  838. u'plan_nodes': [
  839. {
  840. u'num_instances': 1, u'output_card': 28, u'label_detail': u'UNPARTITIONED', u'label': u'02:MERGING-EXCHANGE',
  841. u'is_broadcast': True, u'max_time': u'0.000ns', u'avg_time': u'0.000ns', u'children': [], u'max_time_val': 0
  842. },
  843. {
  844. u'num_instances': 1, u'output_card': 28, u'label_detail': u'', u'data_stream_target': u'02:MERGING-EXCHANGE',
  845. u'label': u'01:TOP-N', u'max_time': u'0.000ns', u'avg_time': u'0.000ns',
  846. u'children': [
  847. {
  848. u'num_instances': 1, u'output_card': 28, u'label_detail': u'default.sample_07', u'label': u'00:SCAN HDFS',
  849. u'max_time': u'250.020ms', u'avg_time': u'250.020ms', u'children': [], u'max_time_val': 250020583
  850. }
  851. ],
  852. u'max_time_val': 0
  853. }
  854. ]
  855. },
  856. u'__common__': {
  857. u'navbar': [
  858. {u'link': u'/backends', u'title': u'/backends'},
  859. {u'link': u'/catalog', u'title': u'/catalog'},
  860. {u'link': u'/hadoop-varz', u'title': u'/hadoop-varz'},
  861. {u'link': u'/log_level', u'title': u'/log_level'},
  862. {u'link': u'/logs', u'title': u'/logs'},
  863. {u'link': u'/memz', u'title': u'/memz'},
  864. {u'link': u'/metrics', u'title': u'/metrics'},
  865. {u'link': u'/queries', u'title': u'/queries'},
  866. {u'link': u'/rpcz', u'title': u'/rpcz'},
  867. {u'link': u'/sessions', u'title': u'/sessions'},
  868. {u'link': u'/threadz', u'title': u'/threadz'},
  869. {u'link': u'/varz', u'title': u'/varz'}
  870. ],
  871. u'process-name': u'impalad'
  872. },
  873. u'stmt': u'SELECT sample_07.description, sample_07.salary\r\nFROM\r\n '\
  874. 'sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY sample_07.salary DESC\r\nLIMIT 1000',
  875. u'summary': u'\nOperator #Hosts Avg Time Max Time #Rows Est. #Rows Peak Mem Est. Peak Mem '\
  876. 'Detail \n-------------------------------------------------------------------------------------------'\
  877. '------------------------\n02:MERGING-EXCHANGE 1 0.000ns 0.000ns 28 0 0 '\
  878. ' 0 UNPARTITIONED \n01:TOP-N 1 0.000ns 0.000ns 28 0 80.00 KB '\
  879. ' 0 \n00:SCAN HDFS 1 250.020ms 250.020ms 28 0 173.00 KB '\
  880. '32.00 MB default.sample_07 ',
  881. u'query_id': u'd424420e0c44ab9:c637ac2900000000',
  882. u'plan': u'\n----------------\nPer-Host Resource Reservation: Memory=0B\nPer-Host Resource Estimates: Memory=32.00MB\nWARNING: '\
  883. 'The following tables have potentially corrupt table statistics.\nDrop and re-compute statistics to resolve this problem.'\
  884. '\ndefault.sample_07\nWARNING: The following tables are missing relevant table and/or column statistics.\n'\
  885. 'default.sample_07\n\nF01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1\nPLAN-ROOT SINK\n| mem-estimate=0B '\
  886. 'mem-reservation=0B\n|\n02:MERGING-EXCHANGE [UNPARTITIONED]\n| order by: salary DESC\n| limit: 1000\n| mem-estimate=0B '\
  887. 'mem-reservation=0B\n| tuple-ids=1 row-size=19B cardinality=0\n|\nF00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1\n01:TOP-N '\
  888. '[LIMIT=1000]\n| order by: salary DESC\n| mem-estimate=0B mem-reservation=0B\n| tuple-ids=1 row-size=19B '\
  889. 'cardinality=0\n|\n00:SCAN HDFS [default.sample_07, RANDOM]\n partitions=1/1 files=1 size=44.98KB\n predicates: '\
  890. '(sample_07.salary > 100000)\n table stats: 0 rows total\n column stats: unavailable\n parquet dictionary predicates: '\
  891. '(sample_07.salary > 100000)\n mem-estimate=32.00MB mem-reservation=0B\n tuple-ids=0 row-size=19B '\
  892. 'cardinality=0\n----------------'
  893. }
  894. }
  895. }
  896. PROFILE = {
  897. '4d497267f34ff17d:817bdfb500000000': {
  898. u'profile': u'Query (id=1a48b5796f8f07f5:49ba9e6b00000000):\n Summary:\n Session ID: 3348564c97187569:1c17ce45bdfbf0b2\n '\
  899. 'Session Type: HIVESERVER2\n HiveServer2 Protocol Version: V6\n Start Time: 2017-10-26 11:19:40.420511000\n End Time: '\
  900. '2017-10-26 11:23:11.426921000\n Query Type: QUERY\n Query State: FINISHED\n Query Status: OK\n Impala Version: '\
  901. 'impalad version 2.9.0-cdh5.12.1 RELEASE (build 6dacae08a283a36bb932335ae0c046977e2474e8)\n User: admin\n Connected User: '\
  902. 'admin\n Delegated User: \n Network Address: 10.16.2.226:63745\n Default Db: default\n Sql Statement: select * from '\
  903. 'customers\n Coordinator: nightly512-unsecure-2.gce.cloudera.com:22000\n Query Options (set by configuration): '\
  904. 'QUERY_TIMEOUT_S=600\n Query Options (set by configuration and planner): QUERY_TIMEOUT_S=600,MT_DOP=0\n Plan: '\
  905. '\n----------------\nPer-Host Resource Reservation: Memory=0B\nPer-Host Resource Estimates: Memory=32.00MB\nWARNING: The '\
  906. 'following tables have potentially corrupt table statistics.\nDrop and re-compute statistics to resolve this problem.'\
  907. '\ndefault.customers\nWARNING: The following tables are missing relevant table and/or column statistics.'\
  908. '\ndefault.customers\n\nF01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1\nPLAN-ROOT SINK\n| mem-estimate=0B '\
  909. 'mem-reservation=0B\n|\n01:EXCHANGE [UNPARTITIONED]\n| mem-estimate=0B mem-reservation=0B\n| tuple-ids=0 row-size=19B '\
  910. 'cardinality=0\n|\nF00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1\n00:SCAN HDFS [default.customers, RANDOM]\n partitions=1/1 '\
  911. 'files=1 size=15.44KB\n table stats: 0 rows total\n column stats: unavailable\n mem-estimate=32.00MB mem-reservation=0B\n '\
  912. 'tuple-ids=0 row-size=19B cardinality=0\n----------------\n Estimated Per-Host Mem: 33554432\n Per-Host Memory Reservation: '\
  913. '0\n Tables Missing Stats: default.customers\n Tables With Corrupt Table Stats: default.customers\n Request Pool: '\
  914. 'root.admin\n Admission result: Admitted immediately\n ExecSummary: \nOperator #Hosts Avg Time Max Time #Rows '\
  915. 'Est. #Rows Peak Mem Est. Peak Mem Detail \n-----------------------------------------------------------------------'\
  916. '------------------------------------\n01:EXCHANGE 1 0.000ns 0.000ns 53 0 0 0 '\
  917. 'UNPARTITIONED \n00:SCAN HDFS 1 215.018ms 215.018ms 53 0 45.02 KB 32.00 MB default.customers '\
  918. '\n Errors: \n Planner Timeline: 5s043ms\n - Metadata load started: 10.215ms (10.215ms)\n - Metadata load '\
  919. 'finished: 4s789ms (4s779ms)\n - Analysis finished: 4s856ms (66.876ms)\n - Equivalence classes computed: 4s894ms '\
  920. '(38.233ms)\n - Single node plan created: 4s945ms (50.928ms)\n - Runtime filters computed: 4s947ms (2.464ms)\n '\
  921. '- Distributed plan created: 4s953ms (5.784ms)\n - Lineage info computed: 4s955ms (2.144ms)\n - Planning finished: '\
  922. '5s043ms (88.082ms)\n Query Timeline: 3m31s\n - Query submitted: 0.000ns (0.000ns)\n - Planning finished: 5s061ms '\
  923. '(5s061ms)\n - Submit for admission: 5s062ms (1.000ms)\n - Completed admission: 5s062ms (0.000ns)\n - Ready to '\
  924. 'start on 1 backends: 5s062ms (0.000ns)\n - All 1 execution backends (2 fragment instances) started: 5s064ms (2.000ms)'\
  925. '\n - Rows available: 5s311ms (247.021ms)\n - First row fetched: 6s565ms (1s254ms)\n - Unregister query: 3m31s '\
  926. '(3m24s)\n - ComputeScanRangeAssignmentTimer: 0.000ns\n ImpalaServer:\n - ClientFetchWaitTimer: 3m25s\n - '\
  927. 'RowMaterializationTimer: 1.000ms\n Execution Profile 1a48b5796f8f07f5:49ba9e6b00000000:(Total: 250.021ms, non-child: 0.000ns, % '\
  928. 'non-child: 0.00%)\n Number of filters: 0\n Filter routing table: \n ID Src. Node Tgt. Node(s) Target type Partition '\
  929. 'filter Pending (Expected) First arrived Completed Enabled\n----------------------------------------------------------------'\
  930. '---------------------------------------------------\n\n Backend startup latencies: Count: 1, min / max: 1ms / 1ms, '\
  931. '25th %-ile: 1ms, 50th %-ile: 1ms, 75th %-ile: 1ms, 90th %-ile: 1ms, 95th %-ile: 1ms, 99.9th %-ile: 1ms\n Per Node Peak Memory '\
  932. 'Usage: nightly512-unsecure-2.gce.cloudera.com:22000(71.09 KB) \n - FiltersReceived: 0 (0)\n - FinalizationTimer: 0.000ns'\
  933. '\n Averaged Fragment F01:(Total: 1s501ms, non-child: 1s256ms, % non-child: 83.68%)\n split sizes: min: 0, max: 0, '\
  934. 'avg: 0, stddev: 0\n completion times: min:1s501ms max:1s501ms mean: 1s501ms stddev:0.000ns\n execution rates: '\
  935. 'min:0.00 /sec max:0.00 /sec mean:0.00 /sec stddev:0.00 /sec\n num instances: 1\n - AverageThreadTokens: 0.00 \n '\
  936. ' - BloomFilterBytes: 0\n - PeakMemoryUsage: 12.41 KB (12712)\n - PerHostPeakMemUsage: 71.09 KB (72800)\n - '\
  937. 'RowsProduced: 53 (53)\n - TotalNetworkReceiveTime: 219.018ms\n - TotalNetworkSendTime: 0.000ns\n - '\
  938. 'TotalStorageWaitTime: 0.000ns\n - TotalThreadsInvoluntaryContextSwitches: 0 (0)\n - TotalThreadsTotalWallClockTime: '\
  939. '1s473ms\n - TotalThreadsSysTime: 9.000us\n - TotalThreadsUserTime: 233.000us\n - '\
  940. 'TotalThreadsVoluntaryContextSwitches: 3 (3)\n Fragment Instance Lifecycle Timings:\n - ExecTime: 1s254ms\n '\
  941. ' - ExecTreeExecTime: 0.000ns\n - OpenTime: 219.018ms\n - ExecTreeOpenTime: 219.018ms\n - PrepareTime: '\
  942. '28.002ms\n - ExecTreePrepareTime: 0.000ns\n BlockMgr:\n - BlockWritesOutstanding: 0 (0)\n - '\
  943. 'BlocksCreated: 0 (0)\n - BlocksRecycled: 0 (0)\n - BufferedPins: 0 (0)\n - MaxBlockSize: 8.00 MB '\
  944. '(8388608)\n - MemoryLimit: 16.33 GB (17534060544)\n - PeakMemoryUsage: 0\n - ScratchBytesRead: 0'\
  945. '\n - ScratchBytesWritten: 0\n - ScratchFileUsedBytes: 0\n - ScratchReads: 0 (0)\n - '\
  946. 'ScratchWrites: 0 (0)\n - TotalBufferWaitTime: 0.000ns\n - TotalEncryptionTime: 0.000ns\n - '\
  947. 'TotalReadBlockTime: 0.000ns\n PLAN_ROOT_SINK:\n - PeakMemoryUsage: 0\n CodeGen:(Total: 26.002ms, non-child: '\
  948. '26.002ms, % non-child: 100.00%)\n - CodegenTime: 0.000ns\n - CompileTime: 0.000ns\n - LoadTime: 0.000ns'\
  949. '\n - ModuleBitcodeSize: 1.98 MB (2077616)\n - NumFunctions: 0 (0)\n - NumInstructions: 0 (0)\n - '\
  950. 'OptimizationTime: 0.000ns\n - PeakMemoryUsage: 0\n - PrepareTime: 25.002ms\n EXCHANGE_NODE (id=1):(Total: '\
  951. '219.018ms, non-child: 219.018ms, % non-child: 100.00%)\n - BytesReceived: 1.54 KB (1578)\n - ConvertRowBatchTime:'\
  952. ' 0.000ns\n - DeserializeRowBatchTimer: 0.000ns\n - FirstBatchArrivalWaitTime: 219.018ms\n - '\
  953. 'PeakMemoryUsage: 0\n - RowsReturned: 53 (53)\n - RowsReturnedRate: 241.00 /sec\n - SendersBlockedTimer:'\
  954. ' 0.000ns\n - SendersBlockedTotalTimer(*): 0.000ns\n Coordinator Fragment F01:\n Instance '\
  955. '1a48b5796f8f07f5:49ba9e6b00000000 (host=nightly512-unsecure-2.gce.cloudera.com:22000):(Total: 1s501ms, non-child: 1s256ms, % '\
  956. 'non-child: 83.68%)\n MemoryUsage(500.000ms): 12.00 KB, 12.00 KB, 12.00 KB\n - AverageThreadTokens: 0.00 \n '\
  957. ' - BloomFilterBytes: 0\n - PeakMemoryUsage: 12.41 KB (12712)\n - PerHostPeakMemUsage: 71.09 KB (72800)\n '\
  958. ' - RowsProduced: 53 (53)\n - TotalNetworkReceiveTime: 219.018ms\n - TotalNetworkSendTime: 0.000ns\n '\
  959. '- TotalStorageWaitTime: 0.000ns\n - TotalThreadsInvoluntaryContextSwitches: 0 (0)\n - '\
  960. 'TotalThreadsTotalWallClockTime: 1s473ms\n - TotalThreadsSysTime: 9.000us\n - TotalThreadsUserTime: 233.000us'\
  961. '\n - TotalThreadsVoluntaryContextSwitches: 3 (3)\n Fragment Instance Lifecycle Timings:\n - ExecTime: '\
  962. '1s254ms\n - ExecTreeExecTime: 0.000ns\n - OpenTime: 219.018ms\n - ExecTreeOpenTime: 219.018ms'\
  963. '\n - PrepareTime: 28.002ms\n - ExecTreePrepareTime: 0.000ns\n BlockMgr:\n - '\
  964. 'BlockWritesOutstanding: 0 (0)\n - BlocksCreated: 0 (0)\n - BlocksRecycled: 0 (0)\n - BufferedPins:'\
  965. ' 0 (0)\n - MaxBlockSize: 8.00 MB (8388608)\n - MemoryLimit: 16.33 GB (17534060544)\n - '\
  966. 'PeakMemoryUsage: 0\n - ScratchBytesRead: 0\n - ScratchBytesWritten: 0\n - ScratchFileUsedBytes: '\
  967. '0\n - ScratchReads: 0 (0)\n - ScratchWrites: 0 (0)\n - TotalBufferWaitTime: 0.000ns\n - '\
  968. 'TotalEncryptionTime: 0.000ns\n - TotalReadBlockTime: 0.000ns\n PLAN_ROOT_SINK:\n - PeakMemoryUsage: '\
  969. '0\n CodeGen:(Total: 26.002ms, non-child: 26.002ms, % non-child: 100.00%)\n - CodegenTime: 0.000ns\n - '\
  970. 'CompileTime: 0.000ns\n - LoadTime: 0.000ns\n - ModuleBitcodeSize: 1.98 MB (2077616)\n - '\
  971. 'NumFunctions: 0 (0)\n - NumInstructions: 0 (0)\n - OptimizationTime: 0.000ns\n - PeakMemoryUsage: '\
  972. '0\n - PrepareTime: 25.002ms\n EXCHANGE_NODE (id=1):(Total: 219.018ms, non-child: 0.000ns, % non-child: 0.00%)\n'\
  973. ' BytesReceived(500.000ms): 1.54 KB, 1.54 KB, 1.54 KB\n - BytesReceived: 1.54 KB (1578)\n - '\
  974. 'ConvertRowBatchTime: 0.000ns\n - DeserializeRowBatchTimer: 0.000ns\n - FirstBatchArrivalWaitTime: 219.018ms\n'\
  975. ' - PeakMemoryUsage: 0\n - RowsReturned: 53 (53)\n - RowsReturnedRate: 241.00 /sec\n - '\
  976. 'SendersBlockedTimer: 0.000ns\n - SendersBlockedTotalTimer(*): 0.000ns\n Averaged Fragment F00:(Total: 241.020ms, '\
  977. 'non-child: 0.000ns, % non-child: 0.00%)\n split sizes: min: 15.44 KB, max: 15.44 KB, avg: 15.44 KB, stddev: 0\n '\
  978. 'completion times: min:248.021ms max:248.021ms mean: 248.021ms stddev:0.000ns\n execution rates: min:62.26 KB/sec '\
  979. 'max:62.26 KB/sec mean:62.26 KB/sec stddev:0.61 B/sec\n num instances: 1\n - AverageThreadTokens: 0.00 \n - '\
  980. 'BloomFilterBytes: 0\n - PeakMemoryUsage: 63.09 KB (64608)\n - PerHostPeakMemUsage: 71.09 KB (72800)\n - '\
  981. 'RowsProduced: 53 (53)\n - TotalNetworkReceiveTime: 0.000ns\n - TotalNetworkSendTime: 0.000ns\n - '\
  982. 'TotalStorageWaitTime: 175.014ms\n - TotalThreadsInvoluntaryContextSwitches: 2 (2)\n - TotalThreadsTotalWallClockTime:'\
  983. ' 378.032ms\n - TotalThreadsSysTime: 1.998ms\n - TotalThreadsUserTime: 24.546ms\n - '\
  984. 'TotalThreadsVoluntaryContextSwitches: 13 (13)\n Fragment Instance Lifecycle Timings:\n - ExecTime: 176.015ms\n'\
  985. ' - ExecTreeExecTime: 176.015ms\n - OpenTime: 26.002ms\n - ExecTreeOpenTime: 1.000ms\n - '\
  986. 'PrepareTime: 39.003ms\n - ExecTreePrepareTime: 19.001ms\n DataStreamSender (dst_id=1):\n - BytesSent: '\
  987. '1.54 KB (1578)\n - NetworkThroughput(*): 0.00 /sec\n - OverallThroughput: 0.00 /sec\n - PeakMemoryUsage:'\
  988. ' 6.09 KB (6240)\n - RowsReturned: 53 (53)\n - SerializeBatchTime: 0.000ns\n - TransmitDataRPCTime: '\
  989. '0.000ns\n - UncompressedRowBatchSize: 2.05 KB (2098)\n CodeGen:(Total: 43.003ms, non-child: 43.003ms, % non-child: '\
  990. '100.00%)\n - CodegenTime: 1.000ms\n - CompileTime: 13.001ms\n - LoadTime: 0.000ns\n - '\
  991. 'ModuleBitcodeSize: 1.98 MB (2077616)\n - NumFunctions: 5 (5)\n - NumInstructions: 98 (98)\n - '\
  992. 'OptimizationTime: 11.000ms\n - PeakMemoryUsage: 49.00 KB (50176)\n - PrepareTime: 18.001ms\n HDFS_SCAN_NODE'\
  993. ' (id=0):(Total: 215.018ms, non-child: 215.018ms, % non-child: 100.00%)\n - AverageHdfsReadThreadConcurrency: 0.00 \n '\
  994. ' - AverageScannerThreadConcurrency: 0.00 \n - BytesRead: 16.71 KB (17111)\n - BytesReadDataNodeCache: 0\n '\
  995. ' - BytesReadLocal: 16.71 KB (17111)\n - BytesReadRemoteUnexpected: 0\n - BytesReadShortCircuit: 16.71 KB '\
  996. '(17111)\n - DecompressionTime: 0.000ns\n - MaxCompressedTextFileLength: 0\n - NumColumns: 2 (2)\n '\
  997. ' - NumDictFilteredRowGroups: 0 (0)\n - NumDisksAccessed: 1 (1)\n - NumRowGroups: 1 (1)\n - '\
  998. 'NumScannerThreadsStarted: 1 (1)\n - NumScannersWithNoReads: 0 (0)\n - NumStatsFilteredRowGroups: 0 (0)\n '\
  999. '- PeakMemoryUsage: 45.02 KB (46101)\n - PerReadThreadRawHdfsThroughput: 0.00 /sec\n - RemoteScanRanges: 0 (0)\n '\
  1000. ' - RowBatchQueueGetWaitTime: 176.015ms\n - RowBatchQueuePutWaitTime: 0.000ns\n - RowsRead: 53 (53)\n '\
  1001. ' - RowsReturned: 53 (53)\n - RowsReturnedRate: 246.00 /sec\n - ScanRangesComplete: 1 (1)\n - '\
  1002. 'ScannerThreadsInvoluntaryContextSwitches: 0 (0)\n - ScannerThreadsTotalWallClockTime: 176.015ms\n - '\
  1003. 'MaterializeTupleTime(*): 0.000ns\n - ScannerThreadsSysTime: 0.000ns\n - ScannerThreadsUserTime: '\
  1004. '819.000us\n - ScannerThreadsVoluntaryContextSwitches: 9 (9)\n - TotalRawHdfsReadTime(*): 0.000ns\n - '\
  1005. 'TotalReadThroughput: 0.00 /sec\n Fragment F00:\n Instance 1a48b5796f8f07f5:49ba9e6b00000001 '\
  1006. '(host=nightly512-unsecure-2.gce.cloudera.com:22000):(Total: 241.020ms, non-child: 0.000ns, % non-child: 0.00%)\n Hdfs '\
  1007. 'split stats (<volume id>:<# splits>/<split lengths>): 0:1/15.44 KB \n - AverageThreadTokens: 0.00 \n - '\
  1008. 'BloomFilterBytes: 0\n - PeakMemoryUsage: 63.09 KB (64608)\n - PerHostPeakMemUsage: 71.09 KB (72800)\n '\
  1009. '- RowsProduced: 53 (53)\n - TotalNetworkReceiveTime: 0.000ns\n - TotalNetworkSendTime: 0.000ns\n - '\
  1010. 'TotalStorageWaitTime: 175.014ms\n - TotalThreadsInvoluntaryContextSwitches: 2 (2)\n - '\
  1011. 'TotalThreadsTotalWallClockTime: 378.032ms\n - TotalThreadsSysTime: 1.998ms\n - TotalThreadsUserTime: '\
  1012. '24.546ms\n - TotalThreadsVoluntaryContextSwitches: 13 (13)\n Fragment Instance Lifecycle Timings:\n - '\
  1013. 'ExecTime: 176.015ms\n - ExecTreeExecTime: 176.015ms\n - OpenTime: 26.002ms\n - '\
  1014. 'ExecTreeOpenTime: 1.000ms\n - PrepareTime: 39.003ms\n - ExecTreePrepareTime: 19.001ms\n '\
  1015. 'DataStreamSender (dst_id=1):\n - BytesSent: 1.54 KB (1578)\n - NetworkThroughput(*): 0.00 /sec\n '\
  1016. '- OverallThroughput: 0.00 /sec\n - PeakMemoryUsage: 6.09 KB (6240)\n - RowsReturned: 53 (53)\n '\
  1017. '- SerializeBatchTime: 0.000ns\n - TransmitDataRPCTime: 0.000ns\n - UncompressedRowBatchSize: 2.05 KB (2098)'\
  1018. '\n CodeGen:(Total: 43.003ms, non-child: 43.003ms, % non-child: 100.00%)\n - CodegenTime: 1.000ms\n '\
  1019. '- CompileTime: 13.001ms\n - LoadTime: 0.000ns\n - ModuleBitcodeSize: 1.98 MB (2077616)\n - '\
  1020. 'NumFunctions: 5 (5)\n - NumInstructions: 98 (98)\n - OptimizationTime: 11.000ms\n - '\
  1021. 'PeakMemoryUsage: 49.00 KB (50176)\n - PrepareTime: 18.001ms\n HDFS_SCAN_NODE (id=0):(Total: 215.018ms, '\
  1022. 'non-child: 215.018ms, % non-child: 100.00%)\n Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:1/15.44 KB \n'\
  1023. ' ExecOption: PARQUET Codegen Enabled, Codegen enabled: 1 out of 1\n Hdfs Read Thread Concurrency Bucket: 0:0% '\
  1024. '1:0% 2:0% 3:0% 4:0% \n File Formats: PARQUET/NONE:2 \n - FooterProcessingTime: (Avg: 168.014ms ; Min: '\
  1025. '168.014ms ; Max: 168.014ms ; Number of samples: 1)\n - AverageHdfsReadThreadConcurrency: 0.00 \n - '\
  1026. 'AverageScannerThreadConcurrency: 0.00 \n - BytesRead: 16.71 KB (17111)\n - BytesReadDataNodeCache: 0\n'\
  1027. ' - BytesReadLocal: 16.71 KB (17111)\n - BytesReadRemoteUnexpected: 0\n - BytesReadShortCircuit: '\
  1028. '16.71 KB (17111)\n - DecompressionTime: 0.000ns\n - MaxCompressedTextFileLength: 0\n - NumColumns:'\
  1029. ' 2 (2)\n - NumDictFilteredRowGroups: 0 (0)\n - NumDisksAccessed: 1 (1)\n - NumRowGroups: 1 (1)\n '\
  1030. ' - NumScannerThreadsStarted: 1 (1)\n - NumScannersWithNoReads: 0 (0)\n - NumStatsFilteredRowGroups: '\
  1031. '0 (0)\n - PeakMemoryUsage: 45.02 KB (46101)\n - PerReadThreadRawHdfsThroughput: 0.00 /sec\n - '\
  1032. 'RemoteScanRanges: 0 (0)\n - RowBatchQueueGetWaitTime: 176.015ms\n - RowBatchQueuePutWaitTime: 0.000ns\n'\
  1033. ' - RowsRead: 53 (53)\n - RowsReturned: 53 (53)\n - RowsReturnedRate: 246.00 /sec\n - '\
  1034. 'ScanRangesComplete: 1 (1)\n - ScannerThreadsInvoluntaryContextSwitches: 0 (0)\n - '\
  1035. 'ScannerThreadsTotalWallClockTime: 176.015ms\n - MaterializeTupleTime(*): 0.000ns\n - '\
  1036. 'ScannerThreadsSysTime: 0.000ns\n - ScannerThreadsUserTime: 819.000us\n - '\
  1037. 'ScannerThreadsVoluntaryContextSwitches: 9 (9)\n - TotalRawHdfsReadTime(*): 0.000ns\n - '\
  1038. 'TotalReadThroughput: 0.00 /sec\n',
  1039. u'query_id': u'1a48b5796f8f07f5:49ba9e6b00000000',
  1040. u'__common__': {
  1041. u'navbar': [
  1042. {u'link': u'/backends', u'title': u'/backends'},
  1043. {u'link': u'/catalog', u'title': u'/catalog'},
  1044. {u'link': u'/hadoop-varz', u'title': u'/hadoop-varz'},
  1045. {u'link': u'/log_level', u'title': u'/log_level'},
  1046. {u'link': u'/logs', u'title': u'/logs'},
  1047. {u'link': u'/memz', u'title': u'/memz'},
  1048. {u'link': u'/metrics', u'title': u'/metrics'},
  1049. {u'link': u'/queries', u'title': u'/queries'},
  1050. {u'link': u'/rpcz', u'title': u'/rpcz'},
  1051. {u'link': u'/sessions', u'title': u'/sessions'},
  1052. {u'link': u'/threadz', u'title': u'/threadz'},
  1053. {u'link': u'/varz', u'title': u'/varz'}
  1054. ],
  1055. u'process-name': u'impalad'
  1056. }
  1057. },
  1058. '8a46a8865624698f:b80b211500000000': {
  1059. u'profile': u'Query (id=d424420e0c44ab9:c637ac2900000000):\n Summary:\n Session ID: 3348564c97187569:1c17ce45bdfbf0b2\n '\
  1060. 'Session Type: HIVESERVER2\n HiveServer2 Protocol Version: V6\n Start Time: 2017-10-26 11:20:11.971764000\n End Time: '\
  1061. '2017-10-26 11:23:11.429110000\n Query Type: QUERY\n Query State: FINISHED\n Query Status: OK\n Impala Version: '\
  1062. 'impalad version 2.9.0-cdh5.12.1 RELEASE (build 6dacae08a283a36bb932335ae0c046977e2474e8)\n User: admin\n Connected User: '\
  1063. 'admin\n Delegated User: \n Network Address: 10.16.2.226:63745\n Default Db: default\n Sql Statement: SELECT '\
  1064. 'sample_07.description, sample_07.salary\r\nFROM\r\n sample_07\r\nWHERE\r\n( sample_07.salary > 100000)\r\nORDER BY '\
  1065. 'sample_07.salary DESC\r\nLIMIT 1000\n Coordinator: nightly512-unsecure-2.gce.cloudera.com:22000\n Query Options '\
  1066. '(set by configuration): QUERY_TIMEOUT_S=600\n Query Options (set by configuration and planner): QUERY_TIMEOUT_S=600,MT_DOP=0'\
  1067. '\n Plan: \n----------------\nPer-Host Resource Reservation: Memory=0B\nPer-Host Resource Estimates: Memory=32.00MB\nWARNING: '\
  1068. 'The following tables have potentially corrupt table statistics.\nDrop and re-compute statistics to resolve this problem.'\
  1069. '\ndefault.sample_07\nWARNING: The following tables are missing relevant table and/or column statistics.\ndefault.sample_07\n\n'\
  1070. 'F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1\nPLAN-ROOT SINK\n| mem-estimate=0B mem-reservation=0B\n|\n02:MERGING-'\
  1071. 'EXCHANGE [UNPARTITIONED]\n| order by: salary DESC\n| limit: 1000\n| mem-estimate=0B mem-reservation=0B\n| tuple-ids=1 '\
  1072. 'row-size=19B cardinality=0\n|\nF00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1\n01:TOP-N [LIMIT=1000]\n| order by: salary '\
  1073. 'DESC\n| mem-estimate=0B mem-reservation=0B\n| tuple-ids=1 row-size=19B cardinality=0\n|\n00:SCAN HDFS [default.sample_07, '\
  1074. 'RANDOM]\n partitions=1/1 files=1 size=44.98KB\n predicates: (sample_07.salary > 100000)\n table stats: 0 rows total\n '\
  1075. 'column stats: unavailable\n parquet dictionary predicates: (sample_07.salary > 100000)\n mem-estimate=32.00MB '\
  1076. 'mem-reservation=0B\n tuple-ids=0 row-size=19B cardinality=0\n----------------\n Estimated Per-Host Mem: 33554432\n '\
  1077. 'Per-Host Memory Reservation: 0\n Tables Missing Stats: default.sample_07\n Tables With Corrupt Table Stats: '\
  1078. 'default.sample_07\n Request Pool: root.admin\n Admission result: Admitted immediately\n ExecSummary: \nOperator'\
  1079. ' #Hosts Avg Time Max Time #Rows Est. #Rows Peak Mem Est. Peak Mem Detail \n-------------------'\
  1080. '------------------------------------------------------------------------------------------------\n02:MERGING-EXCHANGE 1 '\
  1081. ' 0.000ns 0.000ns 28 0 0 0 UNPARTITIONED \n01:TOP-N 1 0.000ns '\
  1082. '0.000ns 28 0 80.00 KB 0 \n00:SCAN HDFS 1 250.020ms 250.020ms '\
  1083. '28 0 173.00 KB 32.00 MB default.sample_07 \n Errors: \n Planner Timeline: 3s275ms\n - Metadata '\
  1084. 'load started: 11.586ms (11.586ms)\n - Metadata load finished: 3s248ms (3s236ms)\n - Analysis finished: 3s254ms '\
  1085. '(6.431ms)\n - Equivalence classes computed: 3s255ms (335.173us)\n - Single node plan created: 3s267ms (12.443ms)\n'\
  1086. ' - Runtime filters computed: 3s267ms (92.906us)\n - Distributed plan created: 3s267ms (223.487us)\n - Lineage '\
  1087. 'info computed: 3s268ms (348.540us)\n - Planning finished: 3s275ms (7.378ms)\n Query Timeline: 2m59s\n - Query '\
  1088. 'submitted: 0.000ns (0.000ns)\n - Planning finished: 3s278ms (3s278ms)\n - Submit for admission: 3s279ms (1.000ms)'\
  1089. '\n - Completed admission: 3s279ms (0.000ns)\n - Ready to start on 2 backends: 3s279ms (0.000ns)\n - All 2 '\
  1090. 'execution backends (2 fragment instances) started: 3s331ms (52.004ms)\n - Rows available: 3s781ms (450.038ms)\n - '\
  1091. 'First row fetched: 5s232ms (1s451ms)\n - Unregister query: 2m59s (2m54s)\n - ComputeScanRangeAssignmentTimer: '\
  1092. '0.000ns\n ImpalaServer:\n - ClientFetchWaitTimer: 2m55s\n - RowMaterializationTimer: 0.000ns\n Execution Profile '\
  1093. 'd424420e0c44ab9:c637ac2900000000:(Total: 502.042ms, non-child: 0.000ns, % non-child: 0.00%)\n Number of filters: 0\n '\
  1094. 'Filter routing table: \n ID Src. Node Tgt. Node(s) Target type Partition filter Pending (Expected) First arrived Completed'\
  1095. ' Enabled\n-------------------------------------------------------------------------------------------------------------------\n'\
  1096. '\n Backend startup latencies: Count: 2, min / max: 1ms / 52ms, 25th %-ile: 1ms, 50th %-ile: 1ms, 75th %-ile: 52ms, 90th %-ile:'\
  1097. ' 52ms, 95th %-ile: 52ms, 99.9th %-ile: 52ms\n Per Node Peak Memory Usage: nightly512-unsecure-2.gce.cloudera.com:22000'\
  1098. '(255.00 KB) nightly512-unsecure-3.gce.cloudera.com:22000(937.09 KB) \n - FiltersReceived: 0 (0)\n - FinalizationTimer: '\
  1099. '0.000ns\n Averaged Fragment F01:(Total: 1s952ms, non-child: 1s452ms, % non-child: 74.39%)\n split sizes: min: 0, max: '\
  1100. '0, avg: 0, stddev: 0\n completion times: min:1s952ms max:1s952ms mean: 1s952ms stddev:0.000ns\n execution rates: '\
  1101. 'min:0.00 /sec max:0.00 /sec mean:0.00 /sec stddev:0.00 /sec\n num instances: 1\n - AverageThreadTokens: 0.00 \n'\
  1102. ' - BloomFilterBytes: 0\n - PeakMemoryUsage: 255.00 KB (261120)\n - PerHostPeakMemUsage: 255.00 KB (261120)\n '\
  1103. ' - RowsProduced: 28 (28)\n - TotalNetworkReceiveTime: 0.000ns\n - TotalNetworkSendTime: 0.000ns\n - '\
  1104. 'TotalStorageWaitTime: 0.000ns\n - TotalThreadsInvoluntaryContextSwitches: 1 (1)\n - TotalThreadsTotalWallClockTime: '\
  1105. '1s934ms\n - TotalThreadsSysTime: 980.000us\n - TotalThreadsUserTime: 28.421ms\n - '\
  1106. 'TotalThreadsVoluntaryContextSwitches: 3 (3)\n Fragment Instance Lifecycle Timings:\n - ExecTime: 1s451ms\n '\
  1107. ' - ExecTreeExecTime: 0.000ns\n - OpenTime: 483.041ms\n - ExecTreeOpenTime: 452.038ms\n - PrepareTime:'\
  1108. ' 18.001ms\n - ExecTreePrepareTime: 0.000ns\n BlockMgr:\n - BlockWritesOutstanding: 0 (0)\n - '\
  1109. 'BlocksCreated: 0 (0)\n - BlocksRecycled: 0 (0)\n - BufferedPins: 0 (0)\n - MaxBlockSize: 8.00 MB '\
  1110. '(8388608)\n - MemoryLimit: 16.33 GB (17534060544)\n - PeakMemoryUsage: 0\n - ScratchBytesRead: 0\n'\
  1111. ' - ScratchBytesWritten: 0\n - ScratchFileUsedBytes: 0\n - ScratchReads: 0 (0)\n - ScratchWrites: '\
  1112. '0 (0)\n - TotalBufferWaitTime: 0.000ns\n - TotalEncryptionTime: 0.000ns\n - TotalReadBlockTime: 0.000ns\n'\
  1113. ' PLAN_ROOT_SINK:\n - PeakMemoryUsage: 0\n CodeGen:(Total: 48.004ms, non-child: 48.004ms, % non-child: 100.00%)'\
  1114. '\n - CodegenTime: 0.000ns\n - CompileTime: 3.000ms\n - LoadTime: 0.000ns\n - ModuleBitcodeSize: '\
  1115. '1.98 MB (2077616)\n - NumFunctions: 27 (27)\n - NumInstructions: 494 (494)\n - OptimizationTime: 26.002ms'\
  1116. '\n - PeakMemoryUsage: 247.00 KB (252928)\n - PrepareTime: 18.001ms\n EXCHANGE_NODE (id=2):(Total: 452.038ms,'\
  1117. ' non-child: 452.038ms, % non-child: 100.00%)\n - BytesReceived: 923.00 B (923)\n - ConvertRowBatchTime: 0.000ns\n'\
  1118. ' - DeserializeRowBatchTimer: 0.000ns\n - FirstBatchArrivalWaitTime: 452.038ms\n - MergeGetNext: 0.000ns\n'\
  1119. ' - MergeGetNextBatch: 0.000ns\n - PeakMemoryUsage: 0\n - RowsReturned: 28 (28)\n - '\
  1120. 'RowsReturnedRate: 61.00 /sec\n - SendersBlockedTimer: 0.000ns\n - SendersBlockedTotalTimer(*): 0.000ns\n'\
  1121. ' Coordinator Fragment F01:\n Instance d424420e0c44ab9:c637ac2900000000 (host=nightly512-unsecure-2.gce.cloudera.com:'\
  1122. '22000):(Total: 1s952ms, non-child: 1s452ms, % non-child: 74.39%)\n MemoryUsage(500.000ms): 8.09 KB, 12.09 KB, 12.09 KB, '\
  1123. '12.09 KB\n - AverageThreadTokens: 0.00 \n - BloomFilterBytes: 0\n - PeakMemoryUsage: 255.00 KB (261120)\n'\
  1124. ' - PerHostPeakMemUsage: 255.00 KB (261120)\n - RowsProduced: 28 (28)\n - TotalNetworkReceiveTime: 0.000ns'\
  1125. '\n - TotalNetworkSendTime: 0.000ns\n - TotalStorageWaitTime: 0.000ns\n - '\
  1126. 'TotalThreadsInvoluntaryContextSwitches: 1 (1)\n - TotalThreadsTotalWallClockTime: 1s934ms\n - '\
  1127. 'TotalThreadsSysTime: 980.000us\n - TotalThreadsUserTime: 28.421ms\n - TotalThreadsVoluntaryContextSwitches: 3 '\
  1128. '(3)\n Fragment Instance Lifecycle Timings:\n - ExecTime: 1s451ms\n - ExecTreeExecTime: 0.000ns\n'\
  1129. ' - OpenTime: 483.041ms\n - ExecTreeOpenTime: 452.038ms\n - PrepareTime: 18.001ms\n - '\
  1130. 'ExecTreePrepareTime: 0.000ns\n BlockMgr:\n - BlockWritesOutstanding: 0 (0)\n - BlocksCreated: 0 (0)\n'\
  1131. ' - BlocksRecycled: 0 (0)\n - BufferedPins: 0 (0)\n - MaxBlockSize: 8.00 MB (8388608)\n - '\
  1132. 'MemoryLimit: 16.33 GB (17534060544)\n - PeakMemoryUsage: 0\n - ScratchBytesRead: 0\n - '\
  1133. 'ScratchBytesWritten: 0\n - ScratchFileUsedBytes: 0\n - ScratchReads: 0 (0)\n - ScratchWrites: 0 '\
  1134. '(0)\n - TotalBufferWaitTime: 0.000ns\n - TotalEncryptionTime: 0.000ns\n - TotalReadBlockTime: '\
  1135. '0.000ns\n PLAN_ROOT_SINK:\n - PeakMemoryUsage: 0\n CodeGen:(Total: 48.004ms, non-child: 48.004ms, % '\
  1136. 'non-child: 100.00%)\n - CodegenTime: 0.000ns\n - CompileTime: 3.000ms\n - LoadTime: 0.000ns\n '\
  1137. ' - ModuleBitcodeSize: 1.98 MB (2077616)\n - NumFunctions: 27 (27)\n - NumInstructions: 494 (494)\n'\
  1138. ' - OptimizationTime: 26.002ms\n - PeakMemoryUsage: 247.00 KB (252928)\n - PrepareTime: 18.001ms\n'\
  1139. ' EXCHANGE_NODE (id=2):(Total: 452.038ms, non-child: 0.000ns, % non-child: 0.00%)\n ExecOption: Codegen Enabled\n'\
  1140. ' BytesReceived(500.000ms): 0, 923.00 B, 923.00 B, 923.00 B\n - BytesReceived: 923.00 B (923)\n - '\
  1141. 'ConvertRowBatchTime: 0.000ns\n - DeserializeRowBatchTimer: 0.000ns\n - FirstBatchArrivalWaitTime: 452.038ms\n'\
  1142. ' - MergeGetNext: 0.000ns\n - MergeGetNextBatch: 0.000ns\n - PeakMemoryUsage: 0\n - '\
  1143. 'RowsReturned: 28 (28)\n - RowsReturnedRate: 61.00 /sec\n - SendersBlockedTimer: 0.000ns\n - '\
  1144. 'SendersBlockedTotalTimer(*): 0.000ns\n Averaged Fragment F00:(Total: 450.037ms, non-child: 55.004ms, % non-child: 12.22%)\n'\
  1145. ' split sizes: min: 44.98 KB, max: 44.98 KB, avg: 44.98 KB, stddev: 0\n completion times: min:450.038ms max:450.038ms'\
  1146. ' mean: 450.038ms stddev:0.000ns\n execution rates: min:99.94 KB/sec max:99.94 KB/sec mean:99.94 KB/sec stddev:0.68 '\
  1147. 'B/sec\n num instances: 1\n - AverageThreadTokens: 2.00 \n - BloomFilterBytes: 0\n - PeakMemoryUsage: '\
  1148. '937.09 KB (959584)\n - PerHostPeakMemUsage: 937.09 KB (959584)\n - RowsProduced: 28 (28)\n - '\
  1149. 'TotalNetworkReceiveTime: 0.000ns\n - TotalNetworkSendTime: 50.004ms\n - TotalStorageWaitTime: 180.014ms\n - '\
  1150. 'TotalThreadsInvoluntaryContextSwitches: 1 (1)\n - TotalThreadsTotalWallClockTime: 570.046ms\n - '\
  1151. 'TotalThreadsSysTime: 3.300ms\n - TotalThreadsUserTime: 157.428ms\n - TotalThreadsVoluntaryContextSwitches: 9 '\
  1152. '(9)\n Fragment Instance Lifecycle Timings:\n - ExecTime: 51.004ms\n - ExecTreeExecTime: 0.000ns\n '\
  1153. ' - OpenTime: 339.027ms\n - ExecTreeOpenTime: 180.014ms\n - PrepareTime: 60.004ms\n - '\
  1154. 'ExecTreePrepareTime: 35.002ms\n BlockMgr:\n - BlockWritesOutstanding: 0 (0)\n - BlocksCreated: 0 '\
  1155. '(0)\n - BlocksRecycled: 0 (0)\n - BufferedPins: 0 (0)\n - MaxBlockSize: 8.00 MB (8388608)\n - '\
  1156. 'MemoryLimit: 16.33 GB (17534060544)\n - PeakMemoryUsage: 0\n - ScratchBytesRead: 0\n - '\
  1157. 'ScratchBytesWritten: 0\n - ScratchFileUsedBytes: 0\n - ScratchReads: 0 (0)\n - ScratchWrites: 0 (0)\n'\
  1158. ' - TotalBufferWaitTime: 0.000ns\n - TotalEncryptionTime: 0.000ns\n - TotalReadBlockTime: 0.000ns\n'\
  1159. ' DataStreamSender (dst_id=2):\n - BytesSent: 923.00 B (923)\n - NetworkThroughput(*): 0.00 /sec\n'\
  1160. ' - OverallThroughput: 0.00 /sec\n - PeakMemoryUsage: 6.09 KB (6240)\n - RowsReturned: 28 (28)\n '\
  1161. ' - SerializeBatchTime: 0.000ns\n - TransmitDataRPCTime: 0.000ns\n - UncompressedRowBatchSize: 1.30 KB '\
  1162. '(1333)\n CodeGen:(Total: 180.014ms, non-child: 180.014ms, % non-child: 100.00%)\n - CodegenTime: 3.000ms\n '\
  1163. '- CompileTime: 42.003ms\n - LoadTime: 0.000ns\n - ModuleBitcodeSize: 1.98 MB (2077616)\n - NumFunctions: '\
  1164. '94 (94)\n - NumInstructions: 1.85K (1846)\n - OptimizationTime: 116.009ms\n - PeakMemoryUsage: 923.00 KB '\
  1165. '(945152)\n - PrepareTime: 21.001ms\n SORT_NODE (id=1):(Total: 215.017ms, non-child: 0.000ns, % non-child: 0.00%)\n '\
  1166. ' - InsertBatchTime: 0.000ns\n - PeakMemoryUsage: 80.00 KB (81920)\n - RowsReturned: 28 (28)\n - '\
  1167. 'RowsReturnedRate: 130.00 /sec\n HDFS_SCAN_NODE (id=0):(Total: 250.020ms, non-child: 250.020ms, % non-child: 100.00%)\n '\
  1168. ' - AverageHdfsReadThreadConcurrency: 0.00 \n - AverageScannerThreadConcurrency: 1.00 \n - BytesRead: 44.98 '\
  1169. 'KB (46055)\n - BytesReadDataNodeCache: 0\n - BytesReadLocal: 44.98 KB (46055)\n - '\
  1170. 'BytesReadRemoteUnexpected: 0\n - BytesReadShortCircuit: 44.98 KB (46055)\n - DecompressionTime: 0.000ns\n '\
  1171. ' - MaxCompressedTextFileLength: 0\n - NumDisksAccessed: 1 (1)\n - NumScannerThreadsStarted: 1 (1)\n - '\
  1172. 'PeakMemoryUsage: 173.00 KB (177152)\n - PerReadThreadRawHdfsThroughput: 0.00 /sec\n - RemoteScanRanges: 0 (0)\n'\
  1173. ' - RowBatchQueueGetWaitTime: 180.014ms\n - RowBatchQueuePutWaitTime: 0.000ns\n - RowsRead: 823 (823)\n '\
  1174. ' - RowsReturned: 28 (28)\n - RowsReturnedRate: 111.00 /sec\n - ScanRangesComplete: 1 (1)\n - '\
  1175. 'ScannerThreadsInvoluntaryContextSwitches: 0 (0)\n - ScannerThreadsTotalWallClockTime: 180.014ms\n - '\
  1176. 'DelimiterParseTime: 0.000ns\n - MaterializeTupleTime(*): 0.000ns\n - ScannerThreadsSysTime: 324.000us\n'\
  1177. ' - ScannerThreadsUserTime: 0.000ns\n - ScannerThreadsVoluntaryContextSwitches: 4 (4)\n - '\
  1178. 'TotalRawHdfsReadTime(*): 0.000ns\n - TotalReadThroughput: 0.00 /sec\n Fragment F00:\n Instance '\
  1179. 'd424420e0c44ab9:c637ac2900000001 (host=nightly512-unsecure-3.gce.cloudera.com:22000):(Total: 450.037ms, non-child: 55.004ms, '\
  1180. '% non-child: 12.22%)\n Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:1/44.98 KB \n '\
  1181. 'MemoryUsage(500.000ms): 130.54 KB\n ThreadUsage(500.000ms): 2\n - AverageThreadTokens: 2.00 \n - '\
  1182. 'BloomFilterBytes: 0\n - PeakMemoryUsage: 937.09 KB (959584)\n - PerHostPeakMemUsage: 937.09 KB (959584)\n '\
  1183. ' - RowsProduced: 28 (28)\n - TotalNetworkReceiveTime: 0.000ns\n - TotalNetworkSendTime: 50.004ms\n '\
  1184. ' - TotalStorageWaitTime: 180.014ms\n - TotalThreadsInvoluntaryContextSwitches: 1 (1)\n - '\
  1185. 'TotalThreadsTotalWallClockTime: 570.046ms\n - TotalThreadsSysTime: 3.300ms\n - TotalThreadsUserTime: '\
  1186. '157.428ms\n - TotalThreadsVoluntaryContextSwitches: 9 (9)\n Fragment Instance Lifecycle Timings:\n '\
  1187. '- ExecTime: 51.004ms\n - ExecTreeExecTime: 0.000ns\n - OpenTime: 339.027ms\n - '\
  1188. 'ExecTreeOpenTime: 180.014ms\n - PrepareTime: 60.004ms\n - ExecTreePrepareTime: 35.002ms\n '\
  1189. 'BlockMgr:\n - BlockWritesOutstanding: 0 (0)\n - BlocksCreated: 0 (0)\n - BlocksRecycled: 0 '\
  1190. '(0)\n - BufferedPins: 0 (0)\n - MaxBlockSize: 8.00 MB (8388608)\n - MemoryLimit: 16.33 GB '\
  1191. '(17534060544)\n - PeakMemoryUsage: 0\n - ScratchBytesRead: 0\n - ScratchBytesWritten: 0\n '\
  1192. ' - ScratchFileUsedBytes: 0\n - ScratchReads: 0 (0)\n - ScratchWrites: 0 (0)\n - '\
  1193. 'TotalBufferWaitTime: 0.000ns\n - TotalEncryptionTime: 0.000ns\n - TotalReadBlockTime: 0.000ns\n '\
  1194. 'DataStreamSender (dst_id=2):\n - BytesSent: 923.00 B (923)\n - NetworkThroughput(*): 0.00 /sec\n '\
  1195. '- OverallThroughput: 0.00 /sec\n - PeakMemoryUsage: 6.09 KB (6240)\n - RowsReturned: 28 (28)\n - '\
  1196. 'SerializeBatchTime: 0.000ns\n - TransmitDataRPCTime: 0.000ns\n - UncompressedRowBatchSize: 1.30 KB (1333)\n'\
  1197. ' CodeGen:(Total: 180.014ms, non-child: 180.014ms, % non-child: 100.00%)\n - CodegenTime: 3.000ms\n - '\
  1198. 'CompileTime: 42.003ms\n - LoadTime: 0.000ns\n - ModuleBitcodeSize: 1.98 MB (2077616)\n - '\
  1199. 'NumFunctions: 94 (94)\n - NumInstructions: 1.85K (1846)\n - OptimizationTime: 116.009ms\n - '\
  1200. 'PeakMemoryUsage: 923.00 KB (945152)\n - PrepareTime: 21.001ms\n SORT_NODE (id=1):(Total: 215.017ms, non-child: '\
  1201. '0.000ns, % non-child: 0.00%)\n ExecOption: Codegen Enabled\n - InsertBatchTime: 0.000ns\n - '\
  1202. 'PeakMemoryUsage: 80.00 KB (81920)\n - RowsReturned: 28 (28)\n - RowsReturnedRate: 130.00 /sec\n '\
  1203. 'HDFS_SCAN_NODE (id=0):(Total: 250.020ms, non-child: 250.020ms, % non-child: 100.00%)\n Hdfs split stats '\
  1204. '(<volume id>:<# splits>/<split lengths>): 0:1/44.98 KB \n ExecOption: TEXT Codegen Enabled, Codegen enabled: 1 out '\
  1205. 'of 1\n Hdfs Read Thread Concurrency Bucket: 0:100% 1:0% 2:0% 3:0% 4:0% \n File Formats: TEXT/NONE:1 \n '\
  1206. ' BytesRead(500.000ms): 0\n - AverageHdfsReadThreadConcurrency: 0.00 \n - '\
  1207. 'AverageScannerThreadConcurrency: 1.00 \n - BytesRead: 44.98 KB (46055)\n - BytesReadDataNodeCache: 0\n'\
  1208. ' - BytesReadLocal: 44.98 KB (46055)\n - BytesReadRemoteUnexpected: 0\n - BytesReadShortCircuit: '\
  1209. '44.98 KB (46055)\n - DecompressionTime: 0.000ns\n - MaxCompressedTextFileLength: 0\n - '\
  1210. 'NumDisksAccessed: 1 (1)\n - NumScannerThreadsStarted: 1 (1)\n - PeakMemoryUsage: 173.00 KB (177152)\n '\
  1211. ' - PerReadThreadRawHdfsThroughput: 0.00 /sec\n - RemoteScanRanges: 0 (0)\n - '\
  1212. 'RowBatchQueueGetWaitTime: 180.014ms\n - RowBatchQueuePutWaitTime: 0.000ns\n - RowsRead: 823 (823)\n'\
  1213. ' - RowsReturned: 28 (28)\n - RowsReturnedRate: 111.00 /sec\n - ScanRangesComplete: 1 (1)\n '\
  1214. ' - ScannerThreadsInvoluntaryContextSwitches: 0 (0)\n - ScannerThreadsTotalWallClockTime: 180.014ms\n '\
  1215. ' - DelimiterParseTime: 0.000ns\n - MaterializeTupleTime(*): 0.000ns\n - ScannerThreadsSysTime: '\
  1216. '324.000us\n - ScannerThreadsUserTime: 0.000ns\n - ScannerThreadsVoluntaryContextSwitches: 4 (4)\n '\
  1217. ' - TotalRawHdfsReadTime(*): 0.000ns\n - TotalReadThroughput: 0.00 /sec\n',
  1218. u'query_id': u'd424420e0c44ab9:c637ac2900000000',
  1219. u'__common__': {
  1220. u'navbar': [
  1221. {u'link': u'/backends', u'title': u'/backends'},
  1222. {u'link': u'/catalog', u'title': u'/catalog'},
  1223. {u'link': u'/hadoop-varz', u'title': u'/hadoop-varz'},
  1224. {u'link': u'/log_level', u'title': u'/log_level'},
  1225. {u'link': u'/logs', u'title': u'/logs'},
  1226. {u'link': u'/memz', u'title': u'/memz'},
  1227. {u'link': u'/metrics', u'title': u'/metrics'},
  1228. {u'link': u'/queries', u'title': u'/queries'},
  1229. {u'link': u'/rpcz', u'title': u'/rpcz'},
  1230. {u'link': u'/sessions', u'title': u'/sessions'},
  1231. {u'link': u'/threadz', u'title': u'/threadz'},
  1232. {u'link': u'/varz', u'title': u'/varz'}
  1233. ],
  1234. u'process-name': u'impalad'
  1235. }
  1236. }
  1237. }
  1238. MEMORY = {
  1239. '4d497267f34ff17d:817bdfb500000000': {
  1240. u'query_id': u'1a48b5796f8f07f5:49ba9e6b00000000',
  1241. u'__common__': {
  1242. u'navbar': [
  1243. {u'link': u'/backends', u'title': u'/backends'},
  1244. {u'link': u'/catalog', u'title': u'/catalog'},
  1245. {u'link': u'/hadoop-varz', u'title': u'/hadoop-varz'},
  1246. {u'link': u'/log_level', u'title': u'/log_level'},
  1247. {u'link': u'/logs', u'title': u'/logs'},
  1248. {u'link': u'/memz', u'title': u'/memz'},
  1249. {u'link': u'/metrics', u'title': u'/metrics'},
  1250. {u'link': u'/queries', u'title': u'/queries'},
  1251. {u'link': u'/rpcz', u'title': u'/rpcz'},
  1252. {u'link': u'/sessions', u'title': u'/sessions'},
  1253. {u'link': u'/threadz', u'title': u'/threadz'},
  1254. {u'link': u'/varz', u'title': u'/varz'}
  1255. ],
  1256. u'process-name': u'impalad'
  1257. },
  1258. u'mem_usage': u'The query is finished, current memory consumption is not available.'
  1259. },
  1260. '8a46a8865624698f:b80b211500000000': {
  1261. u'query_id': u'd424420e0c44ab9:c637ac2900000000',
  1262. u'__common__': {
  1263. u'navbar': [
  1264. {u'link': u'/backends', u'title': u'/backends'},
  1265. {u'link': u'/catalog', u'title': u'/catalog'},
  1266. {u'link': u'/hadoop-varz', u'title': u'/hadoop-varz'},
  1267. {u'link': u'/log_level', u'title': u'/log_level'},
  1268. {u'link': u'/logs', u'title': u'/logs'},
  1269. {u'link': u'/memz', u'title': u'/memz'},
  1270. {u'link': u'/metrics', u'title': u'/metrics'},
  1271. {u'link': u'/queries', u'title': u'/queries'},
  1272. {u'link': u'/rpcz', u'title': u'/rpcz'},
  1273. {u'link': u'/sessions', u'title': u'/sessions'},
  1274. {u'link': u'/threadz', u'title': u'/threadz'},
  1275. {u'link': u'/varz', u'title': u'/varz'}
  1276. ],
  1277. u'process-name': u'impalad'
  1278. },
  1279. u'mem_usage': u'The query is finished, current memory consumption is not available.'
  1280. }
  1281. }
  1282. def __init__(self, url):
  1283. self.url = url
  1284. def get_queries(self, **kwargs):
  1285. return {
  1286. 'completed_queries': [
  1287. MockImpalaQueryApi.APPS['4d497267f34ff17d:817bdfb500000000'],
  1288. MockImpalaQueryApi.APPS['8a46a8865624698f:b80b211500000000']
  1289. ],
  1290. 'in_flight_queries': [],
  1291. 'num_in_flight_queries': 0,
  1292. 'num_executing_queries': 0,
  1293. 'num_waiting_queries': 0
  1294. }
  1295. def get_query(self, query_id):
  1296. return MockImpalaQueryApi.PLAN[query_id]
  1297. def get_query_profile(self, query_id):
  1298. return MockImpalaQueryApi.PROFILE[query_id]
  1299. def get_query_memory(self, query_id):
  1300. return MockImpalaQueryApi.MEMORY[query_id]
  1301. def kill(self, query_id):
  1302. pass
  1303. class MockMapreduce2Api(object):
  1304. """
  1305. MockMapreduceApi and HistoryServerApi are very similar and inherit from it.
  1306. """
  1307. def __init__(self, mr_url=None): pass
  1308. def tasks(self, job_id):
  1309. return {
  1310. u'tasks': {
  1311. u'task': [
  1312. {
  1313. u'finishTime': 1357153330271, u'successfulAttempt': u'attempt_1356251510842_0062_m_000000_0', u'elapsedTime': 1901,
  1314. u'state': u'SUCCEEDED', u'startTime': 1357153328370, u'progress': 100.0, u'type': u'MAP',
  1315. u'id': u'task_1356251510842_0062_m_000000'
  1316. },
  1317. {
  1318. u'finishTime': 0, u'successfulAttempt': u'', u'elapsedTime': 0, u'state': u'SCHEDULED', u'startTime': 1357153326322,
  1319. u'progress': 0.0, u'type': u'REDUCE', u'id': u'task_1356251510842_0062_r_000000'
  1320. }
  1321. ]
  1322. }
  1323. }
  1324. def conf(self, job_id):
  1325. return {
  1326. "conf": {
  1327. "path": "hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml",
  1328. "property": [
  1329. {
  1330. "name": "dfs.datanode.data.dir",
  1331. "value": "/home/hadoop/hdfs/data",
  1332. }, {
  1333. "name": "mapreduce.job.acl-modify-job",
  1334. "value": "test",
  1335. }, {
  1336. "name": "mapreduce.job.acl-view-job",
  1337. "value": "test,test2",
  1338. }
  1339. ]
  1340. }
  1341. }
  1342. def job_attempts(self, job_id):
  1343. return {
  1344. "jobAttempts": {
  1345. "jobAttempt": [
  1346. {
  1347. "nodeId": "host.domain.com:8041",
  1348. "nodeHttpAddress": "host.domain.com:8042",
  1349. "startTime": 1326238773493,
  1350. "id": 1,
  1351. "logsLink": "http://host.domain.com:8042/node/containerlogs/container_1326232085508_0004_01_000001",
  1352. "containerId": "container_1326232085508_0004_01_000001"
  1353. }
  1354. ]
  1355. }
  1356. }
  1357. def task_attempts(self, job_id, task_id):
  1358. return {
  1359. "taskAttempts": {
  1360. "taskAttempt": [
  1361. {
  1362. "elapsedMergeTime": 47,
  1363. "shuffleFinishTime": 1326238780052,
  1364. "assignedContainerId": "container_1326232085508_0004_01_000003",
  1365. "progress": 100,
  1366. "elapsedTime": 0,
  1367. "state": "RUNNING",
  1368. "elapsedShuffleTime": 2592,
  1369. "mergeFinishTime": 1326238780099,
  1370. "rack": "/98.139.92.0",
  1371. "elapsedReduceTime": 0,
  1372. "nodeHttpAddress": "host.domain.com:8042",
  1373. "type": "REDUCE",
  1374. "startTime": 1326238777460,
  1375. "id": "attempt_1326232085508_4_4_r_0_0",
  1376. "finishTime": 0
  1377. }
  1378. ]
  1379. }
  1380. }
  1381. def counters(self, job_id):
  1382. return {
  1383. "jobCounters": {
  1384. "id": "job_1326232085508_4_4",
  1385. "counterGroup": [
  1386. {
  1387. "counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
  1388. "counter": [
  1389. {
  1390. "reduceCounterValue": 0,
  1391. "mapCounterValue": 0,
  1392. "totalCounterValue": 0,
  1393. "name": "BYTES_READ"
  1394. }
  1395. ]
  1396. },
  1397. {
  1398. "counterGroupName": "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
  1399. "counter": [
  1400. {
  1401. "reduceCounterValue": 0,
  1402. "mapCounterValue": 0,
  1403. "totalCounterValue": 0,
  1404. "name": "BYTES_WRITTEN"
  1405. }
  1406. ]
  1407. }
  1408. ]
  1409. }
  1410. }
  1411. def kill(self, job_id):
  1412. job_id = job_id.replace('job', 'application')
  1413. MockResourceManagerApi.APPS[job_id]['state'] = 'KILLED'
  1414. return {}
  1415. class MockMapreduceApi(MockMapreduce2Api):
  1416. def job(self, user, job_id):
  1417. if '1356251510842_0009' not in job_id:
  1418. job = {
  1419. u'job': {
  1420. u'reducesCompleted': 0, u'mapsRunning': 1, u'id': u'job_1356251510842_0054', u'successfulReduceAttempts': 0,
  1421. u'successfulMapAttempts': 0, u'uberized': False, u'reducesTotal': 1, u'elapsedTime': 3426, u'mapsPending': 0,
  1422. u'state': u'RUNNING', u'failedReduceAttempts': 0, u'mapsCompleted': 0, u'killedMapAttempts': 0, u'killedReduceAttempts': 0,
  1423. u'runningReduceAttempts': 0, u'failedMapAttempts': 0, u'mapsTotal': 1, u'user': u'test', u'startTime': 1357152972886,
  1424. u'reducesPending': 1, u'reduceProgress': 0.0, u'finishTime': 0, u'name': u'select avg(salary) from sample_07(Stage-1)',
  1425. u'reducesRunning': 0, u'newMapAttempts': 0, u'diagnostics': u'', u'mapProgress': 0.0,
  1426. u'runningMapAttempts': 1, u'newReduceAttempts': 1,
  1427. # Does not seems to exist in API, we actually skip it in case.
  1428. "acls": [{
  1429. "value": "test",
  1430. "name": "mapreduce.job.acl-modify-job"
  1431. }, {
  1432. "value": "test",
  1433. "name": "mapreduce.job.acl-view-job"
  1434. }
  1435. ],
  1436. }
  1437. }
  1438. job['job']['id'] = job_id
  1439. return job
  1440. class MockSparkHistoryApi(SparkHistoryServerApi):
  1441. def __init__(self):
  1442. self.APPS = [{
  1443. "id": "application_1513618343677_0018",
  1444. "name": "Sleep15minPySpark",
  1445. "attempts": [{
  1446. "attemptId": "1",
  1447. "startTime": "2017-12-20T20:25:19.672GMT",
  1448. "endTime": "2017-12-20T20:40:43.768GMT",
  1449. "sparkUser": "test",
  1450. "completed": True
  1451. }]
  1452. }, {
  1453. "id": "application_1513618343677_0020",
  1454. "name": "Sleep15minPySpark",
  1455. "attempts": [{
  1456. "attemptId": "2",
  1457. "startTime": "2017-12-24T03:19:29.993GMT",
  1458. "endTime": "1969-12-31T23:59:59.999GMT",
  1459. "sparkUser": "test",
  1460. "completed": False
  1461. }, {
  1462. "attemptId": "1",
  1463. "startTime": "2017-12-24T03:12:50.763GMT",
  1464. "endTime": "2017-12-24T03:19:22.178GMT",
  1465. "sparkUser": "test",
  1466. "completed": True
  1467. }]
  1468. }]
  1469. def applications(self):
  1470. return self.APPS
  1471. def executors(self, job):
  1472. EXECUTORS_LISTS = {
  1473. u'application_1513618343677_0018/1': [{
  1474. u'diskUsed': 0,
  1475. u'totalShuffleWrite': 0,
  1476. u'totalCores': 0,
  1477. u'executorLogs': {
  1478. u'stderr': u'http://localhost:8042/node/containerlogs/container_1513618343677_0018_01_000001/test/stderr?start=-4096',
  1479. u'stdout': u'http://localhost:8042/node/containerlogs/container_1513618343677_0018_01_000001/test/stdout?start=-4096'
  1480. },
  1481. u'totalInputBytes': 0,
  1482. u'rddBlocks': 0,
  1483. u'maxMemory': 515553361,
  1484. u'totalShuffleRead': 0,
  1485. u'totalTasks': 0,
  1486. u'activeTasks': 0,
  1487. u'failedTasks': 0,
  1488. u'completedTasks': 0,
  1489. u'hostPort': u'172.31.122.54:43234',
  1490. u'maxTasks': 0, u'totalGCTime': 0,
  1491. u'isBlacklisted': False,
  1492. u'memoryUsed': 0,
  1493. u'id': u'driver',
  1494. u'isActive': True,
  1495. u'totalDuration': 0
  1496. }],
  1497. u'application_1513618343677_0020/2': [{
  1498. u'diskUsed': 0,
  1499. u'totalShuffleWrite': 0,
  1500. u'totalCores': 0,
  1501. u'executorLogs': {
  1502. u'stderr': u'http://localhost:8042/node/containerlogs/container_1513618343677_0020_01_000001/test/stderr?start=-4096',
  1503. u'stdout': u'http://localhost:8042/node/containerlogs/container_1513618343677_0020_01_000001/test/stdout?start=-4096'},
  1504. u'totalInputBytes': 0,
  1505. u'rddBlocks': 0,
  1506. u'maxMemory': 515553361,
  1507. u'totalShuffleRead': 0,
  1508. u'totalTasks': 0,
  1509. u'activeTasks': 0,
  1510. u'failedTasks': 0,
  1511. u'completedTasks': 0,
  1512. u'hostPort': u'172.31.122.65:38210',
  1513. u'maxTasks': 0,
  1514. u'totalGCTime': 0,
  1515. u'isBlacklisted': False,
  1516. u'memoryUsed': 0,
  1517. u'id': u'driver',
  1518. u'isActive': True,
  1519. u'totalDuration': 0}]
  1520. }
  1521. app_id = self.get_real_app_id(job)
  1522. if not app_id:
  1523. return []
  1524. return EXECUTORS_LISTS[app_id] if app_id in EXECUTORS_LISTS else []
  1525. def download_executors_logs(self, request, job, name, offset):
  1526. return 'dummy_logs'
  1527. def download_executor_logs(self, user, executor, name, offset):
  1528. return 'dummy_log'
  1529. def get_executors_loglinks(self, job):
  1530. return None
  1531. class HistoryServerApi(MockMapreduce2Api):
  1532. def __init__(self, hs_url=None): pass
  1533. def job(self, user, job_id):
  1534. if '1356251510842_0054' == job_id:
  1535. return {
  1536. u'job': {
  1537. u'reducesCompleted': 1, u'avgMapTime': 1798, u'avgMergeTime': 1479, u'id': job_id,
  1538. u'successfulReduceAttempts': 1, u'successfulMapAttempts': 2, u'uberized': False, u'reducesTotal': 1,
  1539. u'state': u'KILLED', u'failedReduceAttempts': 0, u'mapsCompleted': 2,
  1540. u'killedMapAttempts': 0, u'diagnostics': u'', u'mapsTotal': 2, u'user': u'test',
  1541. u'startTime': 1357151916268, u'avgReduceTime': 137, u'finishTime': 1357151923925,
  1542. u'name': u'oozie:action:T=map-reduce:W=MapReduce-copy:A=Sleep:ID=0000004-121223003201296-oozie-oozi-W',
  1543. u'avgShuffleTime': 1421, u'queue': u'default', u'killedReduceAttempts': 0, u'failedMapAttempts': 0
  1544. }
  1545. }
  1546. else:
  1547. return {
  1548. u'job': {
  1549. u'reducesCompleted': 1, u'avgMapTime': 1798, u'avgMergeTime': 1479, u'id': u'job_1356251510842_0009',
  1550. u'successfulReduceAttempts': 1, u'successfulMapAttempts': 2, u'uberized': False, u'reducesTotal': 1,
  1551. u'state': u'SUCCEEDED', u'failedReduceAttempts': 0, u'mapsCompleted': 2,
  1552. u'killedMapAttempts': 0, u'diagnostics': u'', u'mapsTotal': 2, u'user': u'test',
  1553. u'startTime': 0, u'avgReduceTime': 137, u'finishTime': 1357151923925,
  1554. u'name': u'oozie:action:T=map-reduce:W=MapReduce-copy:A=Sleep:ID=0000004-121223003201296-oozie-oozi-W',
  1555. u'avgShuffleTime': 1421, u'queue': u'default', u'killedReduceAttempts': 0, u'failedMapAttempts': 0
  1556. }
  1557. }
  1558. def test_make_log_links():
  1559. """
  1560. Unit test for models.LinkJobLogs._make_links
  1561. """
  1562. # FileBrowser
  1563. assert_equal(
  1564. """<a href="/filebrowser/view=/user/romain/tmp">hdfs://localhost:8020/user/romain/tmp</a> &lt;dir&gt;""",
  1565. LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/tmp <dir>')
  1566. )
  1567. assert_equal(
  1568. """<a href="/filebrowser/view=/user/romain/tmp">hdfs://localhost:8020/user/romain/tmp</a>&lt;dir&gt;""",
  1569. LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/tmp<dir>')
  1570. )
  1571. assert_equal(
  1572. """output: <a href="/filebrowser/view=/user/romain/tmp">/user/romain/tmp</a> &lt;dir&gt;""",
  1573. LinkJobLogs._make_links('output: /user/romain/tmp <dir>')
  1574. )
  1575. assert_equal(
  1576. ('Successfully read 3760 records (112648 bytes) from: &quot;<a href="/filebrowser/view=/user/hue/pig/examples/data/midsummer.txt">'
  1577. '/user/hue/pig/examples/data/midsummer.txt</a>&quot;'),
  1578. LinkJobLogs._make_links('Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"')
  1579. )
  1580. assert_equal(
  1581. 'data,upper_case MAP_ONLY <a href="/filebrowser/view=/user/romain/out/fffff">hdfs://localhost:8020/user/romain/out/fffff</a>,',
  1582. LinkJobLogs._make_links('data,upper_case MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff,')
  1583. )
  1584. assert_equal(
  1585. 'MAP_ONLY <a href="/filebrowser/view=/user/romain/out/fffff">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013',
  1586. LinkJobLogs._make_links('MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff\n2013')
  1587. )
  1588. assert_equal(
  1589. ' <a href="/filebrowser/view=/jobs.tsv">/jobs.tsv</a> ',
  1590. LinkJobLogs._make_links(' /jobs.tsv ')
  1591. )
  1592. assert_equal(
  1593. '<a href="/filebrowser/view=/user/romain/job_pos_2012.tsv">hdfs://localhost:8020/user/romain/job_pos_2012.tsv</a>',
  1594. LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/job_pos_2012.tsv')
  1595. )
  1596. # JobBrowser
  1597. assert_equal(
  1598. """<a href="/hue/jobbrowser/jobs/job_201306261521_0058">job_201306261521_0058</a>""",
  1599. LinkJobLogs._make_links('job_201306261521_0058')
  1600. )
  1601. assert_equal(
  1602. """Hadoop Job IDs executed by Pig: <a href="/hue/jobbrowser/jobs/job_201306261521_0058">job_201306261521_0058</a>""",
  1603. LinkJobLogs._make_links('Hadoop Job IDs executed by Pig: job_201306261521_0058')
  1604. )
  1605. assert_equal(
  1606. """MapReduceLauncher - HadoopJobId: <a href="/hue/jobbrowser/jobs/job_201306261521_0058">job_201306261521_0058</a>""",
  1607. LinkJobLogs._make_links('MapReduceLauncher - HadoopJobId: job_201306261521_0058')
  1608. )
  1609. assert_equal(
  1610. ('- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/hue/jobbrowser/jobs/job_201306261521_0058">'
  1611. 'job_201306261521_0058</a>'),
  1612. LinkJobLogs._make_links('- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058')
  1613. )
  1614. assert_equal(
  1615. ' Logging error messages to: <a href="/hue/jobbrowser/jobs/job_201307091553_0028">job_201307091553_0028</a>/attempt_201307091553_002',
  1616. LinkJobLogs._make_links(' Logging error messages to: job_201307091553_0028/attempt_201307091553_002')
  1617. )
  1618. assert_equal(
  1619. """ pig-<a href="/hue/jobbrowser/jobs/job_201307091553_0028">job_201307091553_0028</a>.log""",
  1620. LinkJobLogs._make_links(' pig-job_201307091553_0028.log')
  1621. )
  1622. assert_equal(
  1623. 'MapReduceLauncher - HadoopJobId: <a href="/hue/jobbrowser/jobs/job_201306261521_0058">job_201306261521_0058</a>. Look at the UI',
  1624. LinkJobLogs._make_links('MapReduceLauncher - HadoopJobId: job_201306261521_0058. Look at the UI')
  1625. )