Răsfoiți Sursa

HUE-2143 [beeswax] MR job links are not always displayed

Job Browser understands either job_XXX or application_XXX.
Romain Rigaux 11 ani în urmă
părinte
comite
f3549ea

+ 13 - 2
apps/beeswax/src/beeswax/tests.py

@@ -1596,12 +1596,23 @@ Starting Job = job_201003191517_0002, Tracking URL = http://localhost:50030/jobd
     --- we should be ignoring duplicates ---
     --- we should be ignoring duplicates ---
 Starting Job = job_201003191517_0002, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201003191517_0002
 Starting Job = job_201003191517_0002, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201003191517_0002
 Starting Job = job_201003191517_0003, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201003191517_0003
 Starting Job = job_201003191517_0003, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201003191517_0003
-12/12/27 10:48:22 INFO mapreduce.Job: The url to track the job: http://localhost:8088/proxy/application_1356251510842_0022/
+14/06/10 14:30:55 INFO exec.Task: Starting Job = job_1402420825148_0001, Tracking URL = http://localhost:8088/proxy/application_1402420825148_0001/
 """
 """
-  assert_equal(["job_201003191517_0002", "job_201003191517_0003", "application_1356251510842_0022"],
+  assert_equal(
+    ["job_201003191517_0002", "job_201003191517_0003", "job_1402420825148_0001"],
     beeswax.views._parse_out_hadoop_jobs(sample_log))
     beeswax.views._parse_out_hadoop_jobs(sample_log))
   assert_equal([], beeswax.views._parse_out_hadoop_jobs("nothing to see here"))
   assert_equal([], beeswax.views._parse_out_hadoop_jobs("nothing to see here"))
 
 
+  sample_log_no_direct_url = """
+14/06/09 08:40:38 INFO impl.YarnClientImpl: Submitted application application_1402269517321_0003
+14/06/09 08:40:38 INFO mapreduce.Job: The url to track the job: N/A
+14/06/09 08:40:38 INFO exec.Task: Starting Job = job_1402269517321_0003, Tracking URL = N/A
+14/06/09 08:40:38 INFO exec.Task: Kill Command = /usr/lib/hadoop/bin/hadoop job  -kill job_1402269517321_0003
+14/06/09 08:40:38 INFO cli.CLIService: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=2168d15e-96d2-415a-8d49-e2535e82c2a4]: getOperationStatus()
+"""
+  assert_equal(
+      ["job_1402269517321_0003"],
+      beeswax.views._parse_out_hadoop_jobs(sample_log_no_direct_url))
 
 
 def test_hive_site():
 def test_hive_site():
   tmpdir = tempfile.mkdtemp()
   tmpdir = tempfile.mkdtemp()

+ 3 - 16
apps/beeswax/src/beeswax/views.py

@@ -866,29 +866,16 @@ def parse_query_context(context):
   return pair
   return pair
 
 
 
 
-HADOOP_JOBS_RE = re.compile("(http[^\s]*/jobdetails.jsp\?jobid=([a-z0-9_]*))")
-HADOOP_YARN_JOBS_RE = re.compile("(http[^\s]*/proxy/([a-z0-9_]+?)/)")
+HADOOP_JOBS_RE = re.compile("Starting Job = ([a-z0-9_]+?),")
 
 
 def _parse_out_hadoop_jobs(log):
 def _parse_out_hadoop_jobs(log):
   """
   """
-  Ideally, Hive would tell us what jobs it has run directly
-  from the Thrift interface.  For now, we parse the logs
-  to look for URLs to those jobs.
+  Ideally, Hive would tell us what jobs it has run directly from the Thrift interface.
   """
   """
   ret = []
   ret = []
 
 
   for match in HADOOP_JOBS_RE.finditer(log):
   for match in HADOOP_JOBS_RE.finditer(log):
-    full_job_url, job_id = match.groups()
-    # We ignore full_job_url for now, but it may
-    # come in handy if we support multiple MR clusters
-    # correctly.
-
-    # Ignore duplicates
-    if job_id not in ret:
-      ret.append(job_id)
-
-  for match in HADOOP_YARN_JOBS_RE.finditer(log):
-    full_job_url, job_id = match.groups()
+    job_id = match.group(1)
     if job_id not in ret:
     if job_id not in ret:
       ret.append(job_id)
       ret.append(job_id)
 
 

+ 1 - 1
apps/jobbrowser/src/jobbrowser/templates/jobs.mako

@@ -72,7 +72,7 @@ ${ components.menubar() }
       <th>${_('Queue')}</th>
       <th>${_('Queue')}</th>
       <th>${_('Priority')}</th>
       <th>${_('Priority')}</th>
       <th>${_('Duration')}</th>
       <th>${_('Duration')}</th>
-      <th>${_('Submission')}</th>
+      <th>${_('Submitted')}</th>
       <th data-row-selector-exclude="true"></th>
       <th data-row-selector-exclude="true"></th>
     </tr>
     </tr>
     </thead>
     </thead>