Переглянути джерело

HUE-9046 [jb] Add link to queries-hive in editor.

Change-Id: I2590cd7d2b8a06ad8dd8eccb82be34f9a97d65bb
Jean-Francois Desjeans Gauthier 6 роки тому
батько
коміт
828dcb9458

+ 1 - 1
apps/beeswax/src/beeswax/query_history.py

@@ -126,7 +126,7 @@ class QueryHistory(object):
       by_user = self.by_user.get(request_user)
       if by_user and by_user['filters'] == filters:
         del self.by_user[request_user] # Moving request_user to head of queue
-        by_user[request_user] = by_user
+        self.by_user[request_user] = by_user
         return by_user
       return None
     finally:

+ 23 - 1
apps/beeswax/src/beeswax/tests.py

@@ -79,7 +79,7 @@ import beeswax.views
 from beeswax import conf, hive_site
 from beeswax.common import apply_natural_sort
 from beeswax.conf import HIVE_SERVER_HOST, AUTH_USERNAME, AUTH_PASSWORD, AUTH_PASSWORD_SCRIPT
-from beeswax.views import collapse_whitespace, _save_design, parse_out_jobs
+from beeswax.views import collapse_whitespace, _save_design, parse_out_jobs, parse_out_queries
 from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config,\
   fetch_query_result_data
 from beeswax.design import hql_query
@@ -135,6 +135,28 @@ def get_csv(client, result_response):
   return ''.join(csv_resp.streaming_content)
 
 
+class TestBeeswax(object):
+  def test_parse_out_queries(self):
+    text = """INFO  : Compiling command(queryId=hive_20191029132605_17883ebe-d3d5-41bf-a1e9-01cf207a3c6b): select 1
+INFO  : Semantic Analysis Completed (retrial = false)
+INFO  : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:_c0, type:int, comment:null)], properties:null)
+INFO  : Completed compiling command(queryId=hive_20191029132605_17883ebe-d3d5-41bf-a1e9-01cf207a3c6b); Time taken: 0.031 seconds
+INFO  : Executing command(queryId=hive_20191029132605_17883ebe-d3d5-41bf-a1e9-01cf207a3c6b): select 1
+INFO  : Completed executing command(queryId=hive_20191029132605_17883ebe-d3d5-41bf-a1e9-01cf207a3c6b); Time taken: 0.004 seconds
+INFO  : OK"""
+    jobs = parse_out_queries(text, engine='tez', with_state=True)
+    assert_true(jobs and jobs[0]['job_id'] == 'hive_20191029132605_17883ebe-d3d5-41bf-a1e9-01cf207a3c6b')
+    assert_true(jobs and jobs[0]['started'] == True)
+    assert_true(jobs and jobs[0]['finished'] == True)
+
+    text = """INFO  : Compiling command(queryId=hive_20191029132605_17883ebe-d3d5-41bf-a1e9-01cf207a3c6a): select 1
+INFO  : OK"""
+    jobs = parse_out_queries(text, engine='tez', with_state=True)
+    assert_true(jobs and jobs[0]['job_id'] == 'hive_20191029132605_17883ebe-d3d5-41bf-a1e9-01cf207a3c6a')
+    assert_true(jobs and jobs[0]['started'] == False)
+    assert_true(jobs and jobs[0]['finished'] == False)
+    
+
 class TestBeeswaxWithHadoop(BeeswaxSampleProvider):
   requires_hadoop = True
   integration = True

+ 42 - 0
apps/beeswax/src/beeswax/views.py

@@ -65,6 +65,8 @@ LOG = logging.getLogger(__name__)
 HADOOP_JOBS_RE = re.compile("Starting Job = ([a-z0-9_]+?),")
 SPARK_APPLICATION_RE = re.compile("Running with YARN Application = (?P<application_id>application_\d+_\d+)")
 TEZ_APPLICATION_RE = re.compile("Executing on YARN cluster with App id ([a-z0-9_]+?)\)")
+TEZ_QUERY_RE = re.compile("\(queryId=([a-z0-9_-]+?)\)")
+
 
 
 def index(request):
@@ -953,6 +955,46 @@ def parse_out_jobs(log, engine='mr', with_state=False):
 
   return ret
 
+def parse_out_queries(log, engine=None, with_state=False):
+  """
+  Ideally, Hive would tell us what jobs it has run directly from the Thrift interface.
+
+  with_state: If True, will return a list of dict items with 'job_id', 'started', 'finished'
+  """
+  ret = []
+
+  if engine.lower() == 'tez':
+    start_pattern = TEZ_QUERY_RE
+  else:
+    return ret
+
+  for match in start_pattern.finditer(log):
+    job_id = match.group(1)
+
+    if with_state:
+      if job_id not in list(job['job_id'] for job in ret):
+        ret.append({'job_id': job_id, 'started': False, 'finished': False})
+      start_pattern = 'Executing command(queryId=%s' % job_id
+      end_pattern = 'Completed executing command(queryId=%s' % job_id
+
+      if start_pattern in log:
+        job = next((job for job in ret if job['job_id'] == job_id), None)
+        if job is not None:
+          job['started'] = True
+        else:
+          ret.append({'job_id': job_id, 'started': True, 'finished': False})
+
+      if end_pattern in log:
+        job = next((job for job in ret if job['job_id'] == job_id), None)
+        if job is not None:
+          job['finished'] = True
+        else:
+          ret.append({'job_id': job_id, 'started': True, 'finished': True})
+    else:
+      if job_id not in ret:
+        ret.append(job_id)
+
+  return ret
 
 def _copy_prefix(prefix, base_dict):
   """Copy keys starting with ``prefix``"""

+ 2 - 2
apps/jobbrowser/src/jobbrowser/api2.py

@@ -52,7 +52,7 @@ def jobs(request, interface=None):
   response = {'status': -1}
 
   cluster = json.loads(request.POST.get('cluster', '{}'))
-  interface = json.loads(request.POST.get('interface'))
+  interface = interface or json.loads(request.POST.get('interface'))
   filters = dict([(key, value) for _filter in json.loads(request.POST.get('filters', '[]')) for key, value in list(_filter.items()) if value])
 
   jobs = get_api(request.user, interface, cluster=cluster).apps(filters)
@@ -69,7 +69,7 @@ def job(request, interface=None):
   response = {'status': -1}
 
   cluster = json.loads(request.POST.get('cluster', '{}'))
-  interface = json.loads(request.POST.get('interface'))
+  interface = interface or json.loads(request.POST.get('interface'))
   app_id = json.loads(request.POST.get('app_id'))
 
   response_app = get_api(request.user, interface, cluster=cluster).app(app_id)

+ 5 - 0
apps/jobbrowser/src/jobbrowser/apis/beeswax_query_api.py

@@ -21,8 +21,11 @@ import re
 
 from datetime import datetime
 
+from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.python_util import current_ms_from_utc
 
+from django.utils.translation import ugettext as _
+
 from jobbrowser.apis.base_api import Api
 
 LOG = logging.getLogger(__name__)
@@ -82,6 +85,8 @@ class BeeswaxQueryApi(Api):
     jobs = query_history.get_query_by_id(self.user.get_username(), query_id=appid)
 
     current_time = current_ms_from_utc()
+    if not jobs['data']:
+      raise PopupException(_('Could not find query id %s' % appid))
     job = jobs['data'][0]
     app = {
       'id': job[0],

+ 8 - 1
apps/jobbrowser/src/jobbrowser/conf.py

@@ -58,7 +58,14 @@ MAX_JOB_FETCH = Config(
 
 ENABLE_QUERY_BROWSER = Config(
   key="enable_query_browser",
-  help=_("Show the query section for listing and showing more troubleshooting information."),
+  help=_("Show the Impala query section for listing and showing more troubleshooting information."),
   type=coerce_bool,
   default=True
 )
+
+ENABLE_HIVE_QUERY_BROWSER = Config(
+  key="enable_hive_query_browser",
+  help=_("# Show the Hive query section for listing and showing more troubleshooting information."),
+  type=coerce_bool,
+  default=False
+) 

+ 2 - 2
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -20,7 +20,7 @@ from desktop.conf import CUSTOM, IS_K8S_ONLY
 from desktop.views import commonheader, commonfooter, _ko
 from metadata.conf import PROMETHEUS
 
-from jobbrowser.conf import DISABLE_KILLING_JOBS, MAX_JOB_FETCH, ENABLE_QUERY_BROWSER
+from jobbrowser.conf import DISABLE_KILLING_JOBS, MAX_JOB_FETCH, ENABLE_QUERY_BROWSER, ENABLE_HIVE_QUERY_BROWSER
 %>
 
 <%
@@ -3592,7 +3592,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           return '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('impala') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
         };
         var queryHiveInterfaceCondition = function () {
-          return '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('hive') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
+          return '${ ENABLE_HIVE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('hive') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
         };
 
         var interfaces = [

+ 4 - 1
desktop/conf.dist/hue.ini

@@ -1587,9 +1587,12 @@
   # Show the version 2 of app which unifies all the past browsers into one.
   ## enable_v2=true
 
-  # Show the query section for listing and showing more troubleshooting information.
+  # Show the Impala query section for listing and showing more troubleshooting information.
   ## enable_query_browser=true
 
+  # Show the Hive query section for listing and showing more troubleshooting information.
+  ## enable_hive_query_browser=false
+
 
 ###########################################################################
 # Settings to configure Sentry / Security App.

+ 4 - 1
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1573,9 +1573,12 @@
   # Show the version 2 of app which unifies all the past browsers into one.
   ## enable_v2=true
 
-  # Show the query section for listing and showing more troubleshooting information.
+  # Show the Impala query section for listing and showing more troubleshooting information.
   ## enable_query_browser=true
 
+  # Show the Hive query section for listing and showing more troubleshooting information.
+  ## enable_hive_query_browser=false
+
 
 ###########################################################################
 # Settings to configure Sentry / Security App.

+ 12 - 2
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -64,7 +64,7 @@ try:
   from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, HiveServerQueryHistory, QueryHistory, Session
   from beeswax.server import dbms
   from beeswax.server.dbms import get_query_server_config, QueryServerException
-  from beeswax.views import parse_out_jobs
+  from beeswax.views import parse_out_jobs, parse_out_queries
 except ImportError as e:
   LOG.warn('Hive and HiveServer2 interfaces are not enabled: %s' % e)
   hive_settings = None
@@ -79,12 +79,14 @@ except ImportError as e:
 
 try:
   from jobbrowser.views import get_job
-  from jobbrowser.conf import ENABLE_QUERY_BROWSER
+  from jobbrowser.conf import ENABLE_QUERY_BROWSER, ENABLE_HIVE_QUERY_BROWSER
   from jobbrowser.apis.query_api import _get_api
   has_query_browser = ENABLE_QUERY_BROWSER.get()
+  has_hive_query_browser = ENABLE_HIVE_QUERY_BROWSER.get()
 except (AttributeError, ImportError) as e:
   LOG.warn("Job Browser app is not enabled")
   has_query_browser = False
+  has_hive_query_browser = False
 
 
 DEFAULT_HIVE_ENGINE = 'mr'
@@ -442,6 +444,7 @@ class HS2Api(Api):
     if snippet['type'] == 'hive':
       engine = self._get_hive_execution_engine(notebook, snippet)
       jobs_with_state = parse_out_jobs(logs, engine=engine, with_state=True)
+      queries_with_state = parse_out_queries(logs, engine=engine, with_state=True)
 
       jobs = [{
         'name': job.get('job_id', ''),
@@ -449,6 +452,13 @@ class HS2Api(Api):
         'started': job.get('started', False),
         'finished': job.get('finished', False)
       } for job in jobs_with_state]
+      if has_hive_query_browser:
+        jobs += [{
+          'name': job.get('job_id', ''),
+          'url': 'api/job/queries-hive/',
+          'started': job.get('started', False),
+          'finished': job.get('finished', False)
+        } for job in queries_with_state]
     elif snippet['type'] == 'impala' and has_query_browser:
       query_id = unpack_guid_base64(snippet['result']['handle']['guid'])
       progress = min(self.progress(notebook, snippet, logs), 99) if snippet['status'] != 'available' and snippet['status'] != 'success' else 100