Forráskód Böngészése

HUE-9046 [jb] Hive query history

Change-Id: I959763409fbd94b8bce77c002e554794ef6ceea6
Jean-Francois Desjeans Gauthier 6 éve
szülő
commit
5afb9e5845

+ 3 - 0
apps/beeswax/src/beeswax/hive_site.py

@@ -65,6 +65,7 @@ _CNF_HIVESERVER2_THRIFT_SASL_QOP = 'hive.server2.thrift.sasl.qop'
 _CNF_HIVESERVER2_USE_SASL = 'hive.metastore.sasl.enabled'
 
 _CNF_HIVE_SUPPORT_CONCURRENCY = 'hive.support.concurrency'
+_CNF_HIVE_HOOK_PROTO_BASE_DIR = 'hive.hook.proto.base-directory'
 
 
 # Host is whatever up to the colon. Allow and ignore a trailing slash.
@@ -191,6 +192,8 @@ def has_concurrency_support():
   '''For SQL transactions like INSERT, DELETE, UPDATE since Hive 3. Possibly use set -v in future to obtain properties hive.create.as.acid=true & hive.create.as.insert.only=true'''
   return get_conf().get(_CNF_HIVE_SUPPORT_CONCURRENCY, 'TRUE').upper() == 'TRUE'
 
+def get_hive_hook_proto_base_directory():
+  return get_conf().get(_CNF_HIVE_HOOK_PROTO_BASE_DIR)
 
 def _parse_hive_site():
   """

+ 83 - 0
apps/beeswax/src/beeswax/management/commands/create_table_query_data.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from builtins import str
+
+import logging
+
+from django.core.management.base import BaseCommand
+from django.utils.translation import ugettext as _
+
+
+from desktop.lib import django_mako
+from beeswax.server import dbms
+from beeswax.server.dbms import get_query_server_config
+from desktop.conf import DEFAULT_USER
+
+from beeswax.design import hql_query
+from beeswax import hive_site
+from useradmin.models import User
+
+
+LOG = logging.getLogger(__name__)
+DEFAULT_USER = DEFAULT_USER.get()
+
+
+class Command(BaseCommand):
+  """
+  Create table sys.query_data over hive.hook.proto.base-directory
+  """
+  args = ''
+  help = 'Create table sys.query_data over hive.hook.proto.base-directory'
+
+
+  def handle(self, *args, **options):
+    create_table()
+
+
+def create_table(user=None, query_server=None, table=None):
+  if not user:
+    user = User.objects.get(username=DEFAULT_USER)
+  if not query_server:
+    query_server = get_query_server_config('beeswax')
+  if not table:
+    base_dir = hive_site.get_hive_hook_proto_base_directory()
+    if not base_dir:
+      msg = _('Error creating table query_data hive.hook.proto.base-directory is not configured')
+      LOG.error(msg)
+      return False
+    table = {
+      'name': 'query_data',
+      'external_location': base_dir
+    }
+
+  server = dbms.get(user, query_server)
+  for query in ["create_table_query_data.mako", "msck.mako"]:
+    proposed_query = django_mako.render_to_string(query, {'table': table})
+    query = hql_query(proposed_query)
+    try:
+      handle = server.execute_and_wait(query)
+      if not handle:
+        LOG.error(_('Error executing %s: Operation timeout.' % query))
+        return False
+      server.close(handle)
+    except Exception as ex:
+      LOG.error(_('Error executing %(query)s: %(error)s.') % {'query': query, 'error': ex})
+      return False
+
+  LOG.info(_('Table query_data has been created successfully'))
+  return True

+ 285 - 0
apps/beeswax/src/beeswax/query_history.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from builtins import filter
+from builtins import str
+
+import collections
+import logging
+import json
+import threading
+import uuid
+
+from django.utils.translation import ugettext as _
+
+from beeswax.design import hql_query
+from beeswax.server import dbms
+from beeswax.server.dbms import get_query_server_config
+from beeswax.management.commands import create_table_query_data
+
+from desktop.conf import DEFAULT_USER
+from desktop.lib.exceptions_renderable import raise_popup_exception, PopupException
+from desktop.lib import django_mako
+
+from useradmin.models import User
+
+LOG = logging.getLogger(__name__)
+DEFAULT_USER = DEFAULT_USER.get()
+
+QUERY_HISTORY_CACHE_MAX_USER_COUNT = 10
+QUERY_HISTORY_CACHE_MAX_LENGTH_PER_USER = 25
+HAS_CREATED_TABLE = False
+
+class QueryHistory(object):
+  def __init__(self, max_user=10, max_history_per_user=25):
+    self.max_user=max_user
+    self.max_history_per_user=max_history_per_user
+    self.by_user = collections.OrderedDict()
+    self.no_user_key = str(uuid.uuid4())
+    self.lock = threading.Lock()
+
+  def _remove(self, request_user):
+    if request_user in self.by_user:
+      del self.by_user[request_user]
+
+  # Data has to be sorted & grouped by
+  def _append(self, by_user, data):
+    total_history = data + by_user['queries']
+    if self.max_history_per_user < len(total_history):
+      by_user['queries'] = total_history[:self.max_history_per_user]
+    else:
+      by_user['queries'] = total_history
+
+    by_user['by_id'] = {}
+    by_user['max'] = {'time': 0, 'date': ''}
+    for row in by_user['queries']:
+      by_user['by_id'][row[0]] = row
+      if row[1][-1] > by_user['max']['time']:
+        by_user['max']['time'] = row[1][-1]
+        by_user['max']['date'] = row[7]
+
+    return total_history
+
+  def _add(self, request_user, filters):
+    if len(self.by_user) >= self.max_user:
+      # Remove eldest user and all its queries
+      self.by_user.popitem(last=False)
+
+    by_id = {'queries': [], 'max': {'time': 0, 'date': ''}, 'by_id': {}, 'filters': filters, 'by_id': {}}
+    self.by_user[request_user] = by_id
+    return by_id
+
+  def set(self, request_user, data, filters):
+    if not request_user:
+      request_user = self.no_user_key
+    try:
+      self.lock.acquire()
+      self._remove(request_user)
+      by_id = self._add(request_user, filters)
+      self._append(by_id, data)
+    finally:
+      self.lock.release()
+
+  def update(self, by_user, data):
+
+    try:
+      self.lock.acquire()
+      # Get records not present in current history and append
+      # Update records already in query history
+      results = _groupby(by_user, data)
+      return self._append(by_user, results)
+    finally:
+      self.lock.release()
+
+  def get_by_id(self, request_user, query_id):
+    if not request_user:
+      request_user = self.no_user_key
+
+    try:
+      self.lock.acquire()
+      value = self.by_user.get(request_user)
+      if value:
+        return value['by_id'].get(query_id)
+      else:
+        return value
+    finally:
+      self.lock.release()
+
+  def get_queries(self, request_user, filters):
+    if not request_user:
+      request_user = self.no_user_key
+
+    try:
+      self.lock.acquire()
+      by_user = self.by_user.get(request_user)
+      if by_user and by_user['filters'] == filters:
+        del self.by_user[request_user] # Moving request_user to head of queue
+        by_user[request_user] = by_user
+        return by_user
+      return None
+    finally:
+      self.lock.release()
+
+QUERY_HISTORY = QueryHistory(max_user=QUERY_HISTORY_CACHE_MAX_USER_COUNT, max_history_per_user=QUERY_HISTORY_CACHE_MAX_LENGTH_PER_USER)
+
+# If fresh user get from _get_query_history_latest else get _get_query_history_from. if results set from _get_query_history_from less than limit merge results with cache else call _get_query_history_latest
+def get_query_history(request_user=None, start_date=None, start_time=None, query_id=None, status=None, limit=None):
+  _init_table()
+
+  filters = {'start_date': start_date, 'start_time': start_time, 'query_id': query_id, 'status': status}
+  history = QUERY_HISTORY.get_queries(request_user, filters)
+  if history:
+    # Get last
+    last = history['max']
+    data = _get_query_history_from(request_user=request_user,
+                                   start_date=last['date'],
+                                   start_time=last['time']+1,
+                                   query_id=query_id,
+                                   status=status,
+                                   limit=limit)
+    if not limit or len(data['data']) < limit:
+      cached = QUERY_HISTORY.update(history, data['data'])
+      filter_list = _get_filter_list({'states': status})
+      cached = _n_filter(filter_list, cached)[:limit]
+      return {'data': cached}
+
+  data = _get_query_history_latest(request_user=request_user, start_date=start_date, start_time=start_time, query_id=query_id, status=status, limit=limit, force_refresh=True)
+  QUERY_HISTORY.set(request_user, data['data'], filters)
+  return data
+
+# If id in cache return cache else _get_query_history_from
+def get_query_by_id(request_user=None, query_id=None):
+  _init_table()
+
+  datum = QUERY_HISTORY.get_by_id(request_user, query_id)
+  if datum:
+    return {'data': [datum]}
+  else:
+    data = _get_query_history_from(request_user=request_user, query_id=query_id) # force_refresh?
+    cached = _groupby({'by_id': {}}, data['data'])
+    return {'data': cached}
+
+def _init_table():
+  global HAS_CREATED_TABLE
+  if not HAS_CREATED_TABLE:
+    if create_table_query_data.create_table():
+      HAS_CREATED_TABLE = True
+  if not HAS_CREATED_TABLE:
+    raise PopupException(_('Could not initialize query history table.'))
+
+def _get_query_history_latest(request_user=None, query_id=None, start_date=None, start_time=None, status=None, limit=25, force_refresh=False):
+  proposed_query = django_mako.render_to_string("select_table_query_data_latest.mako", {'table': {'name': 'query_data', 'request_user': request_user, 'query_id': query_id, 'start_date': start_date, 'start_time': start_time, 'status': status, 'limit': limit, 'force_refresh': force_refresh}})
+  data = _execute_query(proposed_query, limit)
+  for row in data['data']:
+    if row[1]:
+      row[1] = json.loads(row[1])
+    if row[5]:
+      row[5] = json.loads(row[5])
+    if row[8]:
+      row[8] = json.loads(row[8])
+  return data
+
+def _get_query_history_from(request_user=None, start_date=None, start_time=None, status=None, query_id=None, limit=25):
+  proposed_query = django_mako.render_to_string("select_table_query_data_from.mako",
+                                                {'table':
+                                                 {'name': 'query_data',
+                                                  'request_user': request_user,
+                                                  'start_date': start_date,
+                                                  'start_time': start_time,
+                                                  'query_id': query_id,
+                                                  'status': status,
+                                                  'limit': limit}})
+  data = _execute_query(proposed_query, limit)
+  for row in data['data']:
+    if row[1]:
+      row[1] = [row[1]]
+    if row[5]:
+      row[5] = json.loads(row[5])
+    if row[8]:
+      row[8] = [row[8]]
+  return data
+
+def _execute_query(proposed_query, limit):
+  user = User.objects.get(username=DEFAULT_USER)
+  query_server = get_query_server_config('beeswax')
+  server = dbms.get(user, query_server)
+  query = hql_query(proposed_query)
+  try:
+    handle = server.execute_and_wait(query)
+    if not handle:
+      LOG.error(_('Error executing %s: Operation timeout.' % query))
+      return []
+    results = server.fetch(handle, True, limit)
+    rows = [row for row in results.rows()]
+    data = {
+        'data': rows,
+        'columns': [column.name for column in results.data_table.cols()]}
+
+    return data
+  except Exception as ex:
+    raise_popup_exception(_('Error fetching query history.'))
+  finally:
+    try:
+      if server and handle:
+        server.close(handle)
+    except Exception as ex:
+      raise_popup_exception(_('Error fetching query history.'))
+
+def _get_filter_list(filters):
+  filter_list = []
+  if filters.get("states"):
+    filter_list.append(lambda app: _get_status(app) in filters.get("states"))
+
+  return filter_list
+
+def _get_status(row):
+  return 'completed' if len(row[1]) >= 2 else 'running'
+
+def _n_filter(filters, tuples):
+  for f in filters:
+    tuples = list(filter(f, tuples))
+  return tuples
+
+def _groupby(by_user, data):
+  results = []
+  for row in data:
+    if not by_user['by_id'].get(row[0]):
+      if not isinstance(row[1], list):
+        row[1] = [row[1]]
+      if not isinstance(row[8], list):
+        row[8] = [row[8]]
+      by_user['by_id'][row[0]] = row
+      results.append(row)
+    else:
+      item = by_user['by_id'][row[0]]
+      if row[8][0] in item[8]: # we have dup
+        continue
+      if row[1]:
+        item[1] += row[1]
+      if row[2]:
+        item[2] = row[2]
+      if row[4]:
+        item[4] = row[4]
+      if row[5]:
+        item[5] = row[5]
+      if row[6]:
+        item[6] = row[6]
+      if row[8]:
+        item[8] += row[8]
+
+  results.sort(key=lambda result: result[1][0], reverse=True)
+  return results

+ 35 - 0
apps/beeswax/src/beeswax/templates/create_table_query_data.mako

@@ -0,0 +1,35 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+create table if not exists sys.${table["name"]} (
+    eventType string,
+    hiveQueryId string,
+    `timestamp` BIGINT,
+    executionMode string,
+    requestUser string,
+    queue string,
+    `user` string,
+    operationId string,
+    tablesWritten string,
+    tablesRead string,
+    otherInfo map<string, string>
+    )
+partitioned by (`date` string)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe'
+WITH SERDEPROPERTIES ('proto.class'='org.apache.hadoop.hive.ql.hooks.proto.HiveHookEvents$HiveHookEventProto', 'proto.maptypes'='org.apache.hadoop.hive.ql.hooks.proto.MapFieldEntry')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat'
+   OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveNullValueSequenceFileOutputFormat'
+LOCATION '${table["external_location"] | n}'
+TBLPROPERTIES ('proto.class'='org.apache.hadoop.hive.ql.hooks.proto.HiveHookEvents$HiveHookEventProto');

+ 16 - 0
apps/beeswax/src/beeswax/templates/msck.mako

@@ -0,0 +1,16 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+MSCK REPAIR TABLE sys.${table["name"]};

+ 54 - 0
apps/beeswax/src/beeswax/templates/select_table_query_data_from.mako

@@ -0,0 +1,54 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+SELECT
+  hiveQueryId,
+  `timestamp`,
+  executionMode,
+  requestUser,
+  queue,
+  otherInfo["QUERY"],
+  otherInfo["PERF"],
+  `date`,
+  eventType
+  FROM sys.${table["name"]}
+% if table["start_date"] or table["start_time"] or table["request_user"] or table["query_id"]:
+WHERE
+  % if table["request_user"]:
+  requestUser = "${table["request_user"]}"
+  % endif
+  % if table["start_date"]:
+    % if table["request_user"]:
+    and
+    % endif
+  `date` >= "${table["start_date"]}"
+  % endif
+  % if table["start_time"]:
+    % if table["request_user"] or table["start_date"]:
+    and
+    % endif
+  `timestamp` >= ${table["start_time"]}
+  % endif
+  % if table["query_id"]:
+    % if table["request_user"] or table["start_date"] or table["start_date"]:
+    and
+    % endif
+  `hiveQueryId` = "${table["query_id"]}"
+  % endif
+% endif
+% if table["limit"]:
+LIMIT ${table["limit"]}
+% endif
+;

+ 56 - 0
apps/beeswax/src/beeswax/templates/select_table_query_data_latest.mako

@@ -0,0 +1,56 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+select hiveQueryId, collect_list(`timestamp`) event_time, max(executionmode) as executionmode, max(requestUser) as requestUser, max(queue) as queue, max(otherInfo['QUERY']) as query, max(otherInfo['PERF']) as perf, max(`date`) as `date`, collect_list(eventType) as eventType
+% if table["force_refresh"]:
+, current_date()
+% endif
+from sys.${table["name"]}
+% if table["request_user"] or table["query_id"] or table["start_date"] or table["start_time"]:
+where
+  % if table["request_user"]:
+  requestUser = "${table["request_user"]}"
+  % endif
+  % if table["query_id"]:
+    % if table["request_user"]:
+      and
+    % endif
+    hiveQueryId = "${table["query_id"]}"
+  % endif
+  % if table["start_date"]:
+    % if table["request_user"] or table["query_id"]:
+      and
+    % endif
+    `date` >= "${table["start_date"]}"
+  % endif
+  % if table["start_time"]:
+    % if table["request_user"] or table["query_id"] or table["start_date"]:
+      and
+    % endif
+    `timestamp` >= ${table["start_time"]}
+  % endif
+% endif
+group by hivequeryId
+% if table["status"] == "completed":
+having count(`timestamp`) >= 2
+% endif
+% if table["status"] == "running":
+having count(`timestamp`) = 1
+% endif
+order by event_time[0] desc
+% if table["limit"]:
+limit ${table["limit"]}
+% endif
+;

+ 4 - 1
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -37,13 +37,16 @@ def get_api(user, interface, cluster=None):
   from jobbrowser.apis.livy_api import LivySessionsApi, LivyJobApi
   from jobbrowser.apis.job_api import JobApi
   from jobbrowser.apis.query_api import QueryApi
+  from jobbrowser.apis.beeswax_query_api import BeeswaxQueryApi
   from jobbrowser.apis.schedule_api import ScheduleApi
   from jobbrowser.apis.workflow_api import WorkflowApi
 
   if interface == 'jobs':
     return JobApi(user)
-  elif interface == 'queries':
+  elif interface == 'queries-impala':
     return QueryApi(user, cluster=cluster)
+  elif interface == 'queries-hive':
+    return BeeswaxQueryApi(user, cluster=cluster)
   elif interface == 'workflows':
     return WorkflowApi(user)
   elif interface == 'schedules':

+ 191 - 0
apps/jobbrowser/src/jobbrowser/apis/beeswax_query_api.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from builtins import filter
+
+import logging
+import re
+
+from datetime import datetime
+
+from desktop.lib.python_util import current_ms_from_utc
+
+from jobbrowser.apis.base_api import Api
+
+LOG = logging.getLogger(__name__)
+
+try:
+  from beeswax import query_history
+except Exception as e:
+  LOG.exception('Some application are not enabled: %s' % e)
+
+
+class BeeswaxQueryApi(Api):
+
+  def __init__(self, user, cluster=None):
+    self.user=user
+    self.cluster=cluster
+
+  def apps(self, filters):
+    filter_map = self._get_filter_map(filters)
+    limit = filters.get('pagination', {'limit': 25}).get('limit')
+    jobs = query_history.get_query_history(request_user=filter_map.get('effective_user'), start_date=filter_map.get('date'), start_time=filter_map.get('time'), query_id=filter_map.get('query_id'), status=filter_map.get('status'), limit=limit)
+
+    current_time = current_ms_from_utc()
+    apps = {
+      'apps': [{
+        'id': job[0],
+        'name': job[5]['queryText'].replace('\r\n', ' ')[:60] + ('...' if len(job[5]) > 60 else '') if job[5] else '',
+        'status': self._get_status(job),
+        'apiStatus': self._api_status(self._get_status(job)),
+        'type': job[2],
+        'user': job[3],
+        'queue': job[4],
+        'progress': '100' if len(job[1]) >= 2 else '',
+        'isRunning': len(job[1]) <= 1,
+        'canWrite': False,
+        'duration': job[1][-1] - job[1][0] if len(job[1]) > 1 else max(current_time - job[1][0], 0),
+        'submitted': job[1][0],
+        # Extra specific
+        'rows_fetched': 0,
+        'waiting': '',
+        'waiting_time': 0,
+        'properties': {
+        'plan': {
+            'stmt': job[5]['queryText'] if job[5] else '',
+            'plan': job[5]['queryPlan'] if job[5] else '',
+            'perf': job[6]
+          }
+        }
+      } for job in jobs['data']],
+      'total': 0
+    }
+
+    apps['total'] = len(apps['apps'])
+
+    return apps
+
+  def app(self, appid):
+    jobs = query_history.get_query_by_id(self.user.get_username(), query_id=appid)
+
+    current_time = current_ms_from_utc()
+    job = jobs['data'][0]
+    app = {
+      'id': job[0],
+      'name': job[5]['queryText'].replace('\r\n', ' ')[:60] + ('...' if len(job[5]) > 60 else '') if job[5] else '',
+      'status': self._get_status(job),
+      'apiStatus': self._api_status(self._get_status(job)),
+      'type': job[2],
+      'user': job[3],
+      'queue': job[4],
+      'progress': '100' if len(job[1]) >= 2 else '',
+      'isRunning': len(job[1]) <= 1,
+      'canWrite': False,
+      'duration': job[1][-1] - job[1][0] if len(job[1]) > 1 else max(current_time - job[1][0], 0),
+      'submitted': job[1][0],
+      # Extra specific
+      'rows_fetched': 0,
+      'waiting': '',
+      'waiting_time': 0,
+      'properties': {
+        'plan': {
+          'stmt': job[5]['queryText'] if job[5] else '',
+          'plan': job[5]['queryPlan'] if job[5] else '',
+          'perf': job[6]
+        }
+      }
+    }
+
+    return app
+
+  def action(self, appid, action):
+    message = {'message': '', 'status': 0}
+
+    return message;
+
+  def logs(self, appid, app_type, log_name=None, is_embeddable=False):
+    return {'logs': ''}
+
+  def profile(self, appid, app_type, app_property, app_filters):
+    message = {'message': '', 'status': 0}
+
+    return message;
+
+  def profile_encoded(self, appid):
+    message = {'message': '', 'status': 0}
+
+    return message;
+
+  def _get_status(self, job):
+    return 'RUNNING' if len(job[1]) <= 1 else "FINISHED"
+
+  def _api_status(self, status):
+    if status == 'FINISHED':
+      return 'SUCCEEDED'
+    elif status == 'EXCEPTION':
+      return 'FAILED'
+    elif status == 'RUNNING':
+      return 'RUNNING'
+    else:
+      return 'PAUSED'
+
+  def _get_filter_map(self, filters):
+    filter_map = {}
+    if filters.get("text"):
+      filter_names = {
+        'user':'effective_user',
+        'id':'query_id',
+        'name':'state',
+        'type':'stmt_type',
+        'status':'status'
+      }
+
+      def make_lambda(name, value):
+        return lambda app: app[name] == value
+
+      for key, name in list(filter_names.items()):
+          text_filter = re.search(r"\s*("+key+")\s*:([^ ]+)", filters.get("text"))
+          if text_filter and text_filter.group(1) == key:
+            filter_map[name] = text_filter.group(2).strip()
+
+    if filters.get("time"):
+      time_filter = filters.get("time")
+      period_ms = self._time_in_ms(float(time_filter.get("time_value")), time_filter.get("time_unit")[0:1])
+      ms_diff = current_ms_from_utc() - period_ms
+      filter_map["date"] = datetime.strftime(datetime.fromtimestamp(ms_diff / 1000), "%Y-%m-%d")
+      if time_filter.get("time_unit")[0:1] != 'd':
+        filter_map["time"] = int(ms_diff)
+    if filters.get("states"):
+      if len(filters.get("states")) == 1:
+        filter_map["status"] = filters.get("states")[0]
+
+    return filter_map
+
+  def _time_in_ms(self, time, period):
+    if period == 'ns':
+      return float(time) / 1000
+    elif period == 'ms':
+      return float(time)
+    elif period == 's':
+      return float(time) * 1000
+    elif period == 'm':
+      return float(time) * 60000 #1000*60
+    elif period == 'h':
+      return float(time) * 3600000 #1000*60*60
+    elif period == 'd':
+      return float(time) * 86400000  # 1000*60*60*24
+    else:
+      return float(time)

+ 53 - 9
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -206,7 +206,12 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           <td data-bind="text: user"></td>
           <td data-bind="text: type"></td>
           <td><span class="label job-status-label" data-bind="text: status"></span></td>
+          <!-- ko if: progress() !== '' -->
           <td data-bind="text: $root.formatProgress(progress)"></td>
+          <!-- /ko -->
+          <!-- ko if: progress() === '' -->
+          <td data-bind="text: ''"></td>
+          <!-- /ko -->
           <td data-bind="text: queue"></td>
           <td data-bind="moment: {data: submitted, format: 'LLL'}"></td>
           <td data-bind="text: duration().toHHMMSS()"></td>
@@ -319,7 +324,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
               <!-- ko if: interface() !== 'slas' && interface() !== 'oozie-info' -->
               <!-- ko if: !$root.job() -->
               <form class="form-inline">
-                <!-- ko if: !$root.isMini() && interface() == 'queries' -->
+                <!-- ko if: !$root.isMini() && interface() == 'queries-impala' -->
                   ${ _('Impala queries from') }
                 <!-- /ko -->
                 <!-- ko if: interface() != 'dataware2-clusters' && interface() != 'engines' -->
@@ -402,7 +407,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
                   <div class="jb-panel" data-bind="template: { name: 'job-page${ SUFFIX }' }"></div>
                 <!-- /ko -->
 
-                <!-- ko if: mainType() == 'queries' -->
+                <!-- ko if: mainType() == 'queries-impala' || mainType() == 'queries-hive' -->
                   <div class="jb-panel" data-bind="template: { name: 'queries-page${ SUFFIX }' }"></div>
                 <!-- /ko -->
 
@@ -1509,7 +1514,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           <!-- ko if: doc_url -->
           <li class="nav-header">${ _('Id') }</li>
           <li>
-            <a data-bind="attr: { href: doc_url_modified }" target="_blank" title="${ _('Open in impalad') }">
+            <a data-bind="attr: { href: doc_url_modified }" target="_blank" title="${ _('Open in Impalad') }">
               <span data-bind="text: id"></span>
             </a>
             <!-- ko if: $root.isMini() -->
@@ -1555,6 +1560,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
     <div data-bind="css:{'span10': !$root.isMini(), 'span12 no-margin': $root.isMini() }">
       <ul class="nav nav-pills margin-top-20">
+        <!-- ko if: $root.job().mainType() == 'queries-impala' -->
         <li>
           <a href="#queries-page-plan${ SUFFIX }" data-bind="click: function(){ $('a[href=\'#queries-page-plan${ SUFFIX }\']').tab('show'); }, event: {'shown': function () { if (!properties.plan || !properties.plan()) { fetchProfile('plan'); } } }">
             ${ _('Plan') }</a>
@@ -1587,11 +1593,27 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           <a href="#queries-page-finstances${ SUFFIX }" data-bind="click: function(){ $('a[href=\'#queries-page-finstances${ SUFFIX }\']').tab('show'); }, event: {'shown': function () { if (!properties.finstances || !properties.finstances().backend_instances) { fetchProfile('finstances'); } } }">
             ${ _('Instances') }</a>
         </li>
+        <!-- /ko -->
+        <!-- ko if: $root.job().mainType() == 'queries-hive' -->
+        <li class="active">
+          <a href="#queries-page-hive-plan-text${ SUFFIX }" data-bind="click: function(){ $('a[href=\'#queries-page-hive-plan-text${ SUFFIX }\']').tab('show'); }">
+            ${ _('Plan') }</a>
+        </li>
+        <li>
+          <a href="#queries-page-hive-stmt${ SUFFIX }" data-bind="click: function(){ $('a[href=\'#queries-page-hive-stmt${ SUFFIX }\']').tab('show'); }">
+            ${ _('Query') }</a>
+        </li>
+        <li>
+          <a href="#queries-page-hive-perf${ SUFFIX }" data-bind="click: function(){ $('a[href=\'#queries-page-hive-perf${ SUFFIX }\']').tab('show'); }">
+            ${ _('Perf') }</a>
+        </li>
+        <!-- /ko -->
       </ul>
 
       <div class="clearfix"></div>
 
       <div class="tab-content">
+        <!-- ko if: $root.job().mainType() == 'queries-impala' -->
         <div class="tab-pane" id="queries-page-plan${ SUFFIX }" data-profile="plan">
           <div data-bind="visible:properties.plan && properties.plan().plan_json && properties.plan().plan_json.plan_nodes.length">
             <div class="query-plan" id="queries-page-plan-graph${ SUFFIX }" data-bind="impalaDagre: { value: properties.plan && properties.plan(), height:$root.isMini() ? 535 : 600 }">
@@ -1667,6 +1689,19 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           <pre data-bind="text: _('The selected tab has no data')"/>
           <!-- /ko -->
         </div>
+        <!-- /ko -->
+
+        <!-- ko if: $root.job().mainType() == 'queries-hive' -->
+        <div class="tab-pane active" id="queries-page-hive-plan-text${ SUFFIX }" data-profile="plan">
+          <pre data-bind="text: (properties.plan && properties.plan().plan && JSON.stringify(ko.toJS(properties.plan().plan), null, 2)) || _('The selected tab has no data')"/>
+        </div>
+        <div class="tab-pane" id="queries-page-hive-stmt${ SUFFIX }" data-profile="stmt">
+          <pre data-bind="text: (properties.plan && properties.plan().stmt) || _('The selected tab has no data')"/>
+        </div>
+        <div class="tab-pane" id="queries-page-hive-perf${ SUFFIX }" data-profile="perf">
+          <pre data-bind="text: (properties.plan && properties.plan().perf && properties.plan().perf && JSON.stringify(JSON.parse(properties.plan().perf), null, 2)) || _('The selected tab has no data')"/>
+        </div>
+        <!-- /ko -->
       </div>
     </div>
     <!-- /ko -->
@@ -2554,7 +2589,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         return self.logsByName()[self.logActive()];
       });
 
-      self.properties = ko.mapping.fromJS(job.properties || { properties: '' });
+      self.properties = ko.mapping.fromJS(job.properties && Object.keys(job.properties).reduce(function(p, key) { p[key] = ''; return p;}, {}) || {'properties': ''});
+      Object.keys(job.properties || []).reduce(function(p, key) { p[key](job.properties[key]); return p;}, self.properties);
       self.mainType = ko.observable(vm.interface());
       self.lastEvent = ko.observable(job.lastEvent || '');
 
@@ -2751,7 +2787,10 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           interface = 'dataware-clusters';
         }
         else if (/[a-z0-9]{16}:[a-z0-9]{16}/.test(self.id())) {
-          interface = 'queries';
+          interface = 'queries-impala';
+        }
+        else if (/hive_[a-z0-9]*_[a-z0-9]*/.test(self.id())) {
+          interface = 'queries-hive';
         }
         else if (/livy-[0-9]+/.test(self.id())) {
           interface = 'livy-sessions';
@@ -2813,7 +2852,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
             vm.resetBreadcrumbs(crumbs);
             // Show is still bound to old job, setTimeout allows knockout model change event done at begining of this method to sends it's notification
             setTimeout(function () {
-              if (vm.job().type() === 'queries' && !$("#queries-page-plan${ SUFFIX }").parent().children().hasClass("active")) {
+              if (vm.job().mainType() === 'queries-impala' && !$("#queries-page-plan${ SUFFIX }").parent().children().hasClass("active")) {
                 $("a[href=\'#queries-page-plan${ SUFFIX }\']").tab("show");
               }
             }, 0);
@@ -3179,7 +3218,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           'workflows',
           'schedules',
           'bundles',
-          'queries',
+          'queries-impala',
           'dataeng-jobs',
           'dataeng-clusters',
           'dataware-clusters',
@@ -3552,6 +3591,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         var queryInterfaceCondition = function () {
           return '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('impala') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
         };
+        var queryHiveInterfaceCondition = function () {
+          return '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('hive') != -1 && (!self.cluster() || self.cluster()['type'].indexOf('altus') == -1);
+        };
 
         var interfaces = [
           {'interface': 'jobs', 'label': '${ _ko('Jobs') }', 'condition': jobsInterfaceCondition},
@@ -3560,7 +3602,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           {'interface': 'dataware-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataWarehouseInterfaceCondition},
           {'interface': 'dataware2-clusters', 'label': '${ _ko('Warehouses') }', 'condition': dataWarehouse2InterfaceCondition},
           {'interface': 'engines', 'label': '${ _ko('') }', 'condition': enginesInterfaceCondition},
-          {'interface': 'queries', 'label': '${ _ko('Queries') }', 'condition': queryInterfaceCondition},
+          {'interface': 'queries-impala', 'label': '${ _ko('Impala') }', 'condition': queryInterfaceCondition},
+          {'interface': 'queries-hive', 'label': '${ _ko('Hive') }', 'condition': queryHiveInterfaceCondition},
           {'interface': 'celery-beat', 'label': '${ _ko('Scheduled Tasks') }', 'condition': schedulerBeatInterfaceCondition},
           {'interface': 'workflows', 'label': '${ _ko('Workflows') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'schedules', 'label': '${ _ko('Schedules') }', 'condition': schedulerInterfaceCondition},
@@ -3749,7 +3792,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           case 'slas':
           case 'oozie-info':
           case 'jobs':
-          case 'queries':
+          case 'queries-impala':
+          case 'queries-hive':
           case 'celery-beat':
           case 'workflows':
           case 'schedules':

+ 6 - 3
desktop/core/src/desktop/lib/python_util.py

@@ -14,11 +14,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# 
+#
 # Extra python utils
 
 from past.builtins import basestring
 from builtins import object
+import datetime
 import select
 import socket
 import sys
@@ -155,7 +156,6 @@ def force_dict_to_strings(dictionary):
 
   return new_dict
 
-
 def isASCII(data):
   try:
     data.decode('ASCII')
@@ -216,4 +216,7 @@ def check_encoding(data):
     elif isGB2312(data):
       return 'gb2312'
     else:
-      return 'cp1252'
+      return 'cp1252'
+
+def current_ms_from_utc():
+  return (datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000