Преглед изворни кода

[beeswax] Get and save configuration values for beeswax sessions

Returns full properties in /beeswax/api/session
Jenny Kim пре 10 година
родитељ
комит
fb90f9d

+ 12 - 10
apps/beeswax/src/beeswax/api.py

@@ -245,6 +245,7 @@ def watch_query_refresh_json(request, id):
 
   return JsonResponse(result)
 
+
 def massage_job_urls_for_json(jobs):
   massaged_jobs = []
   for job in jobs:
@@ -255,6 +256,7 @@ def massage_job_urls_for_json(jobs):
   return massaged_jobs
 
 
+@error_handler
 def close_operation(request, query_history_id):
   response = {
     'status': -1,
@@ -264,16 +266,13 @@ def close_operation(request, query_history_id):
   if request.method != 'POST':
     response['message'] = _('A POST request is required.')
   else:
-    try:
-      query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
-      db = dbms.get(query_history.owner, query_history.get_query_server_config())
-      handle = query_history.get_handle()
-      db.close_operation(handle)
-      query_history.set_to_expired()
-      query_history.save()
-      response['status'] = 0
-    except Exception, e:
-      response['message'] = unicode(e)
+    query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
+    db = dbms.get(query_history.owner, query_history.get_query_server_config())
+    handle = query_history.get_handle()
+    db.close_operation(handle)
+    query_history.set_to_expired()
+    query_history.save()
+    response['status'] = 0
 
   return JsonResponse(response)
 
@@ -404,6 +403,7 @@ def fetch_saved_design(request, design_id):
   response['design'] = design_to_dict(design)
   return JsonResponse(response)
 
+
 @error_handler
 def fetch_query_history(request, query_history_id):
   response = {'status': 0, 'message': ''}
@@ -416,6 +416,7 @@ def fetch_query_history(request, query_history_id):
   response['query_history'] = query_history_to_dict(request, query)
   return JsonResponse(response)
 
+
 @error_handler
 def cancel_query(request, query_history_id):
   response = {'status': -1, 'message': ''}
@@ -800,6 +801,7 @@ def get_top_terms(request, database, table, column, prefix=None):
   return JsonResponse(response)
 
 
+@error_handler
 def get_session(request):
   app_name = get_app_name(request)
   query_server = get_query_server_config(app_name)

+ 4 - 0
apps/beeswax/src/beeswax/models.py

@@ -413,6 +413,10 @@ class Session(models.Model):
     handle_id = THandleIdentifier(secret=secret, guid=guid)
     return TSessionHandle(sessionId=handle_id)
 
+  def get_properties(self):
+    props_dict = json.loads(self.properties) if self.properties is not None else None
+    return props_dict
+
   def __str__(self):
     return '%s %s' % (self.owner, self.last_used)
 

+ 27 - 7
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -595,13 +595,20 @@ class HiveServerClient:
     encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
     properties = json.dumps(res.configuration)
 
-    return Session.objects.create(owner=user,
-                                  application=self.query_server['server_name'],
-                                  status_code=res.status.statusCode,
-                                  secret=encoded_status,
-                                  guid=encoded_guid,
-                                  server_protocol_version=res.serverProtocolVersion,
-                                  properties=properties)
+    session = Session.objects.create(owner=user,
+                                     application=self.query_server['server_name'],
+                                     status_code=res.status.statusCode,
+                                     secret=encoded_status,
+                                     guid=encoded_guid,
+                                     server_protocol_version=res.serverProtocolVersion,
+                                     properties=properties)
+
+    # HS2 does not return properties in TOpenSessionResp
+    if self.query_server['server_name'] == "beeswax" and not session.get_properties():
+      session.properties = json.dumps(self.get_configuration(include_hadoop=False))
+      session.save()
+
+    return session
 
 
   def call(self, fn, req, status=TStatusCode.SUCCESS_STATUS):
@@ -867,6 +874,19 @@ class HiveServerClient:
     return partitions[:max_parts]
 
 
+  def get_configuration(self, include_hadoop=False):
+    configuration = {}
+    query = 'SET'
+    if include_hadoop:
+      query += ' -v'
+
+    results = self.execute_query_statement(query)
+    if results:
+      rows = [row[0] for row in results.rows()]
+      configuration = dict((row.split('=')[0], row.split('=')[1]) for row in rows if '=' in row)
+    return configuration
+
+
   def _get_query_configuration(self, query):
     return dict([(setting['key'], setting['value']) for setting in query.settings])
 

+ 18 - 21
apps/beeswax/src/beeswax/urls.py

@@ -35,7 +35,7 @@ urlpatterns = patterns('beeswax.views',
   url(r'^clone_design/(?P<design_id>\d+)$', 'clone_design', name='clone_design'),
   url(r'^query_history$', 'list_query_history', name='list_query_history'),
 
-  url(r'^configuration$', 'configuration', name='configuration'),
+  url(r'^configuration/?$', 'configuration', name='configuration'),
   url(r'^install_examples$', 'install_examples', name='install_examples'),
   url(r'^query_cb/done/(?P<server_id>\S+)$', 'query_done_cb', name='query_done_cb'),
 )
@@ -58,16 +58,14 @@ urlpatterns += patterns(
   'beeswax.api',
 
   url(r'^api/session/$', 'get_session', name='api_get_session'),
+  url(r'^api/functions/?$', 'get_functions', name='get_functions'),
 
   # Deprecated by Notebook API
-  url(r'^api/autocomplete/$', 'autocomplete', name='api_autocomplete_databases'),
-  url(r'^api/autocomplete/(?P<database>\w+)$', 'autocomplete', name='api_autocomplete_tables'),
-  url(r'^api/autocomplete/(?P<database>\w+)/$', 'autocomplete', name='api_autocomplete_tables'),
-  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)$', 'autocomplete', name='api_autocomplete_columns'),
-  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/$', 'autocomplete', name='api_autocomplete_columns'),
-  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)$', 'autocomplete', name='api_autocomplete_column'),
-  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/$', 'autocomplete', name='api_autocomplete_column'),
-  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/(?P<nested>.+)$', 'autocomplete', name='api_autocomplete_nested'),
+  url(r'^api/autocomplete/?$', 'autocomplete', name='api_autocomplete_databases'),
+  url(r'^api/autocomplete/(?P<database>\w+/?)$', 'autocomplete', name='api_autocomplete_tables'),
+  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/?$', 'autocomplete', name='api_autocomplete_columns'),
+  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/?$', 'autocomplete', name='api_autocomplete_column'),
+  url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/(?P<nested>.+)/?$', 'autocomplete', name='api_autocomplete_nested'),
 
   url(r'^api/design/(?P<design_id>\d+)?$', 'save_query_design', name='api_save_design'),
   url(r'^api/design/(?P<design_id>\d+)/get$', 'fetch_saved_design', name='api_fetch_saved_design'),
@@ -75,21 +73,20 @@ urlpatterns += patterns(
   url(r'^api/query/(?P<query_history_id>\d+)/get$', 'fetch_query_history', name='api_fetch_query_history'),
   url(r'^api/query/parameters$', 'parameters', name='api_parameters'),
   url(r'^api/query/execute/(?P<design_id>\d+)?$', 'execute', name='api_execute'),
-  url(r'^api/query/(?P<query_history_id>\d+)/cancel$', 'cancel_query', name='api_cancel_query'),
+  url(r'^api/query/(?P<query_history_id>\d+)/cancel/?$', 'cancel_query', name='api_cancel_query'),
   url(r'^api/query/(?P<query_history_id>\d+)/close/?$', 'close_operation', name='api_close_operation'),
-  url(r'^api/query/(?P<query_history_id>\d+)/results/save/hive/table$', 'save_results_hive_table', name='api_save_results_hive_table'),
-  url(r'^api/query/(?P<query_history_id>\d+)/results/save/hdfs/file$', 'save_results_hdfs_file', name='api_save_results_hdfs_file'),
-  url(r'^api/query/(?P<query_history_id>\d+)/results/save/hdfs/directory$', 'save_results_hdfs_directory', name='api_save_results_hdfs_directory'),
-  url(r'^api/watch/json/(?P<id>\d+)$', 'watch_query_refresh_json', name='api_watch_query_refresh_json'),
+  url(r'^api/query/(?P<query_history_id>\d+)/results/save/hive/table/?$', 'save_results_hive_table', name='api_save_results_hive_table'),
+  url(r'^api/query/(?P<query_history_id>\d+)/results/save/hdfs/file/?$', 'save_results_hdfs_file', name='api_save_results_hdfs_file'),
+  url(r'^api/query/(?P<query_history_id>\d+)/results/save/hdfs/directory/?$', 'save_results_hdfs_directory', name='api_save_results_hdfs_directory'),
+  url(r'^api/watch/json/(?P<id>\d+)/?$', 'watch_query_refresh_json', name='api_watch_query_refresh_json'),
 
-  url(r'^api/query/clear_history/$', 'clear_history', name='clear_history'),
+  url(r'^api/query/clear_history/?$', 'clear_history', name='clear_history'),
 
-  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)$', 'describe_table', name='describe_table'),
-  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/sample$', 'get_sample_data', name='get_sample_data'),
-  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/indexes$', 'get_indexes', name='get_indexes'),
-  url(r'^api/functions$', 'get_functions', name='get_functions'),
-
-  url(r'^api/analyze/(?P<database>\w+)/(?P<table>\w+)/(?P<columns>\w+)?$', 'analyze_table', name='analyze_table'),
+  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/?$', 'describe_table', name='describe_table'),
+  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/indexes/?$', 'get_indexes', name='get_indexes'),
+  url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/sample/?$', 'get_sample_data', name='get_sample_data'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/stats/(?P<column>\w+)?$', 'get_table_stats', name='get_table_stats'),
   url(r'^api/table/(?P<database>\w+)/(?P<table>\w+)/terms/(?P<column>\w+)/(?P<prefix>\w+)?$', 'get_top_terms', name='get_top_terms'),
+
+  url(r'^api/analyze/(?P<database>\w+)/(?P<table>\w+)/(?P<columns>\w+)?$', 'analyze_table', name='analyze_table'),
 )

+ 0 - 1
apps/impala/src/impala/server.py

@@ -19,7 +19,6 @@ import logging
 
 from beeswax.server.dbms import QueryServerException
 from beeswax.server.hive_server2_lib import HiveServerClient, HiveServerDataTable
-from TCLIService.ttypes import TExecuteStatementReq, TFetchOrientation, TStatusCode
 
 from ImpalaService import ImpalaHiveServer2Service