Эх сурвалжийг харах

[spark] Add a close session call

There is an error coming back from Livy so not totally finished
but most of the logic is there.
Romain Rigaux 10 жил өмнө
parent
commit
8bd15fd

+ 16 - 2
apps/spark/src/spark/api.py

@@ -54,6 +54,20 @@ def create_session(request):
   return JsonResponse(response)
 
 
+@require_POST
+@check_document_access_permission()
+@api_error_handler
+def close_session(request):
+  response = {'status': -1}
+
+  session = json.loads(request.POST.get('session', '{}'))
+
+  response['session'] = get_api(request.user, {'type': session['type']}).close_session(session=session)
+  response['status'] = 0
+
+  return JsonResponse(response)
+
+
 @require_POST
 @check_document_access_permission()
 @api_error_handler
@@ -207,9 +221,9 @@ def close_notebook(request):
 
   notebook = json.loads(request.POST.get('notebook', '{}'))
 
-  for snippet in notebook['snippets']:
+  for session in notebook['sessions']:
     try:
-      response['result'].append(get_api(request.user, snippet).close(notebook, snippet))
+      response['result'].append(get_api(request.user, session).close_session(session))
     except QueryExpired:
       pass
     except Exception, e:

+ 21 - 19
apps/spark/src/spark/models.py

@@ -102,7 +102,7 @@ def _get_snippet_session(notebook, snippet):
   return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0]
 
 
-class TextApi():
+class Api(object):
 
   def __init__(self, user):
     self.user = user
@@ -113,6 +113,16 @@ class TextApi():
         'id': None
     }
 
+  def close_session(self, session):
+    pass
+
+
+# Text
+
+class TextApi(Api):
+
+  pass
+
 
 # HS2
 
@@ -129,10 +139,7 @@ def query_error_handler(func):
   return decorator
 
 
-class HS2Api():
-
-  def __init__(self, user):
-    self.user = user
+class HS2Api(Api):
 
   def _get_handle(self, snippet):
     snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
@@ -277,15 +284,11 @@ class HS2Api():
 
 # Spark
 
-
-class SparkApi():
-
-  def __init__(self, user):
-    self.user = user
+class SparkApi(Api):
 
   def create_session(self, lang='scala', properties=None):
     if properties is None:
-      settings = { 
+      properties = {
           'executor_cores': 1, # Some props only in YARN mode
           'executor_count': 1,
           'executor_memory': '1G',
@@ -294,7 +297,7 @@ class SparkApi():
       }
 
     api = get_spark_api(self.user)
-    print 'TODO: we should use the settings %s for creating the new sessions' % settings
+    print 'TODO: we should use the settings %s for creating the new sessions' % properties
     response = api.create_session(kind=lang)
 
     status = api.get_session(response['id'])
@@ -311,7 +314,7 @@ class SparkApi():
     return {
         'type': lang,
         'id': response['id'],
-        'properties': settings
+        'properties': properties
     }
 
   def execute(self, notebook, snippet):
@@ -425,9 +428,11 @@ class SparkApi():
   def _progress(self, snippet, logs):
     return 50
 
-  def close(self, notebook, snippet):
+  def close(self, notebook, snippet): # Individual statements cannot be closed
+    pass
+
+  def close_session(self, session):
     api = get_spark_api(self.user)
-    session = _get_snippet_session(notebook, snippet)
 
     if session['id'] is not None:
       api.close(session['id'])
@@ -442,10 +447,7 @@ class SparkApi():
     return []
 
 
-class SparkBatchApi():
-
-  def __init__(self, user):
-    self.user = user
+class SparkBatchApi(Api):
 
   def create_session(self, lang, properties=None):
     return {

+ 23 - 3
apps/spark/src/spark/static/spark/js/spark.ko.js

@@ -85,7 +85,7 @@ var Result = function (snippet, result) {
         handle: self.handle
     };
   }
-  
+
   if (typeof result.handle != "undefined" && result.handle != null) {
     $.each(result.handle, function (key, val) {
       self.handle()[key] = val;
@@ -313,7 +313,7 @@ var Snippet = function (vm, notebook, snippet) {
       result: self.result.getContext()
     };
   }
-  
+
   self._ajax_error = function (data, callback) {
     if (data.status == -2) {
       self.create_session(callback);
@@ -604,6 +604,9 @@ var Notebook = function (vm, notebook) {
           snippet.status(status);
         });
       };
+
+      self.closeSession(session);
+
       setSnippetStatus('loading');
       var successCallback = function() {
         setSnippetStatus('ready');
@@ -683,6 +686,8 @@ var Notebook = function (vm, notebook) {
     else {
       _snippet.status('ready');
     }
+
+    logGA('/add_snippet/' + self.selectedSnippet());
     $(document).trigger("snippetAdded", _snippet);
   };
 
@@ -732,6 +737,21 @@ var Notebook = function (vm, notebook) {
       snippet.status('ready');
     });
   };
+
+  self.closeSession = function (session) {
+    $.post("/spark/api/close_session", {
+      session: ko.mapping.toJSON(session)
+    }, function (data) {
+      if (data.status == 0) {
+        notebook.sessions.remove(session);
+      }
+      else {
+        $(document).trigger("error", data.message);
+      }
+    }).fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
 }
 
 
@@ -821,6 +841,6 @@ function EditorViewModel(notebooks, options) {
 
 function logGA(page) {
   if (typeof trackOnGA == 'function') {
-    trackOnGA('editor/' + page);
+    trackOnGA('notebook/' + page);
   }
 }

+ 5 - 2
apps/spark/src/spark/templates/editor_components.mako

@@ -819,8 +819,11 @@ from django.utils.translation import ugettext as _
               </div>
             </div>
             <!-- /ko -->
-            <a style="float: right;" class="btn pointer" title="${ _('Restart Session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().restartSession($parent) }">
-              <i class="fa fa-refresh"></i> ${ _('Session') }
+            <a style="float: right;" class="btn pointer" title="${ _('Restart session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().restartSession($parent) }">
+              <i class="fa fa-refresh"></i> ${ _('Recreate') }
+            </a>
+            <a style="float: right;" class="btn pointer" title="${ _('Close session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().closeSession($parent) }">
+              <i class="fa fa-times"></i> ${ _('Close') }
             </a>
             <!-- /ko -->
             <!-- /ko -->

+ 1 - 0
apps/spark/src/spark/urls.py

@@ -44,6 +44,7 @@ urlpatterns = patterns('spark.views',
 # APIs
 urlpatterns += patterns('spark.api',
   url(r'^api/create_session$', 'create_session', name='create_session'),
+  url(r'^api/close_session$', 'close_session', name='close_session'),
   url(r'^api/execute$', 'execute', name='execute'),
   url(r'^api/check_status$', 'check_status', name='check_status'),
   url(r'^api/fetch_result_data$', 'fetch_result_data', name='fetch_result_data'),

+ 0 - 2
desktop/core/src/desktop/api.py

@@ -21,8 +21,6 @@ import time
 
 from collections import defaultdict
 
-from django.core.urlresolvers import reverse
-
 from django.utils import html
 from django.utils.translation import ugettext as _