Browse Source

[spark] Add a close session call

There is an error coming back from Livy so not totally finished
but most of the logic is there.
Romain Rigaux 10 năm trước cách đây
mục cha
commit
8bd15fd9a2

+ 16 - 2
apps/spark/src/spark/api.py

@@ -54,6 +54,20 @@ def create_session(request):
   return JsonResponse(response)
 
 
+@require_POST
+@check_document_access_permission()
+@api_error_handler
+def close_session(request):
+  response = {'status': -1}
+
+  session = json.loads(request.POST.get('session', '{}'))
+
+  response['session'] = get_api(request.user, {'type': session['type']}).close_session(session=session)
+  response['status'] = 0
+
+  return JsonResponse(response)
+
+
 @require_POST
 @check_document_access_permission()
 @api_error_handler
@@ -207,9 +221,9 @@ def close_notebook(request):
 
   notebook = json.loads(request.POST.get('notebook', '{}'))
 
-  for snippet in notebook['snippets']:
+  for session in notebook['sessions']:
     try:
-      response['result'].append(get_api(request.user, snippet).close(notebook, snippet))
+      response['result'].append(get_api(request.user, session).close_session(session))
     except QueryExpired:
       pass
     except Exception, e:

+ 21 - 19
apps/spark/src/spark/models.py

@@ -102,7 +102,7 @@ def _get_snippet_session(notebook, snippet):
   return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0]
 
 
-class TextApi():
+class Api(object):
 
   def __init__(self, user):
     self.user = user
@@ -113,6 +113,16 @@ class TextApi():
         'id': None
     }
 
+  def close_session(self, session):
+    pass
+
+
+# Text
+
+class TextApi(Api):
+
+  pass
+
 
 # HS2
 
@@ -129,10 +139,7 @@ def query_error_handler(func):
   return decorator
 
 
-class HS2Api():
-
-  def __init__(self, user):
-    self.user = user
+class HS2Api(Api):
 
   def _get_handle(self, snippet):
     snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
@@ -277,15 +284,11 @@ class HS2Api():
 
 # Spark
 
-
-class SparkApi():
-
-  def __init__(self, user):
-    self.user = user
+class SparkApi(Api):
 
   def create_session(self, lang='scala', properties=None):
     if properties is None:
-      settings = { 
+      properties = {
           'executor_cores': 1, # Some props only in YARN mode
           'executor_count': 1,
           'executor_memory': '1G',
@@ -294,7 +297,7 @@ class SparkApi():
       }
 
     api = get_spark_api(self.user)
-    print 'TODO: we should use the settings %s for creating the new sessions' % settings
+    print 'TODO: we should use the settings %s for creating the new sessions' % properties
     response = api.create_session(kind=lang)
 
     status = api.get_session(response['id'])
@@ -311,7 +314,7 @@ class SparkApi():
     return {
         'type': lang,
         'id': response['id'],
-        'properties': settings
+        'properties': properties
     }
 
   def execute(self, notebook, snippet):
@@ -425,9 +428,11 @@ class SparkApi():
   def _progress(self, snippet, logs):
     return 50
 
-  def close(self, notebook, snippet):
+  def close(self, notebook, snippet): # Individual statements cannot be closed
+    pass
+
+  def close_session(self, session):
     api = get_spark_api(self.user)
-    session = _get_snippet_session(notebook, snippet)
 
     if session['id'] is not None:
       api.close(session['id'])
@@ -442,10 +447,7 @@ class SparkApi():
     return []
 
 
-class SparkBatchApi():
-
-  def __init__(self, user):
-    self.user = user
+class SparkBatchApi(Api):
 
   def create_session(self, lang, properties=None):
     return {

+ 23 - 3
apps/spark/src/spark/static/spark/js/spark.ko.js

@@ -85,7 +85,7 @@ var Result = function (snippet, result) {
         handle: self.handle
     };
   }
-  
+
   if (typeof result.handle != "undefined" && result.handle != null) {
     $.each(result.handle, function (key, val) {
       self.handle()[key] = val;
@@ -313,7 +313,7 @@ var Snippet = function (vm, notebook, snippet) {
       result: self.result.getContext()
     };
   }
-  
+
   self._ajax_error = function (data, callback) {
     if (data.status == -2) {
       self.create_session(callback);
@@ -604,6 +604,9 @@ var Notebook = function (vm, notebook) {
           snippet.status(status);
         });
       };
+
+      self.closeSession(session);
+
       setSnippetStatus('loading');
       var successCallback = function() {
         setSnippetStatus('ready');
@@ -683,6 +686,8 @@ var Notebook = function (vm, notebook) {
     else {
       _snippet.status('ready');
     }
+
+    logGA('/add_snippet/' + self.selectedSnippet());
     $(document).trigger("snippetAdded", _snippet);
   };
 
@@ -732,6 +737,21 @@ var Notebook = function (vm, notebook) {
       snippet.status('ready');
     });
   };
+
+  self.closeSession = function (session) {
+    $.post("/spark/api/close_session", {
+      session: ko.mapping.toJSON(session)
+    }, function (data) {
+      if (data.status == 0) {
+        notebook.sessions.remove(session);
+      }
+      else {
+        $(document).trigger("error", data.message);
+      }
+    }).fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
 }
 
 
@@ -821,6 +841,6 @@ function EditorViewModel(notebooks, options) {
 
 function logGA(page) {
   if (typeof trackOnGA == 'function') {
-    trackOnGA('editor/' + page);
+    trackOnGA('notebook/' + page);
   }
 }

+ 5 - 2
apps/spark/src/spark/templates/editor_components.mako

@@ -819,8 +819,11 @@ from django.utils.translation import ugettext as _
               </div>
             </div>
             <!-- /ko -->
-            <a style="float: right;" class="btn pointer" title="${ _('Restart Session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().restartSession($parent) }">
-              <i class="fa fa-refresh"></i> ${ _('Session') }
+            <a style="float: right;" class="btn pointer" title="${ _('Restart session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().restartSession($parent) }">
+              <i class="fa fa-refresh"></i> ${ _('Recreate') }
+            </a>
+            <a style="float: right;" class="btn pointer" title="${ _('Close session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().closeSession($parent) }">
+              <i class="fa fa-times"></i> ${ _('Close') }
             </a>
             <!-- /ko -->
             <!-- /ko -->

+ 1 - 0
apps/spark/src/spark/urls.py

@@ -44,6 +44,7 @@ urlpatterns = patterns('spark.views',
 # APIs
 urlpatterns += patterns('spark.api',
   url(r'^api/create_session$', 'create_session', name='create_session'),
+  url(r'^api/close_session$', 'close_session', name='close_session'),
   url(r'^api/execute$', 'execute', name='execute'),
   url(r'^api/check_status$', 'check_status', name='check_status'),
   url(r'^api/fetch_result_data$', 'fetch_result_data', name='fetch_result_data'),

+ 0 - 2
desktop/core/src/desktop/api.py

@@ -21,8 +21,6 @@ import time
 
 from collections import defaultdict
 
-from django.core.urlresolvers import reverse
-
 from django.utils import html
 from django.utils.translation import ugettext as _