Browse Source

[spark] Add a close session call

There is an error coming back from Livy so not totally finished
but most of the logic is there.
Romain Rigaux 10 năm trước cách đây
mục cha
commit
8bd15fd

+ 16 - 2
apps/spark/src/spark/api.py

@@ -54,6 +54,20 @@ def create_session(request):
   return JsonResponse(response)
   return JsonResponse(response)
 
 
 
 
+@require_POST
+@check_document_access_permission()
+@api_error_handler
+def close_session(request):
+  response = {'status': -1}
+
+  session = json.loads(request.POST.get('session', '{}'))
+
+  response['session'] = get_api(request.user, {'type': session['type']}).close_session(session=session)
+  response['status'] = 0
+
+  return JsonResponse(response)
+
+
 @require_POST
 @require_POST
 @check_document_access_permission()
 @check_document_access_permission()
 @api_error_handler
 @api_error_handler
@@ -207,9 +221,9 @@ def close_notebook(request):
 
 
   notebook = json.loads(request.POST.get('notebook', '{}'))
   notebook = json.loads(request.POST.get('notebook', '{}'))
 
 
-  for snippet in notebook['snippets']:
+  for session in notebook['sessions']:
     try:
     try:
-      response['result'].append(get_api(request.user, snippet).close(notebook, snippet))
+      response['result'].append(get_api(request.user, session).close_session(session))
     except QueryExpired:
     except QueryExpired:
       pass
       pass
     except Exception, e:
     except Exception, e:

+ 21 - 19
apps/spark/src/spark/models.py

@@ -102,7 +102,7 @@ def _get_snippet_session(notebook, snippet):
   return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0]
   return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0]
 
 
 
 
-class TextApi():
+class Api(object):
 
 
   def __init__(self, user):
   def __init__(self, user):
     self.user = user
     self.user = user
@@ -113,6 +113,16 @@ class TextApi():
         'id': None
         'id': None
     }
     }
 
 
+  def close_session(self, session):
+    pass
+
+
+# Text
+
+class TextApi(Api):
+
+  pass
+
 
 
 # HS2
 # HS2
 
 
@@ -129,10 +139,7 @@ def query_error_handler(func):
   return decorator
   return decorator
 
 
 
 
-class HS2Api():
-
-  def __init__(self, user):
-    self.user = user
+class HS2Api(Api):
 
 
   def _get_handle(self, snippet):
   def _get_handle(self, snippet):
     snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
     snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
@@ -277,15 +284,11 @@ class HS2Api():
 
 
 # Spark
 # Spark
 
 
-
-class SparkApi():
-
-  def __init__(self, user):
-    self.user = user
+class SparkApi(Api):
 
 
   def create_session(self, lang='scala', properties=None):
   def create_session(self, lang='scala', properties=None):
     if properties is None:
     if properties is None:
-      settings = { 
+      properties = {
           'executor_cores': 1, # Some props only in YARN mode
           'executor_cores': 1, # Some props only in YARN mode
           'executor_count': 1,
           'executor_count': 1,
           'executor_memory': '1G',
           'executor_memory': '1G',
@@ -294,7 +297,7 @@ class SparkApi():
       }
       }
 
 
     api = get_spark_api(self.user)
     api = get_spark_api(self.user)
-    print 'TODO: we should use the settings %s for creating the new sessions' % settings
+    print 'TODO: we should use the settings %s for creating the new sessions' % properties
     response = api.create_session(kind=lang)
     response = api.create_session(kind=lang)
 
 
     status = api.get_session(response['id'])
     status = api.get_session(response['id'])
@@ -311,7 +314,7 @@ class SparkApi():
     return {
     return {
         'type': lang,
         'type': lang,
         'id': response['id'],
         'id': response['id'],
-        'properties': settings
+        'properties': properties
     }
     }
 
 
   def execute(self, notebook, snippet):
   def execute(self, notebook, snippet):
@@ -425,9 +428,11 @@ class SparkApi():
   def _progress(self, snippet, logs):
   def _progress(self, snippet, logs):
     return 50
     return 50
 
 
-  def close(self, notebook, snippet):
+  def close(self, notebook, snippet): # Individual statements cannot be closed
+    pass
+
+  def close_session(self, session):
     api = get_spark_api(self.user)
     api = get_spark_api(self.user)
-    session = _get_snippet_session(notebook, snippet)
 
 
     if session['id'] is not None:
     if session['id'] is not None:
       api.close(session['id'])
       api.close(session['id'])
@@ -442,10 +447,7 @@ class SparkApi():
     return []
     return []
 
 
 
 
-class SparkBatchApi():
-
-  def __init__(self, user):
-    self.user = user
+class SparkBatchApi(Api):
 
 
   def create_session(self, lang, properties=None):
   def create_session(self, lang, properties=None):
     return {
     return {

+ 23 - 3
apps/spark/src/spark/static/spark/js/spark.ko.js

@@ -85,7 +85,7 @@ var Result = function (snippet, result) {
         handle: self.handle
         handle: self.handle
     };
     };
   }
   }
-  
+
   if (typeof result.handle != "undefined" && result.handle != null) {
   if (typeof result.handle != "undefined" && result.handle != null) {
     $.each(result.handle, function (key, val) {
     $.each(result.handle, function (key, val) {
       self.handle()[key] = val;
       self.handle()[key] = val;
@@ -313,7 +313,7 @@ var Snippet = function (vm, notebook, snippet) {
       result: self.result.getContext()
       result: self.result.getContext()
     };
     };
   }
   }
-  
+
   self._ajax_error = function (data, callback) {
   self._ajax_error = function (data, callback) {
     if (data.status == -2) {
     if (data.status == -2) {
       self.create_session(callback);
       self.create_session(callback);
@@ -604,6 +604,9 @@ var Notebook = function (vm, notebook) {
           snippet.status(status);
           snippet.status(status);
         });
         });
       };
       };
+
+      self.closeSession(session);
+
       setSnippetStatus('loading');
       setSnippetStatus('loading');
       var successCallback = function() {
       var successCallback = function() {
         setSnippetStatus('ready');
         setSnippetStatus('ready');
@@ -683,6 +686,8 @@ var Notebook = function (vm, notebook) {
     else {
     else {
       _snippet.status('ready');
       _snippet.status('ready');
     }
     }
+
+    logGA('/add_snippet/' + self.selectedSnippet());
     $(document).trigger("snippetAdded", _snippet);
     $(document).trigger("snippetAdded", _snippet);
   };
   };
 
 
@@ -732,6 +737,21 @@ var Notebook = function (vm, notebook) {
       snippet.status('ready');
       snippet.status('ready');
     });
     });
   };
   };
+
+  self.closeSession = function (session) {
+    $.post("/spark/api/close_session", {
+      session: ko.mapping.toJSON(session)
+    }, function (data) {
+      if (data.status == 0) {
+        notebook.sessions.remove(session);
+      }
+      else {
+        $(document).trigger("error", data.message);
+      }
+    }).fail(function (xhr, textStatus, errorThrown) {
+      $(document).trigger("error", xhr.responseText);
+    });
+  };
 }
 }
 
 
 
 
@@ -821,6 +841,6 @@ function EditorViewModel(notebooks, options) {
 
 
 function logGA(page) {
 function logGA(page) {
   if (typeof trackOnGA == 'function') {
   if (typeof trackOnGA == 'function') {
-    trackOnGA('editor/' + page);
+    trackOnGA('notebook/' + page);
   }
   }
 }
 }

+ 5 - 2
apps/spark/src/spark/templates/editor_components.mako

@@ -819,8 +819,11 @@ from django.utils.translation import ugettext as _
               </div>
               </div>
             </div>
             </div>
             <!-- /ko -->
             <!-- /ko -->
-            <a style="float: right;" class="btn pointer" title="${ _('Restart Session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().restartSession($parent) }">
-              <i class="fa fa-refresh"></i> ${ _('Session') }
+            <a style="float: right;" class="btn pointer" title="${ _('Restart session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().restartSession($parent) }">
+              <i class="fa fa-refresh"></i> ${ _('Recreate') }
+            </a>
+            <a style="float: right;" class="btn pointer" title="${ _('Close session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().closeSession($parent) }">
+              <i class="fa fa-times"></i> ${ _('Close') }
             </a>
             </a>
             <!-- /ko -->
             <!-- /ko -->
             <!-- /ko -->
             <!-- /ko -->

+ 1 - 0
apps/spark/src/spark/urls.py

@@ -44,6 +44,7 @@ urlpatterns = patterns('spark.views',
 # APIs
 # APIs
 urlpatterns += patterns('spark.api',
 urlpatterns += patterns('spark.api',
   url(r'^api/create_session$', 'create_session', name='create_session'),
   url(r'^api/create_session$', 'create_session', name='create_session'),
+  url(r'^api/close_session$', 'close_session', name='close_session'),
   url(r'^api/execute$', 'execute', name='execute'),
   url(r'^api/execute$', 'execute', name='execute'),
   url(r'^api/check_status$', 'check_status', name='check_status'),
   url(r'^api/check_status$', 'check_status', name='check_status'),
   url(r'^api/fetch_result_data$', 'fetch_result_data', name='fetch_result_data'),
   url(r'^api/fetch_result_data$', 'fetch_result_data', name='fetch_result_data'),

+ 0 - 2
desktop/core/src/desktop/api.py

@@ -21,8 +21,6 @@ import time
 
 
 from collections import defaultdict
 from collections import defaultdict
 
 
-from django.core.urlresolvers import reverse
-
 from django.utils import html
 from django.utils import html
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _