浏览代码

[spark] Fetch log and compute progress

Romain Rigaux 11 年之前
父节点
当前提交
a25a5fd

+ 4 - 2
apps/spark/src/spark/api.py

@@ -143,14 +143,16 @@ def cancel_statement(request):
   return HttpResponse(json.dumps(response), mimetype="application/json")
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
 
 
-def get_log(request):
+def get_logs(request):
   response = {'status': -1}
   response = {'status': -1}
 
 
   notebook = json.loads(request.POST.get('notebook', '{}'))
   notebook = json.loads(request.POST.get('notebook', '{}'))
   snippet = json.loads(request.POST.get('snippet', '{}'))
   snippet = json.loads(request.POST.get('snippet', '{}'))
 
 
   try:
   try:
-    response['logs'] = get_api(request.user, snippet).get_log(snippet)
+    db = get_api(request.user, snippet)
+    response['logs'] = db.get_log(snippet)
+    response['progress'] = db._progress(snippet, response['logs'])
     response['status'] = 0
     response['status'] = 0
   except Exception, e:
   except Exception, e:
     message = force_unicode(str(e))
     message = force_unicode(str(e))

+ 16 - 3
apps/spark/src/spark/models.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 # limitations under the License.
 
 
 import json
 import json
+import re
 
 
 from beeswax import models as beeswax_models
 from beeswax import models as beeswax_models
 from beeswax.design import hql_query
 from beeswax.design import hql_query
@@ -169,10 +170,22 @@ class HS2Api():
     db = dbms.get(self.user)
     db = dbms.get(self.user)
       
       
     handle = self._get_handle(snippet)    
     handle = self._get_handle(snippet)    
-    return db.get_log(self, handle)
+    return db.get_log(handle)
   
   
-  def progress(self):
-    pass  
+  def _progress(self, snippet, logs):
+    if snippet['type'] == 'hive':
+      match = re.search('Total jobs = (\d+)', logs, re.MULTILINE)
+      total = (int(match.group(1)) if match else 1) * 2
+      
+      started = logs.count('Starting Job')
+      ended = logs.count('Ended Job')
+      
+      return int((started + ended) * 100 / total)
+    elif snippet['type'] == 'impala':
+      match = re.search('(\d+)% Complete', logs, re.MULTILINE)
+      return int(match.group(1)) if match else 0
+    else: #'spark-sql'
+      return 50
 
 
 
 
 class SparkApi():  # Pig, DBquery, Phoenix... 
 class SparkApi():  # Pig, DBquery, Phoenix... 

+ 6 - 0
apps/spark/src/spark/templates/editor.mako

@@ -188,6 +188,7 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
         <span data-bind="editable: name, editableOptions: {enabled: $root.isEditing(), placement: 'right'}"></span>
         <span data-bind="editable: name, editableOptions: {enabled: $root.isEditing(), placement: 'right'}"></span>
         <div class="inline pull-right">
         <div class="inline pull-right">
           <strong class="muted" data-bind="text: status"></strong> &nbsp;
           <strong class="muted" data-bind="text: status"></strong> &nbsp;
+          <strong class="muted" data-bind="text: progress"></strong>% &nbsp;
           <a href="javascript:void(0)" data-bind="visible: $root.isEditing, click: function(){ remove($parent, $data);}"><i class="fa fa-times"></i></a>
           <a href="javascript:void(0)" data-bind="visible: $root.isEditing, click: function(){ remove($parent, $data);}"><i class="fa fa-times"></i></a>
         </div>
         </div>
       </h2>
       </h2>
@@ -212,6 +213,11 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
         <a data-bind="visible: status() != 'ready', click: function() { $data.showLogs(! $data.showLogs()); }, css: {'active': $data.showLogs}" href="javascript:void(0)" class="btn" title="${ _('Logs') }"><i class="fa fa-file-text-o"></i></a>
         <a data-bind="visible: status() != 'ready', click: function() { $data.showLogs(! $data.showLogs()); }, css: {'active': $data.showLogs}" href="javascript:void(0)" class="btn" title="${ _('Logs') }"><i class="fa fa-file-text-o"></i></a>
       </div>
       </div>
 
 
+      <div data-bind="visible: showLogs, css: resultsKlass">
+        <span data-bind="visible: result.logs().length == 0">${ _('Loading...') }</span>
+        <span data-bind="text: result.logs"></span>
+      </div>
+
       <div class="row-fluid" data-bind="visible: result.meta().length > 0 && showGrid()">
       <div class="row-fluid" data-bind="visible: result.meta().length > 0 && showGrid()">
         <div class="span2">
         <div class="span2">
           <ul class="nav nav-list" style="border: none; background-color: #FFF">
           <ul class="nav nav-list" style="border: none; background-color: #FFF">

+ 1 - 1
apps/spark/src/spark/urls.py

@@ -33,7 +33,7 @@ urlpatterns += patterns('spark.api',
   url(r'^api/fetch_result_data$', 'fetch_result_data', name='fetch_result_data'),
   url(r'^api/fetch_result_data$', 'fetch_result_data', name='fetch_result_data'),
   url(r'^api/fetch_result_metadata$', 'fetch_result_metadata', name='fetch_result_metadata'),
   url(r'^api/fetch_result_metadata$', 'fetch_result_metadata', name='fetch_result_metadata'),
   url(r'^api/cancel_statement', 'cancel_statement', name='cancel_statement'),
   url(r'^api/cancel_statement', 'cancel_statement', name='cancel_statement'),
-  url(r'^api/get_log', 'get_log', name='get_log'),
+  url(r'^api/get_logs', 'get_logs', name='get_logs'),
 
 
   url(r'^api/notebook/save$', 'save_notebook', name='save_notebook'),
   url(r'^api/notebook/save$', 'save_notebook', name='save_notebook'),
   url(r'^api/notebook/open$', 'open_notebook', name='open_notebook'),
   url(r'^api/notebook/open$', 'open_notebook', name='open_notebook'),

+ 15 - 5
apps/spark/static/js/spark.vm.js

@@ -47,6 +47,7 @@ var Result = function (snippet, result) {
     });
     });
     self.meta.removeAll();
     self.meta.removeAll();
     self.data.removeAll();
     self.data.removeAll();
+    self.logs('');
   };  
   };  
 }
 }
 
 
@@ -124,6 +125,8 @@ var Snippet = function (notebook, snippet) {
   self.showGrid = ko.observable(typeof snippet.showGrid != "undefined" && snippet.showGrid != null ? snippet.showGrid : true);
   self.showGrid = ko.observable(typeof snippet.showGrid != "undefined" && snippet.showGrid != null ? snippet.showGrid : true);
   self.showChart = ko.observable(typeof snippet.showChart != "undefined" && snippet.showChart != null ? snippet.showChart : false);
   self.showChart = ko.observable(typeof snippet.showChart != "undefined" && snippet.showChart != null ? snippet.showChart : false);
   self.showLogs = ko.observable(typeof snippet.showLogs != "undefined" && snippet.showLogs != null ? snippet.showLogs : false);
   self.showLogs = ko.observable(typeof snippet.showLogs != "undefined" && snippet.showLogs != null ? snippet.showLogs : false);
+  self.progress =  ko.observable(typeof snippet.progress != "undefined" && snippet.progress != null ? snippet.progress : 0);
+  
   self.showGrid.subscribe(function (val){
   self.showGrid.subscribe(function (val){
     if (val){
     if (val){
       self.showChart(false);
       self.showChart(false);
@@ -142,7 +145,6 @@ var Snippet = function (notebook, snippet) {
       self.showChart(false);
       self.showChart(false);
     }
     }
   });
   });
-
   self.size = ko.observable(typeof snippet.size != "undefined" && snippet.size != null ? snippet.size : 12).extend({ numeric: 0 });
   self.size = ko.observable(typeof snippet.size != "undefined" && snippet.size != null ? snippet.size : 12).extend({ numeric: 0 });
   self.offset = ko.observable(typeof snippet.offset != "undefined" && snippet.offset != null ? snippet.offset : 0).extend({ numeric: 0 });
   self.offset = ko.observable(typeof snippet.offset != "undefined" && snippet.offset != null ? snippet.offset : 0).extend({ numeric: 0 });
   self.isLoading = ko.computed(function(){
   self.isLoading = ko.computed(function(){
@@ -207,6 +209,7 @@ var Snippet = function (notebook, snippet) {
 	logGA('/execute/' + self.type());
 	logGA('/execute/' + self.type());
     
     
 	self.result.clear();
 	self.result.clear();
+	self.progress(0);
 	self.status('running');
 	self.status('running');
 
 
     $.post("/spark/api/execute", {
     $.post("/spark/api/execute", {
@@ -290,9 +293,11 @@ var Snippet = function (notebook, snippet) {
 
 
           if (self.status() == 'running') {
           if (self.status() == 'running') {
         	self.checkStatusTimeout = setTimeout(self.checkStatus, 1000);
         	self.checkStatusTimeout = setTimeout(self.checkStatus, 1000);
+        	self.getLogs();
           } 
           } 
           else if (self.status() == 'available') {
           else if (self.status() == 'available') {
         	self.fetchResult(100);
         	self.fetchResult(100);
+        	self.progress(100);
           }
           }
 	    } else if (data.status == -2) {
 	    } else if (data.status == -2) {
 	      self.create_session();  
 	      self.create_session();  
@@ -324,13 +329,14 @@ var Snippet = function (notebook, snippet) {
     });
     });
   };
   };
   
   
-  self.get_log = function() {
-    $.post("/spark/api/get_log", {
+  self.getLogs = function() {
+    $.post("/spark/api/get_logs", {
         notebook: ko.mapping.toJSON(notebook),
         notebook: ko.mapping.toJSON(notebook),
         snippet: ko.mapping.toJSON(self)
         snippet: ko.mapping.toJSON(self)
  	  }, function (data) {
  	  }, function (data) {
  	    if (data.status == 0) {
  	    if (data.status == 0) {
- 	      self.result.logs(data.log); // todo smarter? 
+ 	      self.result.logs(data.logs); // Way to append?
+ 	      self.progress(data.progress);
  	    } else {
  	    } else {
  	      $(document).trigger("error", data.message);
  	      $(document).trigger("error", data.message);
  	    }
  	    }
@@ -341,7 +347,11 @@ var Snippet = function (notebook, snippet) {
   
   
   self.init = function() {
   self.init = function() {
 	if (self.status() == 'running') {
 	if (self.status() == 'running') {
-	  self.checkStatus();	
+	  self.checkStatus();
+	} 
+	
+	if (self.status() != 'loading' && self.status() != 'ready') {
+	  self.getLogs();	
 	}
 	}
   };
   };
 }
 }