فهرست منبع

[spark] Save and load a script

Romain Rigaux 12 سال پیش
والد
کامیت
be12dcc1be

+ 2 - 0
apps/beeswax/src/beeswax/models.py

@@ -104,6 +104,8 @@ class QueryHistory(models.Model):
       return 'impala'
     elif query_type == RDBMS:
       return 'rdbms'
+    elif query_type == SPARK:
+      return 'spark'
     else:
       return 'beeswax'
 

+ 3 - 3
apps/beeswax/src/beeswax/templates/layout.mako

@@ -43,7 +43,7 @@ def is_selected(section, matcher):
             </li>
             <li class="${is_selected(section, 'query')}"><a href="${ url('spark:editor') }">${_('Query Editor')}</a></li>
             ##<li class="${is_selected(section, 'my queries')}"><a href="${ url(app_name + ':my_queries') }">${_('My Queries')}</a></li>
-            <li class="${is_selected(section, 'queries')}"><a href="${ url('spark:list_designs') }">${_('Queries')}</a></li>
+            <li class="${is_selected(section, 'saved queries')}"><a href="${ url('spark:list_designs') }">${_('Queries')}</a></li>
             <li class="${is_selected(section, 'history')}"><a href="${ url('spark:list_query_history') }">${_('History')}</a></li>
             <li class="currentApp">
               <a href="/spark">
@@ -58,7 +58,7 @@ def is_selected(section, matcher):
       </div>
     </div>
   </div>
-  % else:  
+  % else:
   <div class="navbar navbar-inverse navbar-fixed-top">
       <div class="navbar-inner">
         <div class="container-fluid">
@@ -78,7 +78,7 @@ def is_selected(section, matcher):
                 % endif
                 </a>
               </li>
-              <li class="${is_selected(section, 'query')}"><a href="${ url(app_name + ':execute_query') }">${_('Query Editor')}</a></li>            
+              <li class="${is_selected(section, 'query')}"><a href="${ url(app_name + ':execute_query') }">${_('Query Editor')}</a></li>
               <li class="${is_selected(section, 'my queries')}"><a href="${ url(app_name + ':my_queries') }">${_('My Queries')}</a></li>
               <li class="${is_selected(section, 'saved queries')}"><a href="${ url(app_name + ':list_designs') }">${_('Saved Queries')}</a></li>
               <li class="${is_selected(section, 'history')}"><a href="${ url(app_name + ':list_query_history') }">${_('History')}</a></li>

+ 8 - 3
apps/beeswax/src/beeswax/views.py

@@ -74,17 +74,22 @@ def save_design(request, form, type_, design, explicit_save):
   Assumes that form.saveform is the SaveForm, and that it is valid.
   """
   assert form.saveform.is_valid()
+  sub_design_form = None # Beeswax/Impala case
 
   if type_ == models.HQL:
     design_cls = beeswax.design.HQLdesign
   elif type_ == models.IMPALA:
     design_cls = beeswax.design.HQLdesign
+  elif type_ == models.SPARK:
+    from spark.design import SparkDesign
+    design_cls = SparkDesign
+    sub_design_form = form.query
   else:
     raise ValueError(_('Invalid design type %(type)s') % {'type': type_})
 
-  # design here means SavedQuery
+  # Design here means SavedQuery
   old_design = design
-  design_obj = design_cls(form, query_type=type_)
+  design_obj = design_cls(sub_design_form, query_type=type_)
   new_data = design_obj.dumps()
 
   # Auto save if (1) the user didn't click "save", and (2) the data is different.
@@ -197,7 +202,7 @@ def list_designs(request):
     text=<frag> - Search for fragment "frag" in names and descriptions.
   """
   DEFAULT_PAGE_SIZE = 20
-  app_name= get_app_name(request)
+  app_name = get_app_name(request)
 
   # Extract the saved query list.
   prefix = 'q-'

+ 0 - 1
apps/oozie/src/oozie/models.py

@@ -45,7 +45,6 @@ from desktop.models import Document
 from hadoop.fs.exceptions import WebHdfsException
 
 from hadoop.fs.hadoopfs import Hdfs
-from liboozie.conf import SECURITY_ENABLED
 from liboozie.submittion import Submission
 from liboozie.submittion import create_directories
 

+ 1 - 1
apps/oozie/src/oozie/views/dashboard.py

@@ -682,7 +682,7 @@ def split_oozie_jobs(user, oozie_jobs):
   jobs_completed = []
 
   for job in oozie_jobs:
-    if job.appName not in ('pig-app-hue-script', 'spark-app-hue-script'):
+    if job.appName != 'pig-app-hue-script':
       if job.is_running():
         if job.type == 'Workflow':
           job = get_oozie(user).get_job(job.id)

+ 24 - 78
apps/spark/src/spark/api.py

@@ -27,14 +27,12 @@ from desktop.lib.exceptions import StructuredException
 
 from beeswax import models as beeswax_models
 from beeswax.forms import SaveForm
-from beeswax.views import authorized_get_history, safe_get_design
+from beeswax.views import authorized_get_history, safe_get_design, save_design
 
-from spark import conf
-from spark.design import SparkDesign
-from spark.views import save_design
 from spark.job_server_api import get_api
-from spark.forms import SparkForm, UploadApp
-import urllib
+from spark.forms import SparkForm, QueryForm
+from desktop.lib.i18n import smart_str
+from spark.design import SparkDesign
 
 
 LOG = logging.getLogger(__name__)
@@ -111,11 +109,12 @@ def delete_context(request):
   api = get_api(request.user)
   try:
     response = api.delete_context(name)
+  except ValueError:
+    # No json is returned
+    response = {'status': 'OK'}
   except Exception, e:
-    if e.message != 'No JSON object could be decoded':
-      response = json.loads(e.message)
-    else:
-      response = {'status': 'OK'}
+    response = json.loads(e.message)
+
   response['name'] = name
 
   return HttpResponse(json.dumps(response), mimetype="application/json")
@@ -182,51 +181,6 @@ def execute(request, design_id=None):
   return HttpResponse(json.dumps(response, cls=ResultEncoder), mimetype="application/json")
 
 
-@error_handler
-def fetch_results(request, id, first_row=0):
-  """
-  Returns the results of the QueryHistory with the given id.
-
-  The query results MUST be ready.
-
-  If ``first_row`` is 0, restarts (if necessary) the query read.  Otherwise, just
-  spits out a warning if first_row doesn't match the servers conception.
-  Multiple readers will produce a confusing interaction here, and that's known.
-  """
-  first_row = long(first_row)
-  results = type('Result', (object,), {
-                'rows': 0,
-                'columns': [],
-                'has_more': False,
-                'start_row': 0,
-            })
-  fetch_error = False
-  error_message = ''
-
-  query_history = authorized_get_history(request, id, must_exist=True)
-  query_server = query_history.get_query_server_config()
-  design = SQLdesign.loads(query_history.design.data)
-  db = dbms.get(request.user, query_server)
-
-  try:
-    database = design.query.get('database', 'default')
-    db.use(database)
-    datatable = db.execute_and_wait(design)
-    results = db.client.create_result(datatable)
-    status = 0
-  except Exception, e:
-    fetch_error = True
-    error_message = str(e)
-    status = -1
-
-  response = {
-    'status': status,
-    'message': fetch_error and error_message or '',
-    'results': results_to_dict(results)
-  }
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
 @error_handler
 def save_query(request, design_id=None):
   response = {'status': -1, 'message': ''}
@@ -237,17 +191,20 @@ def save_query(request, design_id=None):
   app_name = get_app_name(request)
   query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
   design = safe_get_design(request, query_type, design_id)
+  form = QueryForm()
+  api = get_api(request.user)
+  app_names = api.jars()
 
   try:
-    save_form = SaveForm(request.POST.copy())
-    query_form = get_query_form(request, design_id)
+    form.bind(request.POST)
+    form.query.fields['appName'].choices = ((key, key) for key in app_names)
 
-    if query_form.is_valid() and save_form.is_valid():
-      design = save_design(request, save_form, query_form, query_type, design, True)
+    if form.is_valid():
+      design = save_design(request, form, query_type, design, True)
       response['design_id'] = design.id
       response['status'] = 0
     else:
-      response['errors'] = query_form.errors
+      response['message'] = smart_str(form.query.errors) + smart_str(form.saveform.errors)
   except RuntimeError, e:
     response['message'] = str(e)
 
@@ -269,37 +226,26 @@ def fetch_saved_query(request, design_id):
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
-def results_to_dict(results):
-  data = {}
-  rows = []
-  for row in results.rows():
-    rows.append(dict(zip(results.columns, row)))
-  data['rows'] = rows
-  data['start_row'] = results.start_row
-  data['has_more'] = results.has_more
-  data['columns'] = results.columns
-  return data
-
-
 def design_to_dict(design):
-  sql_design = SQLdesign.loads(design.data)
+  spark_design = SparkDesign.loads(design.data)
   return {
     'id': design.id,
-    'query': sql_design.sql_query,
     'name': design.name,
     'desc': design.desc,
-    'server': sql_design.server,
-    'database': sql_design.database
+    'appName': spark_design.appName,
+    'classPath': spark_design.classPath,
+    'autoContext': spark_design.autoContext,
+    'context': spark_design.context,
+    'params': spark_design.params,
   }
 
-
 def get_query_form(request):
   api = get_api(request.user)
 
   app_names = api.jars()
 
   if not app_names:
-    raise RuntimeError(_("Server specified a jar name."))
+    raise RuntimeError(_("Missing application jar list."))
 
   form = SparkForm(request.POST, app_names=app_names)
 

+ 38 - 42
apps/spark/src/spark/design.py

@@ -21,25 +21,20 @@ import logging
 import django.http
 from django.utils.translation import ugettext as _
 
-from beeswax.design import normalize_form_dict, denormalize_form_dict, strip_trailing_semicolon,\
-                           split_statements
+from beeswax.design import normalize_form_dict, denormalize_form_dict
 
 
 LOG = logging.getLogger(__name__)
 
-SERIALIZATION_VERSION = "0.0.1"
+SERIALIZATION_VERSION = "1.0"
 
 
 class SparkDesign(object):
-  """
-  Represents an SQL design, with methods to perform (de)serialization.
-  """
-  _QUERY_ATTRS = [ 'query', 'type', 'database', 'server' ]
+  _QUERY_ATTRS = ['type', 'appName', 'classPath', 'autoContext', 'context', 'params']
 
   def __init__(self, form=None, query_type=None):
-    """Initialize the design from a valid form data."""
     if form is not None:
-      self._data_dict = dict(query = normalize_form_dict(form, SparkDesign._QUERY_ATTRS))
+      self._data_dict = {'query': normalize_form_dict(form, SparkDesign._QUERY_ATTRS)}
       if query_type is not None:
         self._data_dict['query']['type'] = query_type
 
@@ -49,53 +44,48 @@ class SparkDesign(object):
     dic['VERSION'] = SERIALIZATION_VERSION
     return json.dumps(dic)
 
-  @property
-  def sql_query(self):
-    return self._data_dict['query']['query']
-
   @property
   def query(self):
     return self._data_dict['query'].copy()
 
   @property
-  def server(self):
-    return self._data_dict['query']['server']
+  def appName(self):
+    return self._data_dict['query']['appName']
+
+  @property
+  def classPath(self):
+    return self._data_dict['query']['classPath']
+
+  @property
+  def autoContext(self):
+    return self._data_dict['query']['autoContext']
+
+  @property
+  def context(self):
+    return self._data_dict['query']['context']
 
   @property
-  def database(self):
-    return self._data_dict['query']['database']
+  def params(self):
+    return self._data_dict['query']['params']
 
   def get_query_dict(self):
-    # We construct the mform to use its structure and prefix. We don't actually bind data to the forms.
+    # We construct the form to use its structure and prefix. We don't actually bind data to the forms.
     from beeswax.forms import QueryForm
     mform = QueryForm()
     mform.bind()
 
     res = django.http.QueryDict('', mutable=True)
-    res.update(denormalize_form_dict(
-                self._data_dict['query'], mform.query, SparkDesign._QUERY_ATTRS))
+    res.update(denormalize_form_dict(self._data_dict['query'], mform.query, SparkDesign._QUERY_ATTRS))
     return res
 
-  def get_query(self):
-    return self._data_dict["query"]
-
-  @property
-  def statement_count(self):
-    return len(self.statements)
-
-  def get_query_statement(self, n=0):
-    return self.statements[n]
-
-  @property
-  def statements(self):
-    sql_query = strip_trailing_semicolon(self.sql_query)
-    return [strip_trailing_semicolon(statement.strip()) for statement in split_statements(sql_query)]
-
   @staticmethod
   def loads(data):
-    """Returns SQLdesign from the serialized form"""
-    dic = json.loads(data)
-    dic = dict(map(lambda k: (str(k), dic.get(k)), dic.keys()))
+    if data:
+      dic = json.loads(data)
+      dic = dict(map(lambda k: (str(k), dic.get(k)), dic.keys()))
+    else:
+      dic = {'VERSION': SERIALIZATION_VERSION, 'query': {'type': 3, 'appName': '', 'classPath': 'spark.jobserver.WordCountExample', 'autoContext': True, 'context': '', 'params': ''}}
+
     if dic['VERSION'] != SERIALIZATION_VERSION:
       LOG.error('Design version mismatch. Found %s; expect %s' % (dic['VERSION'], SERIALIZATION_VERSION))
 
@@ -103,10 +93,16 @@ class SparkDesign(object):
     del dic['VERSION']
     if 'type' not in dic['query'] or dic['query']['type'] is None:
       dic['query']['type'] = 0
-    if 'server' not in dic['query']:
-      raise RuntimeError(_('No server!'))
-    if 'database' not in dic['query']:
-      raise RuntimeError(_('No database!'))
+    if 'appName' not in dic['query']:
+      dic['query']['appName'] = ''
+    if 'classPath' not in dic['query']:
+      dic['query']['classPath'] = ''
+    if 'autoContext' not in dic['query']:
+      dic['query']['autoContext'] = True
+    if 'context' not in dic['query']:
+      dic['query']['context'] = ''
+    if 'params' not in dic['query']:
+      dic['query']['params'] = ''
 
     design = SparkDesign()
     design._data_dict = dic

+ 12 - 3
apps/spark/src/spark/forms.py

@@ -17,14 +17,15 @@
 
 from django import forms
 from django.utils.translation import ugettext_lazy as _t
+from desktop.lib.django_forms import MultiForm
+from beeswax.forms import SaveForm
 
 
 class SparkForm(forms.Form):
   query = forms.CharField(label=_t("Script parameters"),
                           required=False)
   classPath = forms.CharField(label=_t("Class path"),
-                          required=True,
-                          widget=forms.Textarea(attrs={'class': 'beeswax_query'}))
+                          required=True)
   appName = forms.ChoiceField(required=True,
                              label='',
                              choices=(('default', 'default'),),
@@ -36,11 +37,19 @@ class SparkForm(forms.Form):
 
 
   def __init__(self, *args, **kwargs):
-    app_names = kwargs.pop('app_names')
+    app_names = kwargs.pop('app_names', [])
     super(SparkForm, self).__init__(*args, **kwargs)
     self.fields['appName'].choices = ((key, key) for key in app_names)
 
 
+class QueryForm(MultiForm):
+  def __init__(self):
+    super(QueryForm, self).__init__(
+      query=SparkForm,
+      saveform=SaveForm
+    )
+
+
 class UploadApp(forms.Form):
   app_name = forms.CharField()
   jar_file = forms.FileField()

+ 1 - 1
apps/spark/src/spark/templates/common.mako

@@ -36,7 +36,7 @@ def is_selected(section, matcher):
                 ${ _('Spark Editor') }
               </a>
             </li>
-            <li class="${is_selected(section, 'query')}"><a href="${ url('spark:editor') }">${_('Query Editor')}</a></li>
+            <li class="${is_selected(section, 'editor')}"><a href="${ url('spark:editor') }">${_('Query Editor')}</a></li>
             ##<li class="${is_selected(section, 'my queries')}"><a href="${ url(app_name + ':my_queries') }">${_('My Queries')}</a></li>
             <li class="${is_selected(section, 'queries')}"><a href="${ url('spark:list_designs') }">${_('Queries')}</a></li>
             <li class="${is_selected(section, 'history')}"><a href="${ url('spark:list_query_history') }">${_('History')}</a></li>

+ 10 - 7
apps/spark/src/spark/templates/editor.mako

@@ -88,18 +88,18 @@ ${ common.navbar('editor') }
       <div class="card card-small">
         <div style="margin-bottom: 30px">
           <h1 class="card-heading simple">
-            ${ _('Parameters') }
             % if can_edit_name:
-              :
               <a href="javascript:void(0);"
                  id="query-name"
                  data-type="text"
                  data-name="name"
                  data-value="${design.name}"
-                 data-original-title="${ _('Query name') }"
+                 data-original-title="${ _('Script name') }"
                  data-placement="right">
               </a>
+              :
             %endif
+            ${ _('Parameters') }
           </h1>
           % if can_edit_name:
             <p style="margin-left: 20px">
@@ -446,6 +446,9 @@ ${ common.navbar('editor') }
     viewModel = new sparkViewModel();
     viewModel.fetchAppNames();
     viewModel.fetchContexts();
+    % if design_json:
+      viewModel.loadDesign(${ design_json | n,unicode });
+    % endif
     % if job_id:
       viewModel.openQuery("${ job_id }");
     % endif
@@ -483,8 +486,8 @@ ${ common.navbar('editor') }
   }
 
   function trySaveAsQuery() {
-    var query = getHighlightedQuery() || codeMirror.getValue();
-    viewModel.query.query(query);
+    var params = getHighlightedQuery() || codeMirror.getValue();
+    viewModel.query.params(params);
     $('#saveAsQueryModal').modal('show');
   }
 
@@ -496,13 +499,13 @@ ${ common.navbar('editor') }
   });
 
   function modalSaveAsQuery() {
-    if (viewModel.query.query() && viewModel.query.name()) {
+    if (viewModel.appName() && viewModel.query.name()) {
       viewModel.query.id(-1);
       viewModel.saveQuery();
       $('#saveas-query-name').removeClass('error');
       $('#saveAsQueryModal').modal('hide');
     } else if (viewModel.query.name()) {
-      $.jHueNotify.error("${_('No query provided to save.')}");
+      $.jHueNotify.error("${_('No application name provided.')}");
       $('#saveAsQueryModal').modal('hide');
     } else {
       $('#saveas-query-name').addClass('error');

+ 1 - 0
apps/spark/src/spark/urls.py

@@ -39,6 +39,7 @@ urlpatterns += patterns('spark.api',
   url(r'^api/job/(?P<job_id>.+)$', 'job', name='job'),
   url(r'^api/create_context$', 'create_context', name='create_context'),
   url(r'^api/delete_context', 'delete_context', name='delete_context'),
+  url(r'^api/save_query/((?P<design_id>\d+)/?)?$', 'save_query', name='save_query'),
 )
 
 urlpatterns += patterns('beeswax.views',

+ 2 - 61
apps/spark/src/spark/views.py

@@ -23,17 +23,16 @@ from django.utils.translation import ugettext as _
 from django.core.urlresolvers import reverse
 
 from desktop.context_processors import get_app_name
-from desktop.models import Document
 from desktop.lib.django_util import render
 from django.shortcuts import redirect
 
 from beeswax import models as beeswax_models
 from beeswax.views import safe_get_design
 
-from spark.design import SparkDesign
 from spark.job_server_api import get_api
 from spark.forms import UploadApp
 from desktop.lib.exceptions import StructuredException
+from spark.api import design_to_dict
 
 
 LOG = logging.getLogger(__name__)
@@ -52,6 +51,7 @@ def editor(request, design_id=None):
   return render('editor.mako', request, {
     'action': action,
     'design': design,
+    'design_json': json.dumps(design_to_dict(design)),
     'can_edit_name': design.id and not design.is_auto,
     'job_id': job_id,
   })
@@ -119,62 +119,3 @@ def upload_app(request):
     response['results'] = form.errors
 
   return redirect(reverse('spark:index'))
-
-
-def save_design(request, save_form, query_form, type_, design, explicit_save=False):
-  """
-  save_design(request, save_form, query_form, type_, design, explicit_save) -> SavedQuery
-
-  A helper method to save the design:
-    * If ``explicit_save``, then we save the data in the current design.
-    * If the user clicked the submit button, we do NOT overwrite the current
-      design. Instead, we create a new "auto" design (iff the user modified
-      the data). This new design is named after the current design, with the
-      AUTO_DESIGN_SUFFIX to signify that it's different.
-
-  Need to return a SavedQuery because we may end up with a different one.
-  Assumes that form.saveform is the SaveForm, and that it is valid.
-  """
-
-  if type_ == beeswax_models.SPARK:
-    design_cls = SparkDesign
-  else:
-    raise ValueError(_('Invalid design type %(type)s') % {'type': type_})
-
-  old_design = design
-  design_obj = design_cls(query_form)
-  new_data = design_obj.dumps()
-
-  # Auto save if (1) the user didn't click "save", and (2) the data is different.
-  # Don't generate an auto-saved design if the user didn't change anything
-  if explicit_save:
-    design.name = save_form.cleaned_data['name']
-    design.desc = save_form.cleaned_data['desc']
-    design.is_auto = False
-  elif new_data != old_design.data:
-    # Auto save iff the data is different
-    if old_design.id is not None:
-      # Clone iff the parent design isn't a new unsaved model
-      design = old_design.clone()
-      if not old_design.is_auto:
-        design.name = old_design.name + beeswax_models.SavedQuery.AUTO_DESIGN_SUFFIX
-    else:
-      design.name = beeswax_models.SavedQuery.DEFAULT_NEW_DESIGN_NAME
-    design.is_auto = True
-
-  design.type = type_
-  design.data = new_data
-
-  design.save()
-
-  LOG.info('Saved %s design "%s" (id %s) for %s' % (design.name and '' or 'auto ', design.name, design.id, design.owner))
-
-  if design.doc.exists():
-    design.doc.update(name=design.name, description=design.desc)
-  else:
-    Document.objects.link(design, owner=design.owner, extra=design.type, name=design.name, description=design.desc)
-
-  if design.is_auto:
-    design.doc.get().add_to_history()
-
-  return design

+ 40 - 19
apps/spark/static/js/spark.vm.js

@@ -23,17 +23,21 @@ function sparkViewModel() {
   self.autoContext = ko.observable(true);
   self.contexts = ko.observableArray(); // List of contexts
   self.selectedContext = ko.observable(0);
-  self.classPathes = ko.observableArray(); // Read from upload or edit manually or better API
-  self.classPath = ko.observable('spark.jobserver.WordCountExample');
+  self.classPath = ko.observable('');
+
   self.query = ko.mapping.fromJS({
     'id': -1,
     'jobId': null,
-    'context': null,
-    'query': '', // query == params
     'name': null,
     'description': null,
-    'errors': []
+    'errors': [],
+    'appName': '',
+    'classPath': '',
+    'context': '',
+    'autoContext': true,
+    'params': '',
   });
+
   self.rows = ko.observableArray();
   self.resultsEmpty = ko.observable(false);
 
@@ -41,7 +45,7 @@ function sparkViewModel() {
     'read': function() {
       if (self.appNames().length > 0) {
         return self.appNames()[self.selectedAppName()];
-      } else{
+      } else {
         return null;
       }
     },
@@ -58,7 +62,7 @@ function sparkViewModel() {
     'read': function() {
       if (self.contexts().length > 0) {
         return self.contexts()[self.selectedContext()];
-      } else{
+      } else {
         return null;
       }
     },
@@ -90,7 +94,7 @@ function sparkViewModel() {
     });
     self.appNames(newAppNames);
 
-    var last = $.totalStorage('hueSparkLastAppName') || (newAppNames.length > 0 ? newAppNames[0].name() : null);
+    var last = newAppNames.length > 0 ? newAppNames[0].name() : null;
     if (last) {
       self.appName(last);
     }
@@ -103,7 +107,7 @@ function sparkViewModel() {
     });
     self.contexts(newContexts);
 
-    var last = $.totalStorage('hueSparkLastContext') || (newContexts.length > 0 ? newContexts[0].name() : null);
+    var last = newContexts.length > 0 ? newContexts[0].name() : null;
     if (last) {
       self.context(last);
     }
@@ -116,12 +120,22 @@ function sparkViewModel() {
     };
   };
 
-  self.updateQuery = function(design) {
-    self.query.query(design.query);
+  self.loadDesign = function(design) {
     self.query.id(design.id);
     self.query.name(design.name);
     self.query.description(design.desc);
-    self.server(design.server);
+
+    self.query.appName(design.appName);
+    self.query.classPath(design.classPath);
+    self.query.autoContext(design.autoContext);
+    self.query.params(design.params);
+
+    self.appName(design.appName);
+    self.chooseAppName(design.appName);
+    self.autoContext(design.autoContext);
+    self.context(design.context);
+    self.chooseContext(design.context);
+    self.classPath(design.classPath);
   };
 
   self.chooseAppName = function(value, e) {
@@ -130,7 +144,6 @@ function sparkViewModel() {
         self.selectedAppName(index);
       }
     });
-    $.totalStorage('hueSparkLastAppName', self.appName().name());
   };
 
   self.chooseContext = function(value, e) {
@@ -139,7 +152,6 @@ function sparkViewModel() {
         self.selectedContext(index);
       }
     });
-    $.totalStorage('hueSparkLastContext', self.context().name());
   };
 
   var error_fn = function(jqXHR, status, errorThrown) {
@@ -152,11 +164,16 @@ function sparkViewModel() {
 
   self.saveQuery = function() {
     var self = this;
-    if (self.query.query() && self.query.name()) {
+    if (self.query.name()) {
       var data = ko.mapping.toJS(self.query);
-      data['desc'] = data['description'];
-      data['server'] = self.server().name();
-      var url = '/spark/api/query/';
+      data['saveform-name'] = data['name'];
+      data['saveform-desc'] = data['description'];
+      data['query-appName'] = self.appName().name;
+      data['query-classPath'] = self.classPath();
+      data['query-autoContext'] = self.autoContext();
+      data['query-context'] = self.context().name;
+      data['query-params'] = '';
+      var url = '/spark/api/save_query/';
       if (self.query.id() && self.query.id() != -1) {
         url += self.query.id() + '/';
       }
@@ -165,7 +182,11 @@ function sparkViewModel() {
         dataType: 'json',
         type: 'POST',
         success: function(data) {
-          $(document).trigger('saved.query', data);
+          if (data.status == 0) {
+            $(document).trigger('saved.query', data);
+          } else {
+        	self.query.errors.push(data.message);
+          }
         },
         error: function() {
           $(document).trigger('error.query');

+ 3 - 1
desktop/core/src/desktop/models.py

@@ -169,7 +169,7 @@ class DocumentManager(models.Manager):
     return Document.objects.filter(Q(owner=user) | Q(documentpermission__users=user) | Q(documentpermission__groups__in=user.groups.all()))
 
   def get_docs(self, user, model_class=None, extra=None):
-    docs = Document.objects.documents(user).exclude(name='pig-app-hue-script').exclude(name='spark-app-hue-script')
+    docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')
 
     if model_class is not None:
       ct = ContentType.objects.get_for_model(model_class)
@@ -436,6 +436,8 @@ class Document(models.Model):
       if self.content_type.app_label == 'beeswax':
         if self.extra == '0':
           return apps['beeswax'].icon_path
+        elif self.extra == '3':
+          return apps['spark'].icon_path        
         else:
           return apps['impala'].icon_path
       elif self.content_type.app_label == 'oozie':

+ 3 - 0
desktop/core/src/desktop/templates/home.mako

@@ -102,6 +102,9 @@ ${ commonheader(_('Welcome Home'), "home", user) | n,unicode }
                 % if 'pig' in apps:
                 <li><a href="${ url('beeswax:index') }"><img src="${ apps['pig'].icon_path }"/> ${_('Pig Script')}</a></li>
                 % endif
+                % if 'spark' in apps:
+                <li><a href="${ url('spark:index') }"><img src="${ apps['spark'].icon_path }"/> ${_('Spark Job')}</a></li>
+                % endif
                 % if 'oozie' in apps:
                 <li class="dropdown-submenu">
                   <a href="#"><img src="${ apps['oozie'].icon_path }"/> ${_('Oozie Scheduler')}</a>