Ver Fonte

[spark] Cleanup of old Spark Job Server files

Romain Rigaux há 11 anos atrás
pai
commit
938d6b9c25

+ 6 - 350
apps/spark/src/spark/api.py

@@ -18,163 +18,20 @@
 import json
 import logging
 
-from django.http import HttpResponse, Http404
+from django.http import HttpResponse
 from django.utils.translation import ugettext as _
 
-from desktop.context_processors import get_app_name
-from desktop.lib.exceptions import StructuredException
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 
-from beeswax import models as beeswax_models
-from beeswax.design import hql_query
-from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, QueryHistory
-from beeswax.views import safe_get_design, save_design
-from beeswax.server import dbms
-
-from spark.job_server_api import get_api as get_spark_api
-from spark.forms import SparkForm, QueryForm
-from desktop.lib.i18n import smart_str
-from spark.design import SparkDesign
-from desktop.lib.rest.http_client import RestException
-
 from spark.decorators import json_error_handler
+from spark.models import get_api
 
 
 LOG = logging.getLogger(__name__)
 
 
-def get_api(user, snippet):
-  if snippet['type'] == 'hive':
-    return HS2Api(user)
-  else:
-    return SparkApi(user)
-
-
-def _get_snippet_session(notebook, snippet):
-  return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0] 
-
-
-
-class HS2Api():
-  
-  def __init__(self, user):
-    self.user = user
-    
-  def create_session(self, lang):
-    return {
-        'type': lang,
-        'id': None # Real one at some point
-    }
-  
-  def execute(self, notebook, snippet):
-    db = dbms.get(self.user)
-    query = hql_query(snippet['statement'], QUERY_TYPES[0])
-    handle = db.client.query(query)
-    
-#    if not handle.is_valid():
-#        msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % {'id': query_history.id, 'query': query[:40]}
-#        raise QueryServerException(msg)
-#    except QueryServerException, ex:
-#      LOG.exception(ex)
-#      # Kind of expected (hql compile/syntax error, etc.)
-#      if hasattr(ex, 'handle') and ex.handle:
-#        query_history.server_id, query_history.server_guid = ex.handle.id, ex.handle.id
-#        query_history.log_context = ex.handle.log_context
-#      query_history.save_state(QueryHistory.STATE.failed)
-#      raise ex
-
-    # All good
-    server_id, server_guid  = handle.get()
-    return {
-        'secret': server_id,
-        'guid': server_guid,
-        'operation_type': handle.operation_type,
-        'has_result_set': handle.has_result_set,
-        'modified_row_count': handle.modified_row_count,
-        'log_context': handle.log_context
-    }    
-
-  def check_status(self, notebook, snippet):
-    db = dbms.get(self.user)
-      
-    snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
-    handle = HiveServerQueryHandle(**snippet['result']['handle'])
-    status =  db.get_state(handle)
-    return {'status': 'running' if status.index in (QueryHistory.STATE.running.index, QueryHistory.STATE.submitted.index) else 'finished'}
-
-  def fetch_result(self, notebook, snippet):
-    db = dbms.get(self.user)
-      
-    snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
-    handle = HiveServerQueryHandle(**snippet['result']['handle'])
-    results = db.fetch(handle, start_over=False, rows=10)
-    
-    # no escaping...
-    return {
-        'data': list(results.rows()),
-        'meta': [{
-          'name': column.name,
-          'type': column.type,
-          'comment': column.comment
-        } for column in results.data_table.cols()]
-    }
-
-  def fetch_result_metadata(self):
-    pass 
-
-  def cancel(self):
-    pass
-
-  def get_log(self):
-    pass
-  
-  def progress(self):
-    pass  
-
-
-class SparkApi():  # Pig, DBquery, Phoenix... 
-  
-  def __init__(self, user):
-    self.user = user
-  
-  def create_session(self, lang='scala'):
-    api = get_spark_api(self.user)
-    return {
-        'type': lang,
-        'id': api.create_session(lang=lang)
-    } 
-  
-  def execute(self, notebook, snippet):    
-    api = get_spark_api(self.user)
-    session = _get_snippet_session(notebook, snippet)
-    
-    return {'id': api.submit_statement(session['id'], snippet['statement']).split('cells/')[1]}
-
-  def check_status(self, notebook, snippet):
-    return {'status': 'finished'}
-
-  def fetch_result(self, notebook, snippet):
-    api = get_spark_api(self.user)
-    session = _get_snippet_session(notebook, snippet)
-    cell = snippet['result']['handle']['id']  
-    
-    data = api.fetch_data(session['id'], cell)
-      
-    return {
-        'data': [data['output']],
-        'meta': [{
-          'name': column.name,
-          'type': column.type,
-          'comment': column.comment
-        } for column in []]
-    }
-
-  def cancel(self):
-    pass
-
-
-
+@json_error_handler
 def create_session(request):
   response = {'status': -1}
 
@@ -190,7 +47,7 @@ def create_session(request):
 
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
-
+@json_error_handler
 def execute(request):
   response = {'status': -1}
 
@@ -207,6 +64,7 @@ def execute(request):
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
+@json_error_handler
 def check_status(request):
   response = {'status': -1}
 
@@ -223,6 +81,7 @@ def check_status(request):
   return HttpResponse(json.dumps(response), mimetype="application/json")
 
 
+@json_error_handler
 def fetch_result(request):
   response = {'status': -1}
 
@@ -237,206 +96,3 @@ def fetch_result(request):
     response['error'] = force_unicode(str(e))
 
   return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-
-
-@json_error_handler
-def jars(request):
-  api = get_api(request.user)
-  response = {
-    'jars': api.jars()
-  }
-
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-@json_error_handler
-def contexts(request):
-  api = get_api(request.user)
-  response = {
-    'contexts': api.contexts()
-  }
-
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-def create_context(request):
-  if request.method != 'POST':
-    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Requires a POST'))
-  response = {}
-
-  name = request.POST.get('name', '')
-  memPerNode = request.POST.get('mem-per-node', '512m')
-  numCores = request.POST.get('num-cpu-cores', '1')
-
-  api = get_api(request.user)
-  try:
-    response = api.create_context(name, memPerNode=memPerNode, numCores=numCores)
-  except ValueError:
-    # No json is returned
-    response = {'status': 'OK'}
-  except Exception, e:
-    response = json.loads(e.message)
-
-  response['name'] = name
-
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-def delete_context(request):
-  if request.method != 'DELETE':
-    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Requires a DELETE'))
-  response = {}
-
-  name = request.POST.get('name', '')
-
-  api = get_api(request.user)
-  try:
-    response = api.delete_context(name)
-  except ValueError:
-    # No json is returned
-    response = {'status': 'OK'}
-  except Exception, e:
-    response = json.loads(e.message)
-
-  response['name'] = name
-
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-def job(request, job_id):
-  api = get_api(request.user)
-  response = {}
-  try:
-    response['results'] = api.job(job_id)
-  except RestException, e:
-    response['results'] = json.loads(e.message)
-
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-
-#@json_error_handler
-#def execute(request, design_id=None):
-#  response = {'status': -1, 'message': ''}
-#
-#  if request.method != 'POST':
-#    response['message'] = _('A POST request is required.')
-#
-#  app_name = get_app_name(request)
-#  query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
-#  design = safe_get_design(request, query_type, design_id)
-#
-#  try:
-#    form = get_query_form(request)
-#
-#    if form.is_valid():
-#      #design = save_design(request, SaveForm(), form, query_type, design)
-#
-##      query = SQLdesign(form, query_type=query_type)
-##      query_server = dbms.get_query_server_config(request.POST.get('server'))
-##      db = dbms.get(request.user, query_server)
-##      query_history = db.execute_query(query, design)
-##      query_history.last_state = beeswax_models.QueryHistory.STATE.expired.index
-##      query_history.save()
-#
-#      params = '\n'.join(['%(name)s=%(value)s' % param for param in json.loads(form.cleaned_data['params'])])
-#
-#      try:
-#        api = get_api(request.user)
-#
-#        results = api.submit_job(
-#            form.cleaned_data['appName'],
-#            form.cleaned_data['classPath'],
-#            data=params,
-#            context=None if form.cleaned_data['autoContext'] else form.cleaned_data['context'],
-#            sync=False
-#        )
-#
-#        if results['status'] == 'STARTED':
-#          response['status'] = 0
-#          response['results'] = results
-#        else:
-#          response['message'] = str(results[1]['result'])
-#        response['design'] = design.id
-#      except Exception, e:
-#        response['message'] = str(e)
-#
-#    else:
-#      response['message'] = _('There was an error with your query: %s' % form.errors)
-#      response['errors'] = form.errors
-#  except RuntimeError, e:
-#    response['message']= str(e)
-#
-#  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-@json_error_handler
-def save_query(request, design_id=None):
-  response = {'status': -1, 'message': ''}
-
-  if request.method != 'POST':
-    response['message'] = _('A POST request is required.')
-
-  app_name = get_app_name(request)
-  query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
-  design = safe_get_design(request, query_type, design_id)
-  form = QueryForm()
-  api = get_api(request.user)
-  app_names = api.jars()
-
-  try:
-    form.bind(request.POST)
-    form.query.fields['appName'].choices = ((key, key) for key in app_names)
-
-    if form.is_valid():
-      design = save_design(request, form, query_type, design, True)
-      response['design_id'] = design.id
-      response['status'] = 0
-    else:
-      response['message'] = smart_str(form.query.errors) + smart_str(form.saveform.errors)
-  except RuntimeError, e:
-    response['message'] = str(e)
-
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-@json_error_handler
-def fetch_saved_query(request, design_id):
-  response = {'status': -1, 'message': ''}
-
-  if request.method != 'GET':
-    response['message'] = _('A GET request is required.')
-
-  app_name = get_app_name(request)
-  query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
-  design = safe_get_design(request, query_type, design_id)
-
-  response['design'] = design_to_dict(design)
-  return HttpResponse(json.dumps(response), mimetype="application/json")
-
-
-def design_to_dict(design):
-  spark_design = SparkDesign.loads(design.data)
-  return {
-    'id': design.id,
-    'name': design.name,
-    'desc': design.desc,
-    'appName': spark_design.appName,
-    'classPath': spark_design.classPath,
-    'autoContext': spark_design.autoContext,
-    'context': spark_design.context,
-    'params': json.loads(spark_design.params),
-  }
-
-def get_query_form(request):
-  api = get_api(request.user)
-
-  app_names = api.jars()
-
-  if not app_names:
-    raise RuntimeError(_("Missing application jar list."))
-
-  form = SparkForm(request.POST, app_names=app_names)
-
-  return form

+ 0 - 39
apps/spark/src/spark/forms.py

@@ -14,42 +14,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from django import forms
-from django.utils.translation import ugettext_lazy as _t
-from desktop.lib.django_forms import MultiForm
-from beeswax.forms import SaveForm
-
-
-class SparkForm(forms.Form):
-  params = forms.CharField(label=_t("Script parameters"),
-                          required=False)
-  classPath = forms.CharField(label=_t("Class path"),
-                          required=True)
-  appName = forms.ChoiceField(required=True,
-                             label='',
-                             choices=(('default', 'default'),),
-                             initial=0,
-                             widget=forms.widgets.Select(attrs={'class': 'input-medium'}))
-  autoContext = forms.BooleanField(required=False,
-                                   initial=True)
-  context = forms.CharField(required=False)
-
-
-  def __init__(self, *args, **kwargs):
-    app_names = kwargs.pop('app_names', [])
-    super(SparkForm, self).__init__(*args, **kwargs)
-    self.fields['appName'].choices = ((key, key) for key in app_names)
-
-
-class QueryForm(MultiForm):
-  def __init__(self):
-    super(QueryForm, self).__init__(
-      query=SparkForm,
-      saveform=SaveForm
-    )
-
-
-class UploadApp(forms.Form):
-  app_name = forms.CharField()
-  jar_file = forms.FileField()

+ 1 - 28
apps/spark/src/spark/job_server_api.py

@@ -57,7 +57,6 @@ class JobServerApi(object):
     self._client = HttpClient(self._url, logger=LOG)
     self._root = Resource(self._client)
     self._security_enabled = False
-    # To store user info
     self._thread_local = threading.local()
 
   def __str__(self):
@@ -82,9 +81,7 @@ class JobServerApi(object):
       self._thread_local.user = user
 
   def create_session(self, **kwargs):
-    a = self._root.post('sessions', params=kwargs)
-    print a
-    return a
+    return self._root.post('sessions', params=kwargs)
 
   def submit_statement(self, uuid, statement):
     data = {'statement': statement}
@@ -92,27 +89,3 @@ class JobServerApi(object):
 
   def fetch_data(self, session, cell):
     return self._root.get('sessions/%s/cells/%s' % (session, cell))
-
-#curl http://localhost:8080/sessions/87576bf4-f22c-4681-8f33-d3a329577ec9/cells/0
-#{"id":0,"state":"COMPLETE","input":["1+2"],"output":["res0: Int = 3"],"error":[]}
-
-#  def job(self, job_id):
-#    return self._root.get('jobs/%s' % job_id, headers={'Accept': _JSON_CONTENT_TYPE})
-#
-#  def jobs(self, **kwargs):
-#    return self._root.get('jobs', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
-#
-#  def create_context(self, name, **kwargs):
-#    return self._root.post('contexts/%s' % name, params=kwargs, contenttype=_BINARY_CONTENT_TYPE)
-#
-#  def contexts(self, **kwargs):
-#    return self._root.get('contexts', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
-#
-#  def delete_context(self, name, **kwargs):
-#    return self._root.delete('contexts/%s' % name)
-#
-#  def upload_jar(self, app_name, data):
-#    return self._root.post('jars/%s' % app_name, data=data, contenttype=_BINARY_CONTENT_TYPE)
-
-  def jars(self, **kwargs):
-    return self._root.get('jars', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})

+ 141 - 0
apps/spark/src/spark/models.py

@@ -14,3 +14,144 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from beeswax import models as beeswax_models
+from beeswax.design import hql_query
+from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, QueryHistory
+from beeswax.views import safe_get_design, save_design
+from beeswax.server import dbms
+
+from spark.job_server_api import get_api as get_spark_api
+from desktop.lib.i18n import smart_str
+from desktop.lib.rest.http_client import RestException
+
+
+
+def get_api(user, snippet):
+  if snippet['type'] == 'hive':
+    return HS2Api(user)
+  else:
+    return SparkApi(user)
+
+
+def _get_snippet_session(notebook, snippet):
+  return [session for session in notebook['sessions'] if session['type'] == snippet['type']][0] 
+
+
+
+class HS2Api():
+  
+  def __init__(self, user):
+    self.user = user
+    
+  def create_session(self, lang):
+    return {
+        'type': lang,
+        'id': None # Real one at some point
+    }
+  
+  def execute(self, notebook, snippet):
+    db = dbms.get(self.user)
+    query = hql_query(snippet['statement'], QUERY_TYPES[0])
+    handle = db.client.query(query)
+    
+#    if not handle.is_valid():
+#        msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % {'id': query_history.id, 'query': query[:40]}
+#        raise QueryServerException(msg)
+#    except QueryServerException, ex:
+#      LOG.exception(ex)
+#      # Kind of expected (hql compile/syntax error, etc.)
+#      if hasattr(ex, 'handle') and ex.handle:
+#        query_history.server_id, query_history.server_guid = ex.handle.id, ex.handle.id
+#        query_history.log_context = ex.handle.log_context
+#      query_history.save_state(QueryHistory.STATE.failed)
+#      raise ex
+
+    # All good
+    server_id, server_guid  = handle.get()
+    return {
+        'secret': server_id,
+        'guid': server_guid,
+        'operation_type': handle.operation_type,
+        'has_result_set': handle.has_result_set,
+        'modified_row_count': handle.modified_row_count,
+        'log_context': handle.log_context
+    }    
+
+  def check_status(self, notebook, snippet):
+    db = dbms.get(self.user)
+      
+    snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
+    handle = HiveServerQueryHandle(**snippet['result']['handle'])
+    status =  db.get_state(handle)
+    return {'status': 'running' if status.index in (QueryHistory.STATE.running.index, QueryHistory.STATE.submitted.index) else 'finished'}
+
+  def fetch_result(self, notebook, snippet):
+    db = dbms.get(self.user)
+      
+    snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
+    handle = HiveServerQueryHandle(**snippet['result']['handle'])
+    results = db.fetch(handle, start_over=False, rows=10)
+    
+    # no escaping...
+    return {
+        'data': list(results.rows()),
+        'meta': [{
+          'name': column.name,
+          'type': column.type,
+          'comment': column.comment
+        } for column in results.data_table.cols()]
+    }
+
+  def fetch_result_metadata(self):
+    pass 
+
+  def cancel(self):
+    pass
+
+  def get_log(self):
+    pass
+  
+  def progress(self):
+    pass  
+
+
+class SparkApi():  # Pig, DBquery, Phoenix... 
+  
+  def __init__(self, user):
+    self.user = user
+  
+  def create_session(self, lang='scala'):
+    api = get_spark_api(self.user)
+    return {
+        'type': lang,
+        'id': api.create_session(lang=lang)
+    } 
+  
+  def execute(self, notebook, snippet):    
+    api = get_spark_api(self.user)
+    session = _get_snippet_session(notebook, snippet)
+    
+    return {'id': api.submit_statement(session['id'], snippet['statement']).split('cells/')[1]}
+
+  def check_status(self, notebook, snippet):
+    return {'status': 'finished'}
+
+  def fetch_result(self, notebook, snippet):
+    api = get_spark_api(self.user)
+    session = _get_snippet_session(notebook, snippet)
+    cell = snippet['result']['handle']['id']  
+    
+    data = api.fetch_data(session['id'], cell)
+      
+    return {
+        'data': [data['output']],
+        'meta': [{
+          'name': column.name,
+          'type': column.type,
+          'comment': column.comment
+        } for column in []]
+    }
+
+  def cancel(self):
+    pass

+ 0 - 120
apps/spark/src/spark/templates/common.mako

@@ -1,120 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements.  See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership.  Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License.  You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-
-<%! from django.utils.translation import ugettext as _ %>
-
-<%!
-def is_selected(section, matcher):
-  if section == matcher:
-    return "active"
-  else:
-    return ""
-%>
-
-## Nav bar is also duplicated in beeswax layout.mako.
-
-<%def name="navbar(section='editor')">
-  <div class="navbar navbar-inverse navbar-fixed-top">
-    <div class="navbar-inner">
-      <div class="container-fluid">
-        <div class="nav-collapse">
-          <ul class="nav">
-            <li class="currentApp">
-              <a href="/spark">
-                <img src="/spark/static/art/icon_spark_48.png" class="app-icon"/>
-                ${ _('Spark Igniter') }
-              </a>
-            </li>
-            <li class="${is_selected(section, 'editor')}"><a href="${ url('spark:editor') }">${_('Editor')}</a></li>
-            ##<li class="${is_selected(section, 'my queries')}"><a href="${ url(app_name + ':my_queries') }">${_('My Queries')}</a></li>
-            <li class="${is_selected(section, 'saved queries')}"><a href="${ url('spark:list_designs') }">${_('Applications')}</a></li>
-            ##<li class="${is_selected(section, 'history')}"><a href="${ url('spark:list_query_history') }">${_('History')}</a></li>
-            <li class="${is_selected(section, 'jobs')}"><a href="${ url('spark:list_jobs') }">${_('Dashboard')}</a></li>
-            <li class="${is_selected(section, 'contexts')}"><a href="${ url('spark:list_contexts') }">${_('Contexts')}</a></li>
-            <li class="${is_selected(section, 'applications')}"><a href="${ url('spark:list_applications') }">${_('Uploads')}</a></li>
-          </ul>
-        </div>
-      </div>
-    </div>
-  </div>
-</%def>
-
-<%def name="createContextModal()">
-<div id="createContextModal" class="modal hide fade">
-  <div class="modal-header">
-    <a href="#" class="close" data-dismiss="modal">&times;</a>
-    <h3>${_('Create context')}</h3>
-  </div>
-  <div class="modal-body">
-    <form class="form-horizontal" id="createContextForm">
-      <div class="control-group">
-        <label class="control-label">${ _("Name") }</label>
-        <div class="controls">
-          <input type="text" name="name" data-default="">
-        </div>
-      </div>
-      <div class="control-group">
-        <label class="control-label">${ _("Num cpu cores") }</label>
-        <div class="controls">
-          <input type="text" name="num-cpu-cores" value="1" data-default="1">
-        </div>
-      </div>
-      <div class="control-group">
-        <label class="control-label">${ _("Memory per node") }</label>
-        <div class="controls">
-          <input type="text" name="mem-per-node" value="512m" data-default="512m">
-        </div>
-      </div>
-    </form>
-  </div>
-  <div class="modal-footer">
-    <button class="btn" data-dismiss="modal">${_('Cancel')}</button>
-    <button id="createContextBtn" data-bind="click: createContext" class="btn btn-primary disable-feedback">${_('Create')}</button>
-  </div>
-</div>
-</%def>
-
-
-<%def name="uploadAppModal()">
-<div id="uploadAppModal" class="modal hide fade">
-  <form class="form-horizontal" id="uploadAppForm" action="${ url('spark:upload_app') }" method="POST" enctype="multipart/form-data">
-  ${ csrf_token(request) | n,unicode }
-  <div class="modal-header">
-    <a href="#" class="close" data-dismiss="modal">&times;</a>
-    <h3>${_('Upload application')}</h3>
-  </div>
-  <div class="modal-body">
-    ${ _('One class of the jar should implement SparkJob.') }
-    <div class="control-group">
-      <label class="control-label">${ _("Local jar file") }</label>
-      <div class="controls">
-        <input type="file" name="jar_file" id="jar_file">
-      </div>
-    </div>
-    <div class="control-group">
-      <label class="control-label">${ _("App name") }</label>
-      <div class="controls">
-        <input type="text" name="app_name" id="app_name">
-      </div>
-    </div>
-  </div>
-  <div class="modal-footer">
-    <button class="btn" data-dismiss="modal">${_('Cancel')}</button>
-    <input type="submit" class="btn btn-primary" value="${_('Upload')}"/>
-  </div>
-  </form>
-</div>
-</%def>

+ 0 - 111
apps/spark/src/spark/templates/list_applications.mako

@@ -1,111 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements.  See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership.  Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License.  You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-<%!
-  from desktop.views import commonheader, commonfooter
-  from django.utils.translation import ugettext as _
-%>
-<%namespace name="actionbar" file="actionbar.mako" />
-<%namespace name="common" file="common.mako" />
-
-${ commonheader(_('Applications'), app_name, user) | n,unicode }
-
-${ common.navbar('applications') }
-
-<div class="container-fluid">
-  <div class="card card-small">
-    <h1 class="card-heading simple">${_('Applications')}</h1>
-
-    <%actionbar:render>
-      <%def name="search()">
-          <input id="filterInput" type="text" class="input-xlarge search-query" placeholder="${_('Search for name')}">
-      </%def>
-      <%def name="creation()">
-         <button type="button" class="btn uploadAppModalBtn"><i class="fa fa-plus-circle"></i> ${ _('Upload app') }</button>
-      </%def>
-    </%actionbar:render>
-
-
-
-    <table class="table table-condensed datatables">
-    <thead>
-      <tr>
-        <th>${_('Name')}</th>
-        <th>${_('Upload time')}</th>
-      </tr>
-    </thead>
-    <tbody>
-    % for name, ts in applications.iteritems():
-      <tr>
-        <td>
-          <a href="${ url('spark:editor') }#applicationId=${ name }" data-row-selector="true" title="${ _('Click to open and execute') }">
-            ${ name }
-          </a>
-        </td>
-        <td>${ ts }</td>
-      </tr>
-    % endfor
-
-    </tbody>
-  </table>
-    <div class="card-body">
-      <p>
-        ## ${ comps.pagination(page) }
-      </p>
-    </div>
-  </div>
-</div>
-
-
-${ common.uploadAppModal() }
-
-
-<script src="/static/ext/js/knockout-min.js" type="text/javascript" charset="utf-8"></script>
-
-<script type="text/javascript" charset="utf-8">
-  $(document).ready(function () {
-
-    var apps = $(".datatables").dataTable({
-      "sDom":"<'row'r>t<'row'<'span8'i><''p>>",
-      "bPaginate": false,
-      "bLengthChange": false,
-      "bInfo": false,
-      "aaSorting":[
-        [1, 'desc']
-      ],
-      "aoColumns":[
-        null,
-        null
-      ],
-      "oLanguage":{
-        "sEmptyTable":"${_('No data available')}",
-        "sZeroRecords":"${_('No matching records')}",
-      },
-      "bStateSave": true
-    });
-
-    $("#filterInput").keyup(function () {
-      apps.fnFilter($(this).val());
-    });
-
-    $(".uploadAppModalBtn").on("click", function(){
-      $("#uploadAppModal").modal("show");
-    });
-
-    $("a[data-row-selector='true']").jHueRowSelector();
-  });
-</script>
-
-${ commonfooter(messages) | n,unicode }

+ 0 - 252
apps/spark/src/spark/templates/list_contexts.mako

@@ -1,252 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements.  See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership.  Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License.  You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-<%!
-  from desktop.views import commonheader, commonfooter
-  from django.utils.translation import ugettext as _
-%>
-<%namespace name="actionbar" file="actionbar.mako" />
-<%namespace name="common" file="common.mako" />
-
-${ commonheader(_('Context'), app_name, user) | n,unicode }
-
-${ common.navbar('contexts') }
-
-<div class="container-fluid">
-  <div class="card card-small">
-    <h1 class="card-heading simple">${_('Contexts')}</h1>
-
-    <%actionbar:render>
-      <%def name="search()">
-          <input id="filterInput" type="text" class="input-xlarge search-query" placeholder="${_('Search for name')}">
-      </%def>
-      <%def name="actions()">
-        <button type="button" id="deleteContextBtn" title="${_('Delete forever')}" class="btn" disabled="disabled">
-          <i class="fa fa-bolt"></i> ${ _('Delete selected') }
-        </button>
-      </%def>
-      <%def name="creation()">
-         <button type="button" class="btn createContextModalBtn"><i class="fa fa-plus-circle"></i> ${ _('Create context') }</button>
-      </%def>
-    </%actionbar:render>
-
-    <table class="table table-condensed datatables" id="contextTable">
-    <thead>
-      <th width="1%"><div class="hueCheckbox selectAll fa" data-selectables="savedCheck"></div></th>
-      <th>${ _('Name') }</th>
-    </thead>
-    <tbody>
-      % for contextz in contexts:
-      <tr>
-        <td data-row-selector-exclude="true">
-          <div class="hueCheckbox savedCheck fa" data-delete-name="${ contextz  }" data-row-selector-exclude="true"></div>
-        </td>
-        <td data-name="${ contextz }">${ contextz }</td>
-      </tr>
-      % endfor
-    </tbody>
-    <tfoot>
-      <tr class="hide">
-        <td colspan="2" style="text-align: center">
-          <!--[if !IE]><!--><i class="fa fa-spinner fa-spin" style="font-size: 20px; color: #DDD"></i><!--<![endif]-->
-          <!--[if IE]><img src="/static/art/spinner.gif"/><![endif]-->
-        </td>
-      </tr>
-    </tfoot>
-  </table>
-    <div class="card-body">
-      <p>
-        ## ${ comps.pagination(page) }
-      </p>
-    </div>
-  </div>
-</div>
-
-<div id="deleteContext" class="modal hide fade">
-  <form id="deleteContextForm" action="${ url('spark:delete_contexts') }" method="POST">
-    ${ csrf_token(request) | n,unicode }
-    <input type="hidden" name="skipTrash" id="skipTrash" value="false"/>
-    <div class="modal-header">
-      <a href="#" class="close" data-dismiss="modal">&times;</a>
-      <h3 id="deleteContextMessage">${_('Confirm action')}</h3>
-    </div>
-    <div class="modal-footer">
-      <input type="button" class="btn" data-dismiss="modal" value="${_('Cancel')}" />
-      <input type="submit" class="btn btn-danger" value="${_('Yes')}"/>
-    </div>
-    <div class="hide">
-      <select name="contexts_selection" data-bind="options: availableSavedContexts, selectedOptions: chosenSavedContexts" multiple="true"></select>
-    </div>
-  </form>
-</div>
-
-
-${ common.createContextModal() }
-
-
-<script src="/static/ext/js/knockout-min.js" type="text/javascript" charset="utf-8"></script>
-<script src="/static/ext/js/knockout.mapping-2.3.2.js" type="text/javascript" charset="utf-8"></script>
-
-<script src="/spark/static/js/spark.vm.js"></script>
-
-<script type="text/javascript" charset="utf-8">
-
-  $(document).ready(function() {
-    var viewModel = {
-        availableSavedContexts : ko.observableArray(${ contexts_json | n,unicode }),
-        chosenSavedContexts : ko.observableArray([])
-    };
-
-    ko.applyBindings(viewModel, $('#deleteContext')[0]);
-
-    var sViewModel = new sparkViewModel();
-    ko.applyBindings(sViewModel, $('#createContextModal')[0]);
-
-    var savedContexts;
-    function createDataTable () {
-      savedContexts = $(".datatables").dataTable({
-        "sDom":"<'row'r>t<'row'<'span8'i><''p>>",
-        "bPaginate":false,
-        "bLengthChange":false,
-        "bInfo":false,
-        "aaSorting":[
-          [1, "desc"]
-        ],
-        "aoColumns":[
-          {"bSortable": false},
-          null
-        ],
-        "oLanguage":{
-          "sEmptyTable":"${_('No data available')}",
-          "sZeroRecords":"${_('No matching records')}",
-        },
-        "bDestroy": true
-      });
-    }
-
-    createDataTable();
-
-    $("#filterInput").on("keyup", function () {
-      savedContexts.fnFilter($(this).val());
-    });
-
-    $("a[data-row-selector='true']").jHueRowSelector();
-
-    $(".selectAll").on("click", function () {
-      if ($(this).attr("checked")) {
-        $(this).removeAttr("checked").removeClass("fa-check");
-        $("." + $(this).data("selectables")).removeClass("fa-check").removeAttr("checked");
-      }
-      else {
-        $(this).attr("checked", "checked").addClass("fa-check");
-        $("." + $(this).data("selectables")).addClass("fa-check").attr("checked", "checked");
-      }
-      toggleActions();
-    });
-
-    $(document).on("click", ".savedCheck", function () {
-      if ($(this).attr("checked")) {
-        $(this).removeClass("fa-check").removeAttr("checked");
-      }
-      else {
-        $(this).addClass("fa-check").attr("checked", "checked");
-      }
-      $(".selectAll").removeAttr("checked").removeClass("fa-check");
-      toggleActions();
-    });
-
-    function toggleActions() {
-      $(".toolbarBtn").attr("disabled", "disabled");
-
-      var selector = $(".hueCheckbox[checked='checked']");
-      if (selector.length == 1) {
-        if (selector.data("edit-url")) {
-          $("#editBtn").removeAttr("disabled").click(function () {
-            location.href = selector.data("edit-url");
-          });
-        }
-        if (selector.data("clone-url")) {
-          $("#cloneBtn").removeAttr("disabled").click(function () {
-            location.href = selector.data("clone-url")
-          });
-        }
-        if (selector.data("history-url")) {
-          $("#historyBtn").removeAttr("disabled").click(function () {
-            location.href = selector.data("history-url")
-          });
-        }
-      }
-      if (selector.length >= 1) {
-        $("#trashContextBtn").removeAttr("disabled");
-        $("#trashContextCaretBtn").removeAttr("disabled");
-        $("#deleteContextBtn").removeAttr("disabled");
-      }
-    }
-
-    function deleteContexts() {
-      viewModel.chosenSavedContexts.removeAll();
-      $(".hueCheckbox[checked='checked']").each(function( index ) {
-        viewModel.chosenSavedContexts.push($(this).data("delete-name"));
-      });
-
-      $("#deleteContext").modal("show");
-    }
-
-    $("#deleteContextBtn").on("click", function () {
-      $("#skipTrash").val(true);
-      $("#deleteContextMessage").text("${ _('Delete the selected contexts?') }");
-      deleteContexts();
-    });
-
-    $(".createContextModalBtn").on("click", function(){
-      $("#createContextModal").modal("show");
-    });
-
-    $("a[data-row-selector='true']").jHueRowSelector();
-
-    $(document).on("created.context", function() {
-      $(".datatables tfoot tr").removeClass("hide");
-      $(".datatables tbody").empty();
-      savedContexts.fnClearTable();
-      $.getJSON("${ url('spark:list_contexts')}", function(data){
-        viewModel.availableSavedContexts(data.contexts);
-        var _h  = "";
-        $(data.contexts).each(function(cnt, ctx){
-          _h += getContextRow(ctx);
-        });
-        $(".datatables").find("tbody").html(_h);
-        createDataTable();
-      });
-      $("#createContextModal").modal("hide");
-      $("#createContextBtn").button("reset");
-      $("input[data-default]").each(function(){
-        $(this).val($(this).data("default"));
-      });
-      $(".datatables tfoot tr").addClass("hide");
-    });
-
-    function getContextRow(context) {
-      return '<tr>' +
-        '<td data-row-selector-exclude="true">' +
-        '<div class="hueCheckbox savedCheck fa" data-delete-name="'+context+'" data-row-selector-exclude="true"></div>' +
-        '</td>' +
-        '<td data-name="'+context+'">'+context+'</td>' +
-      '</tr>';
-    }
-
-  });
-</script>
-
-${ commonfooter(messages) | n,unicode }

+ 0 - 27
apps/spark/src/spark/urls.py

@@ -21,17 +21,6 @@ from django.conf.urls.defaults import patterns, url
 # Views
 urlpatterns = patterns('spark.views',
   url(r'^$', 'editor', name='index'),
-  url(r'^editor/design/(?P<design_id>.+)$', 'editor', name='execute_design'), # For Beeswax
-  url(r'^editor/query/(?P<query_history_id>.+)$', 'editor', name='watch_query_history'), # For Beeswax
-  url(r'^editor/(?P<design_id>.+)?$', 'editor', name='view_job'), # For browser
-  url(r'^editor/(?P<design_id>.+)?$', 'editor', name='editor'),
-  url(r'^editor/(?P<design_id>.+)?$', 'editor', name='execute_query'), # For Beeswax
-  url(r'^list_jobs', 'list_jobs', name='list_jobs'),
-  url(r'^list_contexts', 'list_contexts', name='list_contexts'),
-  url(r'^delete_contexts', 'delete_contexts', name='delete_contexts'),
-  url(r'^list_applications', 'list_applications', name='list_applications'),
-  url(r'^upload_app$', 'upload_app', name='upload_app'),
-  url(r'^download_result/(?P<job_id>.+)?$', 'download_result', name='download_result'),
 )
 
 # APIs
@@ -40,21 +29,5 @@ urlpatterns += patterns('spark.api',
   url(r'^api/execute$', 'execute', name='execute'),
   url(r'^api/check_status', 'check_status', name='check_status'),
   url(r'^api/fetch_result$', 'fetch_result', name='fetch_result'),
-                        
-  url(r'^api/jars$', 'jars', name='jars'),  
-  url(r'^api/contexts$', 'contexts', name='contexts'),
-  url(r'^api/job/(?P<job_id>.+)$', 'job', name='job'),
-  url(r'^api/create_context$', 'create_context', name='create_context'),
-  url(r'^api/delete_context', 'delete_context', name='delete_context'),
-  url(r'^api/save_query/((?P<design_id>\d+)/?)?$', 'save_query', name='save_query'),
 )
 
-urlpatterns += patterns('beeswax.views',
-  url(r'^my_queries$', 'my_queries', name='my_queries'),
-  url(r'^list_designs$', 'list_designs', name='list_designs'),
-  url(r'^list_trashed_designs$', 'list_trashed_designs', name='list_trashed_designs'),
-  url(r'^delete_designs$', 'delete_design', name='delete_design'),
-  url(r'^restore_designs$', 'restore_design', name='restore_design'),
-  url(r'^clone_design/(?P<design_id>\d+)$', 'clone_design', name='clone_design'),
-  url(r'^query_history$', 'list_query_history', name='list_query_history')
-)

+ 0 - 94
apps/spark/src/spark/views.py

@@ -18,22 +18,7 @@
 import json
 import logging
 
-from django.http import HttpResponse
-from django.utils.translation import ugettext as _
-from django.core.urlresolvers import reverse
-
-from desktop.context_processors import get_app_name
 from desktop.lib.django_util import render
-from django.shortcuts import redirect
-
-from beeswax import models as beeswax_models
-from beeswax.views import safe_get_design
-
-from spark.job_server_api import get_api
-from spark.forms import UploadApp
-from desktop.lib.exceptions import StructuredException
-from spark.api import design_to_dict
-
 from spark.decorators import view_error_handler
 
 
@@ -45,82 +30,3 @@ def editor(request):
   return render('editor.mako', request, {
       'notebooks_json': json.dumps([{'snippets': [{'type': 'scala', 'result': {}}]}])
   })
-
-@view_error_handler
-def list_jobs(request):
-  api = get_api(request.user)
-  jobs = api.jobs()
-
-  return render('list_jobs.mako', request, {
-    'jobs': jobs,
-    'jobs_json': json.dumps(jobs)
-  })
-
-@view_error_handler
-def list_contexts(request):
-  api = get_api(request.user)
-  contexts = api.contexts()
-
-  return render('list_contexts.mako', request, {
-    'contexts': contexts,
-    'contexts_json': json.dumps(contexts)
-  })
-
-@view_error_handler
-def delete_contexts(request):
-  if request.method == 'POST':
-    api = get_api(request.user)
-    ids = request.POST.getlist('contexts_selection')
-    for name in ids:
-      api.delete_context(name)
-    return redirect(reverse('spark:list_contexts'))
-  else:
-    return render('confirm.mako', request, {'url': request.path, 'title': _('Delete context(s)?')})
-
-@view_error_handler
-def list_applications(request):
-  api = get_api(request.user)
-  applications = api.jars()
-
-  return render('list_applications.mako', request, {
-    'applications': applications,
-    'applications_json': json.dumps([applications])
-  })
-
-
-def upload_app(request):
-  if request.method != 'POST':
-    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Requires a POST'))
-  response = {
-    'status': -1
-  }
-
-  form = UploadApp(request.POST, request.FILES)
-
-  if form.is_valid():
-    app_name = form.cleaned_data['app_name']
-    try:
-      data = form.cleaned_data['jar_file'].read()
-      api = get_api(request.user)
-      response['status'] = 0
-      response['results'] = api.upload_jar(app_name, data)
-    except ValueError:
-      # No json is returned
-      pass
-  else:
-    response['results'] = form.errors
-
-  return redirect(request.META['HTTP_REFERER'])
-
-@view_error_handler
-def download_result(request, job_id):
-  api = get_api(request.user)
-  result = api.job(job_id)
-
-  mimetype = 'application/json'
-  gen = json.dumps(result['result'])
-
-  resp = HttpResponse(gen, mimetype=mimetype)
-  resp['Content-Disposition'] = 'attachment; filename=query_result.%s' % format
-
-  return resp