浏览代码

[spark] Basic download of table data

Romain Rigaux 10 年之前
父节点
当前提交
47f3eb8
共有 3 个文件被更改,包括 62 次插入6 次删除
  1. 51 0
      apps/spark/src/spark/data_export.py
  2. 10 5
      apps/spark/src/spark/models.py
  3. 1 1
      apps/spark/src/spark/templates/editor.mako

+ 51 - 0
apps/spark/src/spark/data_export.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+
+from django.utils.encoding import smart_str
+
+from desktop.lib import export_csvxls
+
+
+LOG = logging.getLogger(__name__)
+DL_FORMATS = [ 'csv', 'xls' ]
+
+
+def download(api, session, cell, format):
+  if format not in DL_FORMATS:
+    LOG.error('Unknown download format "%s"' % format)
+    return
+
+  content_generator = SparkDataAdapter(api, session, cell)
+  generator = export_csvxls.create_generator(content_generator, format)
+  return export_csvxls.make_response(generator, format, 'script_result')
+
+
+def SparkDataAdapter(api, session, cell):
+  response = api.fetch_data(session, cell)
+
+  content = response['output']
+  data = content['data']
+
+  table = data['application/vnd.livy.table.v1+json']
+
+  rows = table['data']
+  headers = table['headers']
+
+  yield headers, rows

+ 10 - 5
apps/spark/src/spark/models.py

@@ -31,6 +31,7 @@ from beeswax.server.dbms import get_query_server_config, QueryServerException
 from beeswax.views import safe_get_design, save_design, _parse_out_hadoop_jobs
 
 from spark.job_server_api import get_api as get_spark_api
+from spark.data_export import download as spark_download
 
 
 # To move to Editor API
@@ -329,9 +330,6 @@ class SparkApi():
     content = response['output']
 
     if content['status'] == 'ok':
-      # The frontend expects a table, so simulate that by putting our text
-      # into a single cell.
-
       data = content['data']
 
       try:
@@ -346,7 +344,7 @@ class SparkApi():
         meta = [{'name': h['name'], 'type': h['type'], 'comment': ''} for h in headers]
         type = 'table'
 
-      # start_over not supported
+      # Non start_over not supported
       if not start_over:
         data = []
 
@@ -370,7 +368,14 @@ class SparkApi():
       raise QueryError(msg)
     
   def download(self, notebook, snippet, format):
-    return NotImplementedError()
+    try:
+      api = get_spark_api(self.user)
+      session = _get_snippet_session(notebook, snippet)
+      cell = snippet['result']['handle']['id']
+
+      return spark_download(api, session['id'], cell, format)
+    except Exception, e:
+      raise PopupException(e)
 
   def cancel(self, notebook, snippet):
     api = get_spark_api(self.user)

+ 1 - 1
apps/spark/src/spark/templates/editor.mako

@@ -400,7 +400,7 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
               <input type="hidden" name="snippet" data-bind="value: ko.mapping.toJSON($data)"/>
               <input type="hidden" name="format" class="download-format"/>
 
-              <div class="btn-group" data-bind="visible: status() == 'available' && result.hasSomeResults()">
+              <div class="btn-group" data-bind="visible: status() == 'available' && result.hasSomeResults() && result.type() == 'table'">
                 <button class="btn dropdown-toggle" data-toggle="dropdown">
                   <i class="fa fa-download"></i>
                   <i class="fa fa-caret-down"></i>