Преглед изворни кода

[spark] Edit session properties

Move interactive doc above the batch API
Romain Rigaux пре 10 година
родитељ
комит
db4dc66648

+ 143 - 116
apps/spark/java/README.rst

@@ -225,6 +225,118 @@ Community
 REST API
 ========
 
+GET /sessions
+-------------
+
+Returns all the active interactive sessions.
+
+Response Body
+^^^^^^^^^^^^^
+
++----------+-----------------+------+
+| name     | description     | type |
++==========+=================+======+
+| sessions | `session`_ list | list |
++----------+-----------------+------+
+
+
+POST /sessions
+--------------
+
+Creates a new interative Scala or Python shell in the cluster.
+
+Request Body
+^^^^^^^^^^^^
+
++----------------+--------------------------------------------------+----------------------------+
+| name           | description                                      | type                       |
++================+==================================================+============================+
+| lang           | session kind (scala or python)                   | `session kind`_ (required) |
++----------------+--------------------------------------------------+----------------------------+
+| file           | archive holding the file                         | path (required)            |
++----------------+--------------------------------------------------+----------------------------+
+| args           | command line arguments                           | list of strings            |
++----------------+--------------------------------------------------+----------------------------+
+| className      | application's java/spark main class              | string                     |
++----------------+--------------------------------------------------+----------------------------+
+| jars           | files to be placed on the java classpath         | list of paths              |
++----------------+--------------------------------------------------+----------------------------+
+| pyFiles        | files to be placed on the PYTHONPATH             | list of paths              |
++----------------+--------------------------------------------------+----------------------------+
+| files          | files to be placed in executor working directory | list of paths              |
++----------------+--------------------------------------------------+----------------------------+
+| driverMemory   | memory for driver                                | string                     |
++----------------+--------------------------------------------------+----------------------------+
+| driverCores    | number of cores used by driver                   | int                        |
++----------------+--------------------------------------------------+----------------------------+
+| executorMemory | memory for executor                              | string                     |
++----------------+--------------------------------------------------+----------------------------+
+| executorCores  | number of cores used by executor                 | int                        |
++----------------+--------------------------------------------------+----------------------------+
+| numExecutors   | number of executor                               | int                        |
++----------------+--------------------------------------------------+----------------------------+
+| archives       |                                                  | list of paths              |
++----------------+--------------------------------------------------+----------------------------+
+
+
+Response Body
+^^^^^^^^^^^^^
+
+The created `Session`_.
+
+
+GET /sessions/{sessionId}
+-------------------------
+
+Return the session information
+
+Response
+^^^^^^^^
+
+The `Session`_.
+
+
+DELETE /sessions/{sessionId}
+-------------------------
+
+Kill the `Session`_ job.
+
+
+GET /sessions/{sessionId}/statements
+------------------------------------
+
+Return all the statements in a session.
+
+Response Body
+^^^^^^^^^^^^^
+
++------------+-------------------+------+
+| name       | description       | type |
++============+===================+======+
+| statements | `statement`_ list | list |
++------------+-------------------+------+
+
+
+POST /sessions/{sessionId}/statements
+-------------------------------------
+
+Execute a statement in a session.
+
+Request Body
+^^^^^^^^^^^^
+
++------+---------------------+--------+
+| name | description         | type   |
++======+=====================+========+
+| code | The code to execute | string |
++------+---------------------+--------+
+
+Response Body
+^^^^^^^^^^^^^
+
+The `statement`_ object.
+
+
 GET /batches
 ------------
 
@@ -314,125 +426,9 @@ DELETE /batches/{batchId}
 Kill the `Batch`_ job.
 
 
-GET /sessions
--------------
-
-Returns all the active interactive sessions.
-
-Response Body
-^^^^^^^^^^^^^
-
-+----------+-----------------+------+
-| name     | description     | type |
-+==========+=================+======+
-| sessions | `session`_ list | list |
-+----------+-----------------+------+
-
-
-POST /sessions
---------------
-
-Request Body
-^^^^^^^^^^^^
-
-+------+--------------+----------------------------+
-| name | description  | type                       |
-+======+==============+============================+
-| lang | session kind | `session kind`_ (required) |
-+------+--------------+----------------------------+
-
-Response Body
-^^^^^^^^^^^^^
-
-The created `Session`_.
-
-
-GET /sessions/{sessionId}
--------------------------
-
-Return the session information
-
-Response
-^^^^^^^^
-
-The `Session`_.
-
-
-DELETE /sessions/{sessionId}
--------------------------
-
-Kill the `Session`_ job.
-
-
-GET /sessions/{sessionId}/statements
-------------------------------------
-
-Return all the statements in a session.
-
-Response Body
-^^^^^^^^^^^^^
-
-+------------+-------------------+------+
-| name       | description       | type |
-+============+===================+======+
-| statements | `statement`_ list | list |
-+------------+-------------------+------+
-
-
-POST /sessions/{sessionId}/statements
--------------------------------------
-
-Execute a statement in a session.
-
-Request Body
-^^^^^^^^^^^^
-
-+------+---------------------+--------+
-| name | description         | type   |
-+======+=====================+========+
-| code | The code to execute | string |
-+------+---------------------+--------+
-
-Response Body
-^^^^^^^^^^^^^
-
-The `statement`_ object.
-
-
 REST Objects
 ============
 
-Batch
------
-
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| name           | description                                                                    | type            |
-+================+================================================================================+=================+
-| file           | archive holding the file                                                       | path (required) |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| args           | command line arguments                                                         | list of strings |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| className      | application's java/spark main class                                            | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| jars           | files to be placed on the java classpath                                       | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| pyFiles        | files to be placed on the PYTHONPATH                                           | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| files          | files to be placed in executor working directory                               | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| driverMemory   | memory for driver                                                              | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| driverCores    | number of cores used by driver (YARN mode only)                                | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| executorMemory | memory for executor                                                            | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| executorCores  | number of cores used by executor                                               | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| numExecutors   | number of executors (YARN mode only)                                           | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| archives       | Archives to be uncompressed in the executor working directory (YARN mode only) | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-
 Session
 -------
 
@@ -543,6 +539,37 @@ Statement Output
 |                 |                   | will be a JSON value             |
 +-----------------+-------------------+----------------------------------+
 
+Batch
+-----
+
++----------------+--------------------------------------------------------------------------------+-----------------+
+| name           | description                                                                    | type            |
++================+================================================================================+=================+
+| file           | archive holding the file                                                       | path (required) |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| args           | command line arguments                                                         | list of strings |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| className      | application's java/spark main class                                            | string          |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| jars           | files to be placed on the java classpath                                       | list of paths   |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| pyFiles        | files to be placed on the PYTHONPATH                                           | list of paths   |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| files          | files to be placed in executor working directory                               | list of paths   |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| driverMemory   | memory for driver                                                              | string          |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| driverCores    | number of cores used by driver (YARN mode only)                                | int             |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| executorMemory | memory for executor                                                            | string          |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| executorCores  | number of cores used by executor                                               | int             |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| numExecutors   | number of executors (YARN mode only)                                           | int             |
++----------------+--------------------------------------------------------------------------------+-----------------+
+| archives       | Archives to be uncompressed in the executor working directory (YARN mode only) | list of paths   |
++----------------+--------------------------------------------------------------------------------+-----------------+
+
 
 License
 =======

+ 5 - 9
apps/spark/src/spark/models.py

@@ -288,17 +288,13 @@ class SparkApi(Api):
 
   def create_session(self, lang='scala', properties=None):
     if properties is None:
-      properties = {
-          'executor_cores': 1, # Some props only in YARN mode
-          'executor_count': 1,
-          'executor_memory': '1G',
-          'driver_cores': 1,
-          'driver_memory': '1G'
-      }
+      properties = {'executorMemory': '500M'}
+
+    properties['kind'] = lang
 
     api = get_spark_api(self.user)
-    print 'TODO: we should use the settings %s for creating the new sessions' % properties
-    response = api.create_session(kind=lang)
+
+    response = api.create_session(**properties)
 
     status = api.get_session(response['id'])
     count = 0

+ 12 - 9
apps/spark/src/spark/static/spark/js/spark.ko.js

@@ -647,10 +647,10 @@ var Notebook = function (vm, notebook) {
   self.newSnippet = function () {
   	var properties = {};
 
-    var addYarnProperties = function() {
-      properties['driver_cores'] = '';
-      properties['executor_cores'] = '';
-      properties['num_executors'] = '';
+    var addSparkYarnProperties = function() {
+      properties['driverCores'] = '';
+      properties['executorCores'] = '';
+      properties['numExecutors'] = '';
       properties['queue'] = '';
       properties['archives'] = [];
     };
@@ -659,15 +659,18 @@ var Notebook = function (vm, notebook) {
       properties['app_jar'] = '';
       properties['class'] = '';
       properties['arguments'] = [];
-      addYarnProperties();
-    } else if (self.selectedSnippet() == 'py') {
+      addSparkYarnProperties();
+    }
+    else if (self.selectedSnippet() == 'py') {
       properties['py_file'] = '';
       properties['arguments'] = [];
-      addYarnProperties();
-    } else if (self.selectedSnippet() == 'hive') {
+      addSparkYarnProperties();
+    }
+    else if (self.selectedSnippet() == 'hive') {
       properties['settings'] = [];
       properties['files'] = [];
-    } else if (self.selectedSnippet() == 'pig') {
+    }
+    else if (self.selectedSnippet() == 'pig') {
       properties['parameters'] = [];
       properties['hadoop_properties'] = [];
       properties['files'] = [];

+ 2 - 1
apps/spark/src/spark/views.py

@@ -25,7 +25,7 @@ from desktop.lib.django_util import render, JsonResponse
 from desktop.lib.json_utils import JSONEncoderForHTML
 from desktop.models import Document2, Document
 
-from spark.conf import LANGUAGES
+from spark.conf import LANGUAGES, LIVY_SERVER_SESSION_KIND
 from spark.decorators import check_document_access_permission,\
   check_document_modify_permission
 from spark.models import Notebook, get_api
@@ -63,6 +63,7 @@ def notebook(request):
           }
       }),
       'autocomplete_base_url': autocomplete_base_url,
+      'is_yarn_mode': LIVY_SERVER_SESSION_KIND.get()
   })