|
|
@@ -81,8 +81,10 @@ class Notebook():
|
|
|
|
|
|
|
|
|
def get_api(user, snippet):
|
|
|
- if snippet['type'] in ('hive', 'impala', 'spark-sql', 'jar'):
|
|
|
+ if snippet['type'] in ('hive', 'impala', 'spark-sql'):
|
|
|
return HS2Api(user)
|
|
|
+ elif snippet['type'] == 'jar':
|
|
|
+ return SparkBatchApi(user)
|
|
|
elif snippet['type'] == 'text':
|
|
|
return TextApi(user)
|
|
|
else:
|
|
|
@@ -171,7 +173,7 @@ class HS2Api():
|
|
|
db = self._get_db(snippet)
|
|
|
|
|
|
handle = self._get_handle(snippet)
|
|
|
- operation = db.get_operation_status(handle)
|
|
|
+ operation = db.get_operation_status(handle)
|
|
|
status = HiveServerQueryHistory.STATE_MAP[operation.operationState]
|
|
|
|
|
|
if status.index in (QueryHistory.STATE.failed.index, QueryHistory.STATE.expired.index):
|
|
|
@@ -294,7 +296,7 @@ class SparkApi():
|
|
|
session = _get_snippet_session(notebook, snippet)
|
|
|
|
|
|
try:
|
|
|
- response = api.submit_statement(session['id'], snippet['statement']) ## --> post all props in json
|
|
|
+ response = api.submit_statement(session['id'], snippet['statement'])
|
|
|
return {
|
|
|
'id': response['id'],
|
|
|
'has_result_set': True,
|
|
|
@@ -405,3 +407,57 @@ class SparkApi():
|
|
|
|
|
|
def _get_jobs(self, log):
|
|
|
return []
|
|
|
+
|
|
|
+
|
|
|
+class SparkBatchApi():
|
|
|
+
|
|
|
+ def __init__(self, user):
|
|
|
+ self.user = user
|
|
|
+
|
|
|
+ def create_session(self, lang):
|
|
|
+ return {
|
|
|
+ 'type': lang,
|
|
|
+ 'id': None
|
|
|
+ }
|
|
|
+
|
|
|
+ def execute(self, notebook, snippet):
|
|
|
+ api = get_spark_api(self.user)
|
|
|
+
|
|
|
+ properties = {
|
|
|
+ 'file': snippet['properties']['app_jar'],
|
|
|
+ 'className': snippet['properties']['class'],
|
|
|
+ 'args': snippet['properties']['arguments'],
|
|
|
+ # pyFiles
|
|
|
+ # files
|
|
|
+ # driverMemory
|
|
|
+ # driverCores
|
|
|
+ # executorMemory
|
|
|
+ # executorCores
|
|
|
+ # archives
|
|
|
+ }
|
|
|
+
|
|
|
+ response = api.submit_batch(properties)
|
|
|
+ return {
|
|
|
+ 'id': response['id'],
|
|
|
+ 'has_result_set': True,
|
|
|
+ }
|
|
|
+
|
|
|
+ def check_status(self, notebook, snippet):
|
|
|
+ api = get_spark_api(self.user)
|
|
|
+
|
|
|
+ response = api.get_batch(snippet['result']['handle']['id'])
|
|
|
+ return {
|
|
|
+ 'status': response['state'],
|
|
|
+ }
|
|
|
+
|
|
|
+ def get_log(self, snippet):
|
|
|
+ api = get_spark_api(self.user)
|
|
|
+
|
|
|
+ response = api.get_batch(snippet['result']['handle']['id'])
|
|
|
+ return '\n'.join(response['lines'])
|
|
|
+
|
|
|
+ def _progress(self, snippet, logs):
|
|
|
+ return 50
|
|
|
+
|
|
|
+ def _get_jobs(self, log):
|
|
|
+ return []
|