Browse Source

HUE-8497 [cluster] Properly submit a remote Spark job

Note: for now display cluster context in any case in multi cluster mode
(in case we only have one remote compute cluster)
Romain Rigaux 7 years ago
parent
commit
d907dc53ff

+ 1 - 1
apps/jobbrowser/src/jobbrowser/urls.py

@@ -53,7 +53,7 @@ urlpatterns = [
 
 
 # V2
 # V2
 urlpatterns += [
 urlpatterns += [
-  url(r'apps$', jobbrowser_views.apps, name='apps'),
+  url(r'apps$', jobbrowser_views.apps, name='jobbrowser.views.apps'),
 ]
 ]
 
 
 urlpatterns += [
 urlpatterns += [

+ 2 - 2
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -139,13 +139,13 @@ class DataEngApi():
     if main_class:
     if main_class:
       job["mainClass"] = main_class
       job["mainClass"] = main_class
 
 
-    return self.submit_jobs(cluster_name, [{'sparkJob': job}])
+    return self.submit_jobs(cluster_name, [{'sparkJob': job, 'name': None, 'failureAction': 'NONE'}]),
 
 
   def submit_yarn_job(self):
   def submit_yarn_job(self):
     return _exec('dataeng', 'submitJobs')
     return _exec('dataeng', 'submitJobs')
 
 
   def submit_jobs(self, cluster_name, jobs):
   def submit_jobs(self, cluster_name, jobs):
-    return _exec('dataeng', 'submitJobs', {'clusterName': cluster_name, 'jobs': json.dumps(jobs)})
+    return _exec('dataeng', 'submitJobs', {'clusterName': cluster_name, 'jobs': jobs})
 
 
   def terminate_job(self, job_id):
   def terminate_job(self, job_id):
     return _exec('dataeng', 'terminateJob', {'jobId': job_id})
     return _exec('dataeng', 'terminateJob', {'jobId': job_id})

+ 9 - 7
desktop/libs/notebook/src/notebook/connectors/dataeng.py

@@ -100,18 +100,20 @@ class DataEngApi(Api):
       AltusDataEngApi(self.user).terminate_job(job_id=job_id)
       AltusDataEngApi(self.user).terminate_job(job_id=job_id)
       response = {'status': 0}
       response = {'status': 0}
     else:
     else:
-      response = {'status': -1, 'message': _('Could not cancel because of unsuccessful submition.')}
+      response = {'status': -1, 'message': _('Could not cancel because of unsuccessful submission.')}
 
 
     return response
     return response
 
 
 
 
   def get_log(self, notebook, snippet, startFrom=0, size=None):
   def get_log(self, notebook, snippet, startFrom=0, size=None):
-    logs = WorkfloadAnalyticsClient(self.user).get_mr_task_attempt_log(
-        operation_execution_id='cedb71ae-0956-42e1-8578-87b9261d4a37',
-        attempt_id='attempt_1499705340501_0045_m_000000_0'
-    )
-
-    return ''.join(re.findall('(?<=>>> Invoking Beeline command line now >>>)(.*?)(?=<<< Invocation of Beeline command completed <<<)', logs['stdout'], re.DOTALL))
+    # Currently no way to get the logs properly easily
+
+    # logs = WorkfloadAnalyticsClient(self.user).get_mr_task_attempt_log(
+    #    operation_execution_id='cedb71ae-0956-42e1-8578-87b9261d4a37',
+    #    attempt_id='attempt_1499705340501_0045_m_000000_0'
+    # )
+    # return ''.join(re.findall('(?<=>>> Invoking Beeline command line now >>>)(.*?)(?=<<< Invocation of Beeline command completed <<<)', logs['stdout'], re.DOTALL))
+    return ''
 
 
 
 
   def progress(self, snippet, logs):
   def progress(self, snippet, logs):

+ 17 - 16
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -978,24 +978,25 @@ ${ sqlSyntaxDropdown.sqlSyntaxDropdown() }
 </script>
 </script>
 
 
 <script type="text/html" id="snippet-header-database-selection">
 <script type="text/html" id="snippet-header-database-selection">
-  <!-- ko if: loadingContext() && (isSqlDialect() || type() == 'spark2') -->
-  <i class="fa fa-spinner fa-spin muted"></i>
-  <!-- /ko -->
-
-  <!-- ko if: !loadingContext() && (isSqlDialect() || type() == 'spark2') -->
   <!-- ko if: window.HAS_MULTI_CLUSTER -->
   <!-- ko if: window.HAS_MULTI_CLUSTER -->
-  <!-- ko if: availableComputes().length > 0 -->
-  <span class="editor-header-title">${ _('Compute') }</span>
-  <div data-bind="component: { name: 'hue-drop-down', params: { value: compute, entries: availableComputes, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Active compute') }' } }" style="display: inline-block"></div>
-  <!-- /ko -->
-  <!-- ko if: availableComputes().length === 0 -->
-  <span class="editor-header-title"><i class="fa fa-warning"></i> ${ _('No computes found') }</span>
-  <!-- /ko -->
-  <!-- /ko -->
+    <!-- ko if: loadingContext() && (isSqlDialect() || type() == 'spark2') -->
+    <i class="fa fa-spinner fa-spin muted"></i>
+    <!-- /ko -->
 
 
-  <!-- ko if: window.HAS_MULTI_CLUSTER -->
-  <span class="editor-header-title">${ _('Namespace') }</span>
-  <div data-bind="component: { name: 'hue-drop-down', params: { value: namespace, entries: availableNamespaces, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Active namespace') }' } }" style="display: inline-block"></div>
+    <!-- ko if: !loadingContext() && (isSqlDialect() || type() == 'spark2') -->
+    <!-- ko if: availableComputes().length > 0 -->
+    <span class="editor-header-title">${ _('Compute') }</span>
+    <div data-bind="component: { name: 'hue-drop-down', params: { value: compute, entries: availableComputes, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Active compute') }' } }" style="display: inline-block"></div>
+    <!-- /ko -->
+    <!-- ko if: availableComputes().length === 0 -->
+    <span class="editor-header-title"><i class="fa fa-warning"></i> ${ _('No computes found') }</span>
+    <!-- /ko -->
+    <!-- /ko -->
+
+    <!-- ko if: availableNamespaces().length > 0 -->
+    <span class="editor-header-title">${ _('Namespace') }</span>
+    <div data-bind="component: { name: 'hue-drop-down', params: { value: namespace, entries: availableNamespaces, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Active namespace') }' } }" style="display: inline-block"></div>
+    <!-- /ko -->
   <!-- /ko -->
   <!-- /ko -->
 
 
   <!-- ko if: isSqlDialect() -->
   <!-- ko if: isSqlDialect() -->