Selaa lähdekoodia

HUE-8500 [spark] Implement support for Spark parameters in the editor

Romain Rigaux 7 vuotta sitten
vanhempi
commit
c69a045

+ 8 - 5
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -132,13 +132,14 @@ class DataEngApi():
     job = {
       "jars": jars if jars else [],
       "applicationArguments": arguments if arguments else [],
-      #"sparkArguments": "string",
       #"propertiesFile": "string"
     }
+    if spark_arguments:
+      job['sparkArguments'] = ' '.join(spark_arguments)
     if main_class:
       job["mainClass"] = main_class
 
-    return self.submit_jobs(cluster_name, [{'sparkJob': job, 'name': None, 'failureAction': 'NONE'}]),
+    return self.submit_jobs(cluster_name, [{'sparkJob': job, 'name': None, 'failureAction': 'NONE'}])
 
   def submit_yarn_job(self):
     return _exec('dataeng', 'submitJobs')
@@ -180,7 +181,7 @@ class DataEngApi():
 
     params = {
       u'additionalClusterResourceTags': [],
-      u'automaticTerminationCondition': u'EMPTY_JOB_QUEUE',
+      u'automaticTerminationCondition': u'NONE',
       u'cdhVersion': u'CDH514',
       u'clouderaManagerPassword': u'guest',
       u'clouderaManagerUsername': u'guest',
@@ -197,11 +198,13 @@ class DataEngApi():
       u'jobs': [{
         u'failureAction': u'INTERRUPT_JOB_QUEUE',
         u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
-        u'sparkJob': {u'applicationArguments': [u'filesystems3.conf'],
+        u'sparkJob': {
+          u'applicationArguments': [u'filesystems3.conf'],
           u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
           u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
           u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'}
-        }],
+        }
+      ],
       u'namespaceName': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
       u'publicKey': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDuTEfNIW8LEcVgprUrourbYjoW1RaTLhfzPnnBjJrg14koQrosl+s9phrpBBLTWmQuQdvy9iC2ma//gY5nz/7e+QuaeENhhoEiZn1PDBbFakD/AOjZXIu6DTEgCrOeXsQauFZKOkcFvrBGJC0qigYU3b8Eys4cun3RQ4S9WkDW6538wOSnsm6sXcL84KqbH+ay5gTk+lz3bi/6plALZMItbRz9IulXnLM4QfCwMxXTU/IjtnT+ltZVvKsWpfvDQ3Oyu/a6gK369iXcSP0e07KAzWiv2WYX46sNzZ8+de9ho1/VMaXnI4WrooV9lxByKWD+WsXkqtctT16VfxpX8CeR romain@unreal\n',
       u'serviceType': u'SPARK',

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/dataeng.py

@@ -49,7 +49,7 @@ class DataEngApi(Api):
           jars=snippet['properties']['jars'],
           main_class=snippet['properties']['class'],
           arguments=snippet['properties']['spark_arguments'],
-#           spark_arguments,
+          spark_arguments=snippet['properties']['spark_opts'],
 #           properties_file
       )
     else:

+ 1 - 0
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -1082,6 +1082,7 @@ ${ sqlSyntaxDropdown.sqlSyntaxDropdown() }
         <!-- ko template: { if: typeof properties().functions != 'undefined', name: 'property', data: { type: 'functions', label: '${ _ko('Functions') }', value: properties().functions, visibleObservable: settingsVisible, title: '${ _ko('UDFs name and class') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().settings != 'undefined', name: 'property', data: { type: 'settings', label: '${ _ko('Settings') }', value: properties().settings, visibleObservable: settingsVisible, title: '${ _ko('Properties') }'}} --><!-- /ko -->
 
+        <!-- ko template: { if: typeof properties().spark_opts != 'undefined', name: 'property', data: { type: 'csv', label: '${ _ko('Spark Arguments') }', value: properties().spark_opts, title: '${ _ko('Names and values of Spark parameters') }', placeholder: '${ _ko('e.g. --executor-memory 20G --num-executors 50') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().parameters != 'undefined', name: 'property', data: { type: 'csv', label: '${ _ko('Parameters') }', value: properties().parameters, title: '${ _ko('Names and values of Pig parameters and options') }', placeholder: '${ _ko('e.g. input /user/data, -param input=/user/data, -optimizer_off SplitFilter, -verbose') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().hadoopProperties != 'undefined', name: 'property', data: { type: 'csv', label: '${ _ko('Hadoop properties') }', value: properties().hadoopProperties, title: '${ _ko('Name and values of Hadoop properties') }', placeholder: '${ _ko('e.g. mapred.job.queue.name=production, mapred.map.tasks.speculative.execution=false') }'}} --><!-- /ko -->
         <!-- ko template: { if: typeof properties().resources != 'undefined', name: 'property', data: { type: 'csv-hdfs-files', label: '${ _ko('Resources') }', value: properties().resources, title: '${ _ko('HDFS Files or compressed files') }', placeholder: '${ _ko('e.g. /tmp/file, /tmp.file.zip') }'}} --><!-- /ko -->