|
@@ -16,7 +16,6 @@
|
|
|
# limitations under the License.
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
import logging
|
|
import logging
|
|
|
-import json
|
|
|
|
|
|
|
|
|
|
from datetime import datetime, timedelta
|
|
from datetime import datetime, timedelta
|
|
|
|
|
|
|
@@ -162,8 +161,55 @@ class DataEngApi():
|
|
|
|
|
|
|
|
return _exec('dataeng', 'listClusters', args)
|
|
return _exec('dataeng', 'listClusters', args)
|
|
|
|
|
|
|
|
- def create_cluster(self):
|
|
|
|
|
- return _exec('dataeng', 'createCluster')
|
|
|
|
|
|
|
+ def create_cluster(self, cloud_provider, cluster_name, cdh_version, public_key, instance_type, environment_name, workers_group_size=3, namespace_name=None):
|
|
|
|
|
+ # [--cloudera-manager-username <value>]
|
|
|
|
|
+ # [--cloudera-manager-password <value>]
|
|
|
|
|
+
|
|
|
|
|
+ params = { # cloud_provider: AWS, Azure...
|
|
|
|
|
+ 'clusterName': cluster_name,
|
|
|
|
|
+ 'cdhVersion': cdh_version,
|
|
|
|
|
+ 'publicKey': public_key,
|
|
|
|
|
+ 'instanceType': instance_type,
|
|
|
|
|
+ 'environmentName': environment_name,
|
|
|
|
|
+ 'workersGroupSize': workers_group_size,
|
|
|
|
|
+ #'automaticTerminationCondition': "EMPTY_JOB_QUEUE"
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if namespace_name:
|
|
|
|
|
+ params['namespaceName'] = namespace_name
|
|
|
|
|
+
|
|
|
|
|
+ params = {
|
|
|
|
|
+ u'additionalClusterResourceTags': [],
|
|
|
|
|
+ u'automaticTerminationCondition': u'EMPTY_JOB_QUEUE',
|
|
|
|
|
+ u'cdhVersion': u'CDH514',
|
|
|
|
|
+ u'clouderaManagerPassword': u'guest',
|
|
|
|
|
+ u'clouderaManagerUsername': u'guest',
|
|
|
|
|
+ u'clusterName': u'analytics4',
|
|
|
|
|
+ u'computeWorkersConfiguration': {
|
|
|
|
|
+ u'bidUSDPerHr': 0,
|
|
|
|
|
+ u'groupSize': 0,
|
|
|
|
|
+ u'useSpot': False
|
|
|
|
|
+ },
|
|
|
|
|
+ u'environmentName': u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946',
|
|
|
|
|
+ u'instanceBootstrapScript': u'',
|
|
|
|
|
+ u'instanceType': u'm4.xlarge',
|
|
|
|
|
+ u'jobSubmissionGroupName': u'',
|
|
|
|
|
+ u'jobs': [{
|
|
|
|
|
+ u'failureAction': u'INTERRUPT_JOB_QUEUE',
|
|
|
|
|
+ u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
|
|
|
|
|
+ u'sparkJob': {u'applicationArguments': [u'filesystems3.conf'],
|
|
|
|
|
+ u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
|
|
|
|
|
+ u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
|
|
|
|
|
+ u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'}
|
|
|
|
|
+ }],
|
|
|
|
|
+ u'namespaceName': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
|
|
|
|
|
+ u'publicKey': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDuTEfNIW8LEcVgprUrourbYjoW1RaTLhfzPnnBjJrg14koQrosl+s9phrpBBLTWmQuQdvy9iC2ma//gY5nz/7e+QuaeENhhoEiZn1PDBbFakD/AOjZXIu6DTEgCrOeXsQauFZKOkcFvrBGJC0qigYU3b8Eys4cun3RQ4S9WkDW6538wOSnsm6sXcL84KqbH+ay5gTk+lz3bi/6plALZMItbRz9IulXnLM4QfCwMxXTU/IjtnT+ltZVvKsWpfvDQ3Oyu/a6gK369iXcSP0e07KAzWiv2WYX46sNzZ8+de9ho1/VMaXnI4WrooV9lxByKWD+WsXkqtctT16VfxpX8CeR romain@unreal\n',
|
|
|
|
|
+ u'serviceType': u'SPARK',
|
|
|
|
|
+ u'workersConfiguration': {},
|
|
|
|
|
+ u'workersGroupSize': u'3'
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return _exec('dataeng', 'createAWSCluster', params)
|
|
|
|
|
|
|
|
def delete_cluster(self):
|
|
def delete_cluster(self):
|
|
|
return _exec('dataeng', 'deleteCluster')
|
|
return _exec('dataeng', 'deleteCluster')
|