Эх сурвалжийг харах

HUE-8500 [cluster] Add call to create a remote dataeng cluster

Not all parameters are attached yet
Romain Rigaux 7 жил өмнө
parent
commit
21e9b60

+ 78 - 0
desktop/libs/metadata/src/metadata/dataeng_api.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import json
+
+from django.utils.translation import ugettext as _
+from django.views.decorators.http import require_POST
+
+from desktop.lib.django_util import JsonResponse
+from desktop.lib.i18n import force_unicode
+
+from notebook.connectors.altus import DataEngApi
+
+
+LOG = logging.getLogger(__name__)
+
+
+def error_handler(view_fn):
+  def decorator(*args, **kwargs):
+    try:
+      return view_fn(*args, **kwargs)
+    except Exception, e:
+      LOG.exception(e)
+      response = {
+        'status': -1,
+        'message': force_unicode(e)
+      }
+    return JsonResponse(response, status=500)
+  return decorator
+
+
+@require_POST
+@error_handler
+def create_cluster(request):
+  response = {'status': -1}
+
+  cluster_name = request.POST.get('cluster_name')
+  cdh_version = request.POST.get('cdh_version')
+  public_key = request.POST.get('public_key')
+  instance_type = request.POST.get('instance_type', "workers_group_size"'')
+  environment_name = request.POST.get('environment_name')
+  workers_group_size = request.POST.get('workers_group_size', '3')
+  namespace_name = request.POST.get('namespace_name', 'null')
+
+  api = DataEngApi(request.user)
+  data = api.create_cluster(
+      cloud_provider='aws',
+      cluster_name=cluster_name,
+      cdh_version=cdh_version,
+      public_key=public_key,
+      instance_type=instance_type,
+      environment_name=environment_name,
+      namespace_name=namespace_name,
+      workers_group_size=workers_group_size
+  )
+
+  if data:
+    response['status'] = 0
+    response['data'] = data
+  else:
+    response['message'] = 'Data Engineering: %s' % data['details']
+
+  return JsonResponse(response)

+ 7 - 4
desktop/libs/metadata/src/metadata/urls.py

@@ -16,11 +16,13 @@
 # limitations under the License.
 
 from django.conf.urls import url
-from metadata import catalog_api as metadata_catalog_api, analytic_db_api
+
+from metadata import catalog_api as metadata_catalog_api, analytic_db_api, dataeng_api
 from metadata import optimizer_api as metadata_optimizer_api
 from metadata import workload_analytics_api as metadata_workload_analytics_api
 from metadata import manager_api as metadata_manager_api
 
+
 # Catalog
 urlpatterns = [
   url(r'^api/catalog/search_entities/?$', metadata_catalog_api.search_entities, name='search_entities'),
@@ -81,12 +83,13 @@ urlpatterns += [
 ]
 
 
-# ADB API
+# Altus API
 urlpatterns += [
   url(r'^api/analytic_db/create_cluster/?$', analytic_db_api.create_cluster, name='create_cluster'),
 ]
-
-# Workload Analytics API
+urlpatterns += [
+  url(r'^api/dataeng/create_cluster/?$', dataeng_api.create_cluster, name='create_cluster'),
+]
 urlpatterns += [
   url(r'^api/workload_analytics/get_operation_execution_details/?$', metadata_workload_analytics_api.get_operation_execution_details, name='get_operation_execution_details'),
   url(r'^api/workload_analytics/get_impala_query/?$', metadata_workload_analytics_api.get_impala_query, name='get_impala_query'),

+ 49 - 3
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -16,7 +16,6 @@
 # limitations under the License.
 
 import logging
-import json
 
 from datetime import datetime,  timedelta
 
@@ -162,8 +161,55 @@ class DataEngApi():
 
     return _exec('dataeng', 'listClusters', args)
 
-  def create_cluster(self):
-    return _exec('dataeng', 'createCluster')
+  def create_cluster(self, cloud_provider, cluster_name, cdh_version, public_key, instance_type, environment_name, workers_group_size=3, namespace_name=None):
+    # [--cloudera-manager-username <value>]
+    # [--cloudera-manager-password <value>]
+
+    params = { # cloud_provider: AWS, Azure...
+      'clusterName': cluster_name,
+      'cdhVersion': cdh_version,
+      'publicKey': public_key,
+      'instanceType': instance_type,
+      'environmentName': environment_name,
+      'workersGroupSize': workers_group_size,
+      #'automaticTerminationCondition': "EMPTY_JOB_QUEUE"
+    }
+
+    if namespace_name:
+      params['namespaceName'] = namespace_name
+
+    params = {
+      u'additionalClusterResourceTags': [],
+      u'automaticTerminationCondition': u'EMPTY_JOB_QUEUE',
+      u'cdhVersion': u'CDH514',
+      u'clouderaManagerPassword': u'guest',
+      u'clouderaManagerUsername': u'guest',
+      u'clusterName': u'analytics4',
+      u'computeWorkersConfiguration': {
+        u'bidUSDPerHr': 0,
+        u'groupSize': 0,
+        u'useSpot': False
+      },
+      u'environmentName': u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946',
+      u'instanceBootstrapScript': u'',
+      u'instanceType': u'm4.xlarge',
+      u'jobSubmissionGroupName': u'',
+      u'jobs': [{
+        u'failureAction': u'INTERRUPT_JOB_QUEUE',
+        u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
+        u'sparkJob': {u'applicationArguments': [u'filesystems3.conf'],
+          u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
+          u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
+          u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'}
+        }],
+      u'namespaceName': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
+      u'publicKey': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDuTEfNIW8LEcVgprUrourbYjoW1RaTLhfzPnnBjJrg14koQrosl+s9phrpBBLTWmQuQdvy9iC2ma//gY5nz/7e+QuaeENhhoEiZn1PDBbFakD/AOjZXIu6DTEgCrOeXsQauFZKOkcFvrBGJC0qigYU3b8Eys4cun3RQ4S9WkDW6538wOSnsm6sXcL84KqbH+ay5gTk+lz3bi/6plALZMItbRz9IulXnLM4QfCwMxXTU/IjtnT+ltZVvKsWpfvDQ3Oyu/a6gK369iXcSP0e07KAzWiv2WYX46sNzZ8+de9ho1/VMaXnI4WrooV9lxByKWD+WsXkqtctT16VfxpX8CeR romain@unreal\n',
+      u'serviceType': u'SPARK',
+      u'workersConfiguration': {},
+      u'workersGroupSize': u'3'
+  }
+
+    return _exec('dataeng', 'createAWSCluster', params)
 
   def delete_cluster(self):
     return _exec('dataeng', 'deleteCluster')