Bläddra i källkod

HUE-8591 [cluster] Wire in API for listing and creating k8 clusters

Romain Rigaux 7 år sedan
förälder
incheckning
26c336a

+ 8 - 3
apps/jobbrowser/src/jobbrowser/apis/data_warehouse.py

@@ -17,6 +17,10 @@
 
 import logging
 
+from datetime import datetime
+from dateutil import parser
+
+from django.utils import timezone
 from django.utils.translation import ugettext as _
 
 from notebook.connectors.altus import AnalyticDbApi, DataWarehouse2Api
@@ -24,6 +28,7 @@ from notebook.connectors.altus import AnalyticDbApi, DataWarehouse2Api
 from jobbrowser.apis.base_api import Api
 
 
+
 LOG = logging.getLogger(__name__)
 
 
@@ -50,9 +55,9 @@ class DataWarehouseClusterApi(Api):
         'apiStatus': self._api_status(app['status']),
         'type': 'Altus %(workersGroupSize)s %(instanceType)s %(cdhVersion)s' % app,
         'user': app['clusterName'].split('-', 1)[0],
-        'progress': 100,
+        'progress': app.get('progress', 100),
         'queue': 'group',
-        'duration': 1,
+        'duration': (datetime.now() - parser.parse(app['creationDate']).replace(tzinfo=None)).seconds * 1000,
         'submitted': app['creationDate'],
         'canWrite': True
       } for app in sorted(jobs['clusters'], key=lambda a: a['creationDate'], reverse=True)],
@@ -106,7 +111,7 @@ class DataWarehouseClusterApi(Api):
     return {}
 
   def _api_status(self, status):
-    if status in ['CREATING', 'CREATED']:
+    if status in ['CREATING', 'CREATED', 'ONLINE', 'SCALING_UP', 'SCALING_DOWN', 'STOPPED', 'STARTING']:
       return 'RUNNING'
     elif status in ['ARCHIVING', 'COMPLETED', 'TERMINATING']:
       return 'SUCCEEDED'

+ 64 - 9
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -236,9 +236,23 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
                     <i class="fa fa-refresh"></i>
                   </a>
 
-                  <a class="btn" title="${ _('Create cluster') }" data-bind="visible: $root.cluster() && $root.cluster()['type'].indexOf('altus') >= 0, click: jobs.createCluster">
-                    <i class="fa fa-plus"></i>
+                  <a class="btn" title="${ _('Create cluster') }" data-bind="visible: $root.cluster() && $root.cluster()['type'].indexOf('altus') >= 0, toggle: jobs.createClusterShow">
+                    <!-- ko if: jobs.createClusterShow-->
+                      ${ _('Cancel') }
+                    <!-- /ko -->
+                    <!-- ko ifnot: jobs.createClusterShow-->
+                      ${ _('Create') }
+                    <!-- /ko -->
                   </a>
+                  
+                  <span data-bind="visible: jobs.createClusterShow">
+                    <input type="text" data-bind="clearable: jobs.createClusterName" placeholder="${_('Cluster name')}">
+                    <input type="number" data-bind="value: jobs.createClusterWorkers" class="input-small" placeholder="${_('Size')}">${ _('workers') }
+                    
+                    <button class="btn" data-bind="click: jobs.createCluster, enable: jobs.createClusterName().length > 0 && jobs.createClusterWorkers() > 0" title="${ _('Start creation') }">
+                      <i class="fa fa-plus"></i>
+                    </button>
+                  </span>
                 <!-- /ko -->
 
                 <div data-bind="template: { name: 'job-actions${ SUFFIX }', 'data': jobs }" class="pull-right"></div>
@@ -1104,10 +1118,45 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
 
 
 <script type="text/html" id="dataware-clusters-page${ SUFFIX }">
+
+  <div class="row-fluid">
+    <div data-bind="css:{'span2': !$root.isMini(), 'span12': $root.isMini() }">
+      <div class="sidebar-nav">
+        <ul class="nav nav-list">
+          <li class="nav-header">${ _('Id') }</li>
+          <li><span data-bind="text: id"></span></li>
+          <li class="nav-header">${ _('Name') }</li>
+          <li><span data-bind="text: name"></span></li>
+          <li class="nav-header">${ _('Type') }</li>
+          <li><span data-bind="text: properties['properties']['cdhVersion']"></span></li>
+          <li class="nav-header">${ _('Status') }</li>
+          <li><span data-bind="text: status"></span></li>
+          <li class="nav-header">${ _('Progress') }</li>
+          <li>
+            <span data-bind="text: properties['properties']['workerReplicasOnline']"></span>
+            /
+            <span data-bind="text: properties['properties']['workerReplicas']"></span>
+          </li>
+          <li>
+            <div class="progress-job progress" style="background-color: #FFF; width: 100%" data-bind="css: {'progress-warning': apiStatus() !== 'FAILED' && progress() < 100, 'progress-success': apiStatus() !== 'FAILED' && progress() === 100, 'progress-danger': apiStatus() === 'FAILED'}">
+              <div class="bar" data-bind="style: {'width': '100%'}"></div>
+            </div>
+          </li>
+          <li class="nav-header">${ _('Submitted') }</li>
+          <li><span data-bind="text: submitted"></span></li>
+        </ul>
+      </div>
+    </div>
+    <div data-bind="css:{'span10': !$root.isMini(), 'span12 no-margin': $root.isMini() }">
+      <div class="pull-right" data-bind="template: { name: 'job-actions${ SUFFIX }' }"></div>
+    </div>
+  </div>
+
+  <br>
+
   <button class="btn" title="${ _('Troubleshoot') }" data-bind="click: troubleshoot">
     <i class="fa fa-tachometer"></i> ${ _('Troubleshoot') }
   </button>
-
 </script>
 
 
@@ -2215,7 +2264,7 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
           interface = 'dataeng-clusters';
         }
         else if (/altus:dataware/.test(self.id()) && /:cluster:/.test(self.id())) {
-          interface = 'dataware-clusters';
+          interface = 'dataware2-clusters';
         }
         else if (/[a-z0-9]{16}:[a-z0-9]{16}/.test(self.id())) {
           interface = 'queries';
@@ -2224,7 +2273,7 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
           interface = 'livy-sessions';
         }
 
-        interface = vm.isValidInterface(interface);
+        interface = interface.indexOf('dataeng') || interface.indexOf('dataware') ? interface : vm.isValidInterface(interface); // TODO: support multi cluster selection in isValidInterface
         vm.interface(interface);
 
         lastFetchJobRequest = self._fetchJob(function (data) {
@@ -2271,10 +2320,10 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
             }
 
             if (vm.job().type() == 'SPARK_EXECUTOR') {
-               crumbs.push({'id': vm.job().id(), 'name': vm.job().properties['executor_id'](), 'type': vm.job().type()});
+              crumbs.push({'id': vm.job().id(), 'name': vm.job().properties['executor_id'](), 'type': vm.job().type()});
             }
             else {
-               crumbs.push({'id': vm.job().id(), 'name': vm.job().name(), 'type': vm.job().type()});
+              crumbs.push({'id': vm.job().id(), 'name': vm.job().name(), 'type': vm.job().type()});
             }
 
             vm.resetBreadcrumbs(crumbs);
@@ -2700,6 +2749,10 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
         });
       };
 
+      self.createClusterShow = ko.observable(false);
+      self.createClusterName = ko.observable('');
+      self.createClusterWorkers = ko.observable(3);
+
       self.createCluster = function() {
         if (vm.interface().indexOf('dataeng') != -1) {
           $.post("/metadata/api/dataeng/create_cluster/", {
@@ -2717,12 +2770,13 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
           });
         } else {
           $.post("/metadata/api/analytic_db/create_cluster/", {
-            "cluster_name": "Analytics",
+            "is_k8": vm.interface().indexOf('dataware2-clusters') != -1,
+            "cluster_name": self.createClusterName(),
             "cdh_version": "CDH515",
             "public_key": "public_key",
             "instance_type": "m4.xlarge",
             "environment_name": "crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:jheyming-secure/b4e6d99a-261f-4ada-9b4a-576aa0af8979",
-            "workers_group_size": "3",
+            "workers_group_size": self.createClusterWorkers(),
             "namespace_name": "crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410"
           }, function(data) {
             console.log(ko.mapping.toJSON(data));
@@ -2730,6 +2784,7 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
             self.updateJobs();
           });
         }
+        self.createClusterShow(false);
       }
 
       self.control = function (action) {

+ 6 - 5
desktop/libs/metadata/src/metadata/analytic_db_api.py

@@ -24,7 +24,7 @@ from django.views.decorators.http import require_POST
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.i18n import force_unicode
 
-from notebook.connectors.altus import AnalyticDbApi
+from notebook.connectors.altus import AnalyticDbApi, DataWarehouse2Api
 
 
 LOG = logging.getLogger(__name__)
@@ -49,15 +49,16 @@ def error_handler(view_fn):
 def create_cluster(request):
   response = {'status': -1}
 
-  cluster_name = request.POST.get('cluster_name')
+  is_k8 = request.POST.get('is_k8') == 'true'
+  cluster_name = request.POST.get('cluster_name') or 'Analytic Cluster'
   cdh_version = request.POST.get('cdh_version')
   public_key = request.POST.get('public_key')
   instance_type = request.POST.get('instance_type', "workers_group_size"'')
   environment_name = request.POST.get('environment_name')
-  workers_group_size = request.POST.get('workers_group_size', '3')
+  workers_group_size = int(request.POST.get('workers_group_size', '3'))
   namespace_name = request.POST.get('namespace_name', 'null')
 
-  api = AnalyticDbApi(request.user)
+  api = DataWarehouse2Api(request.user) if is_k8 else AnalyticDbApi(request.user)
   data = api.create_cluster(
       cloud_provider='aws',
       cluster_name=cluster_name,
@@ -73,6 +74,6 @@ def create_cluster(request):
     response['status'] = 0
     response['data'] = data
   else:
-    response['message'] = 'Workload Analytics: %s' % data['details']
+    response['message'] = 'Data Warehouse API: %s' % data['details']
 
   return JsonResponse(response)

+ 29 - 33
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -16,8 +16,9 @@
 # limitations under the License.
 
 import logging
+import json
 
-from datetime import datetime,  timedelta
+from datetime import datetime, timedelta
 
 from django.urls import reverse
 from django.utils.translation import ugettext as _
@@ -276,53 +277,48 @@ class AnalyticDbApi():
 class DataWarehouse2Api():
 
   def __init__(self, user=None):
-    self._api_url = '%s/wa' % K8.API_URL.get().rstrip('/')
+    self._api_url = '%s/dw' % K8.API_URL.get().rstrip('/')
 
     self.user = user
     self._client = HttpClient(self._api_url, logger=LOG)
     self._client.set_verify(False)
     self._root = Resource(self._client)
 
-  def create_cluster(self, cluster_name, cdh_version, cpu_minimum, cpu_maximum, memory_minimum, memory_maximum):
-    params = {
+
+  def create_cluster(self, cloud_provider, cluster_name, cdh_version, public_key, instance_type, environment_name, workers_group_size=3, namespace_name=None,
+        cloudera_manager_username='hue', cloudera_manager_password='hue'):
+    data = {
       'clusterName': cluster_name,
-      'cdhVersion': cdh_version,
-      'cpuMinimum': cpu_minimum,
-      'cpuMaximum': cpu_maximum,
-      'memoryMinimum': memory_minimum,
-      'memoryMaximum': memory_maximum,
+      'cdhVersion': cdh_version or 'CDH6.3',
+      'workerCpuCores': 1,
+      'workerMemoryInGib': 1,
+      'workerReplicas': workers_group_size,
     }
 
-    return _exec('dw', 'createCluster', params)
+    return self._root.post('createCluster', data=json.dumps(data), contenttype="application/json")
+
 
   def list_clusters(self):
-#     {"clusters":[
-#       {"clusterName":"fake-tristan",
-#        "crn":"crn:altus:datawa44eed-a1f3-4935-89c1-71eb56889581",
-#        "creationDate":"2018-09-19T22:27:28.740Z","cdhVersion":"CDH6.3","workerCpuCores":4,"workerMemoryInGib":8,"workerReplicas":4,"workerReplicasOnline":4}]}
-
-    try:
-      clusters = self._root.post('listClusters', contenttype="application/json")
-      for cluster in clusters['clusters']:
-        cluster['workersGroupSize'] = cluster['workerReplicas']
-        cluster['instanceType'] = 'Medium'
-      return clusters
-    except Exception, e:
-      print e
-      return {'clusters': [
-        {'crn': 'crn1', 'clusterName': 'clusterName1', 'status': 'CREATED', 'workersGroupSize': 1, 'instanceType': 'Medium', 'cdhVersion': '6.0', 'creationDate': 'September 14, 2018 12:48 PM'}
-      ]}
+    clusters = self._root.post('listClusters', contenttype="application/json")
+    for cluster in clusters['clusters']:
+      cluster['workersGroupSize'] = cluster['workerReplicas']
+      cluster['instanceType'] = '%(workerCpuCores)s CPU %(workerMemoryInGib)s Memory' % cluster
+      cluster['progress'] = '%(workerReplicasOnline)s / %(workerReplicas)s' % cluster
+      cluster['creationDate'] = str(datetime.now())
+    return clusters
+
 
   def delete_cluster(self, cluster_id):
-    return _exec('dw', 'deleteCluster', {'clusterName': cluster_id})
+    data = json.dumps({'clusterName': cluster_id})
+    return self._root.post('deleteCluster', data=data, contenttype="application/json")
+
 
   def describe_cluster(self, cluster_id):
-    try:
-      return self._root.post('describeCluster', contenttype="application/json")
-    except:
-      return {'cluster':
-        {'crn': 'crn1', 'clusterName': 'clusterName1', 'status': 'CREATED', 'workersGroupSize': 1, 'instanceType': 'Medium', 'cdhVersion': '6.0', 'creationDate': 'September 14, 2018 12:48 PM'}
-      }
+    data = json.dumps({'clusterName': cluster_id})
+    cluster = self._root.post('describeCluster', data=data, contenttype="application/json")
+    cluster['creationDate'] = str(datetime.now())
+    return {'cluster': cluster}
+
 
   def update_cluster(self, cluster_name, cdh_version, cpu_minimum, cpu_maximum, memory_minimum, memory_maximum):
     params = {