Pārlūkot izejas kodu

HUE-8330 [connectors] Refactor to properly list installed and configured connectors

Romain Rigaux 6 gadi atpakaļ
vecāks
revīzija
c0671d615c

+ 1 - 1
apps/about/src/about/templates/admin_wizard.mako

@@ -51,7 +51,7 @@ ${ layout.menubar(section='quick_start') }
            <div class="span2">
             <ul class="nav nav-pills nav-vertical-pills">
               <li class="active"><a href="#step1" class="step">${ _('Step 1:') } <i class="fa fa-cogs"></i> ${ _('Check Configuration') }</a></li>
-              <li><a href="#step2" class="step">${ _('Step 2:') } <i class="fa fa-book"></i> ${ _('Connectors') }</a></li>
+              <li><a href="#step2" class="step">${ _('Step 2:') } <i class="fa fa-icon-exchange"></i> ${ _('Connectors') }</a></li>
               <li><a href="#step3" class="step">${ _('Step 3:') } <i class="fa fa-book"></i> ${ _('Examples') }</a></li>
               <li><a id="lastStep" href="#step4" class="step">${ _('Step 4:') } <i class="fa fa-group"></i> ${ _('Users') }</a></li>
             </ul>

+ 93 - 55
desktop/core/src/desktop/lib/connectors/api.py

@@ -28,83 +28,121 @@ from desktop.lib.exceptions_renderable import PopupException
 
 LOG = logging.getLogger(__name__)
 
+
+INSTALLED_CONNECTORS = [
+  {'name': 'Impala', 'type': Impala().NAME, 'settings': Impala().PROPERTIES, 'id': 1, 'category': 'engines', 'description': ''},
+  {'name': 'Hive', 'type': Hive().NAME, 'settings': Hive().PROPERTIES, 'id': 2, 'category': 'engines', 'description': ''},
+]
+
 CONNECTOR_TYPES = [
-  {'name': connector.NAME, 'settings': connector.PROPERTIES}
+  {'name': connector.NAME, 'type': connector.TYPE, 'settings': connector.PROPERTIES, 'id': None, 'category': 'engines', 'description': ''}
     for connector in [
-      Impala(), Hive()
+      Impala(),
+      Hive()
     ]
 ]
 
-CONNECTORS = {
-  "timestamp": "2019-04-05T23:36:47.533981",
-  "connectors": [
-    {"category": "Query Engines", "values": [
-      {"name": "Impala", "id": 1, "instances": []},
-      {"name": "SQL Database", "id": 2, "instances": []},
-      {"name": "Hive", "id": 3, "instances": [1, 2]},
-      {"name": "Hive Tez", "id": 4, "instances": []},
-      {"name": "Hive LLAP", "id": 5, "instances": []},
-      {"name": "Druid", "id": 6, "instances": []},
-      {"name": "Kafka SQL", "id": 6, "instances": []},
-      {"name": "SparkSQL", "id": 6, "instances": []},
-      {"name": "Presto", "id": 6, "instances": []},
-      {"name": "Athena", "id": 6, "instances": []},
-      {"name": "Redshift", "id": 6, "instances": []},
-      {"name": "Big Query", "id": 6, "instances": []},
-      {"name": "Oracle", "id": 6, "instances": []},
-    ]},
-    {"category": "Browsers", "values": [
-      {"name": "HDFS", "id": 30, "instances": []},
-      {"name": "YARN", "id": 30, "instances": []},
-      {"name": "S3", "id": 31, "instances": []},
-      {"name": "ADLS", "id": 32, "instances": []}
-    ]},
-    {"category": "Catalogs", "values": [
-      {"name": "Navigator", "id": 7, "instances": []},
-      {"name": "Atlas", "id": 8, "instances": []}
-    ]},
-    {"category": "Optimizers", "values": [
-      {"name": "Optimizer", "id": 9, "instances": []}
-    ]},
-    {"category": "Schedulers", "values": [
-      {"name": "Oozie", "id": 10, "instances": []},
-      {"name": "Celery", "id": 11, "instances": []}
-    ]},
-    {"category": "Apps", "values": []},
-    {"category": "Plugins", "values": []},
-  ]
+CONNECTOR_TYPES += [
+  {'name': "SQL Database", 'type': 'sql-alchemy', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Hive Tez", 'type': 'hive-tez', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Hive LLAP", 'type': 'hive-llap', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Druid", 'type': 'druid', 'settings': [{'name': 'connection_url', 'value': 'druid://druid-host.com:8082/druid/v2/sql/'}], 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Kafka SQL", 'type': 'kafka-sql', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "SparkSQL", 'type': 'spark-sql', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Presto", 'type': 'presto', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Athena", 'type': 'athena', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Redshift", 'type': 'redshift', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Big Query", 'type': 'bigquery', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "Oracle", 'type': 'oracle', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+
+  {'name': "HDFS", 'type': 'hdfs', 'settings': {}, 'id': None, 'category': 'browsers', 'description': ''},
+  {'name': "YARN", 'type': 'yarn', 'settings': {}, 'id': None, 'category': 'browsers', 'description': ''},
+  {'name': "S3", 'type': 's3', 'settings': {}, 'id': None, 'category': 'browsers', 'description': ''},
+  {'name': "ADLS", 'type': 'adls-v1', 'settings': {}, 'id': None, 'category': 'browsers', 'description': ''},
+
+  {'name': "Atlas", 'type': 'atlas', 'settings': {}, 'id': None, 'category': 'catalogs', 'description': ''},
+  {'name': "Navigator", 'type': 'navigator', 'settings': {}, 'id': None, 'category': 'catalogs', 'description': ''},
+
+  {'name': "Optimizer", 'type': 'optimizer', 'settings': {}, 'id': None, 'category': 'optimizers', 'description': ''},
+
+  {'name': "Oozie", 'type': 'oozie', 'settings': {}, 'id': None, 'category': 'schedulers', 'description': ''},
+  {'name': "Celery", 'type': 'celery', 'settings': {}, 'id': None, 'category': 'schedulers', 'description': '' },
+]
+
+CATEGORIES = [
+  {"name": "Query Engines", 'type': 'engines', 'description': ''},
+  {"name": "Browsers", 'type': 'browsers', 'description': ''},
+  {"name": "Catalogs", 'type': 'catalogs', 'description': ''},
+  {"name": "Optimizers", 'type': 'optimizers', 'description': ''},
+  {"name": "Schedulers", 'type': 'schedulers', 'description': ''},
+  {"name": "Apps", 'type': 'apps', 'description': ''},
+  {"name": "Plugins", 'type': 'plugins', 'description': ''},
+]
+
+AVAILABLE_CONNECTORS = {
+  "connectors": [{
+    'category': category['name'],
+    'values': [_connector for _connector in CONNECTOR_TYPES if _connector['category'] == category['type']],
+    'description': category['description'],
+  } for category in CATEGORIES]
 }
 
 
 def connectors(request):
   return JsonResponse({
-    'connectors': CONNECTOR_TYPES
+    'connectors': INSTALLED_CONNECTORS
   })
 
 
-def get_connector(request, name):
-  instance = _get_connector(name)
+def new_connector(request, type):
+  instance = _get_connector_by_type(type)
+
+  return JsonResponse({'connector': instance})
+
+
+def get_connector(request, id):
+  instance = _get_connector_by_id(id)
 
   return JsonResponse(instance)
 
+CONNECTOR_IDS = 10
 
 def update_connector(request):
+  global CONNECTOR_IDS
+
   connector = json.loads(request.POST.get('connector'), '{}')
 
-  instance = _get_connector(connector['name'])
-  instance.update(connector)
+  if connector.get('id'):
+    instance = _get_connector_by_id(connector['id'])
+    instance.update(connector)
+  else:
+    instance = connector
+    instance['id'] = CONNECTOR_IDS
+    CONNECTOR_IDS += 1
+    INSTALLED_CONNECTORS.append(instance)
 
   return JsonResponse(instance)
 
 
-def delete_connector(request):
+def _get_connector_by_type(type):
   global CONNECTOR_TYPES
 
+  instance = filter(lambda connector: connector['type'] == type, CONNECTOR_TYPES)
+
+  if instance:
+    return instance[0]
+  else:
+    raise PopupException(_('No connector with the type %s found.') % type)
+
+
+def delete_connector(request):
+  global INSTALLED_CONNECTORS
+
   connector = json.loads(request.POST.get('connector'), '{}')
 
-  size_before = len(CONNECTOR_TYPES)
-  CONNECTOR_TYPES = filter(lambda _connector: _connector['name'] != connector['name'], CONNECTOR_TYPES)
-  size_after = len(CONNECTOR_TYPES)
+  size_before = len(INSTALLED_CONNECTORS)
+  INSTALLED_CONNECTORS = filter(lambda _connector: _connector['name'] != connector['name'], INSTALLED_CONNECTORS)
+  size_after = len(INSTALLED_CONNECTORS)
 
   if size_before == size_after + 1:
     return JsonResponse({})
@@ -112,18 +150,18 @@ def delete_connector(request):
     raise PopupException(_('No connector with the name %(name)s found.') % connector)
 
 
-def _get_connector(name):
-  global CONNECTOR_TYPES
+def _get_connector_by_id(id):
+  global INSTALLED_CONNECTORS
 
-  instance = filter(lambda connector: connector['name'] == name, CONNECTOR_TYPES)
+  instance = filter(lambda connector: connector['id'] == id, INSTALLED_CONNECTORS)
 
   if instance:
     return instance[0]
   else:
-    raise PopupException(_('No connector with the name %s found.') % name)
+    raise PopupException(_('No connector with the id %s found.') % id)
 
 
 def connector_types(request):
-  global CONNECTORS
+  global AVAILABLE_CONNECTORS
 
-  return JsonResponse(CONNECTORS)
+  return JsonResponse(AVAILABLE_CONNECTORS)

+ 2 - 1
desktop/core/src/desktop/lib/connectors/lib/hive.py

@@ -1,6 +1,7 @@
 
 class Hive():
-  NAME = 'hive'
+  NAME = 'Hive'
+  TYPE = 'hive'
 
   VERSION = 1
   APP = 'notebook'

+ 2 - 1
desktop/core/src/desktop/lib/connectors/lib/impala.py

@@ -1,6 +1,7 @@
 
 class Impala():
-  NAME = 'impala'
+  NAME = 'Impala'
+  TYPE = 'impala'
 
   VERSION = 1
   APP = 'notebook'

+ 0 - 12
desktop/core/src/desktop/lib/connectors/models.py

@@ -23,18 +23,6 @@ from django.db.models.query import QuerySet
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
 
-class ImpalaConnector():
-  NAME = 'impala'
-
-  VERSION = 1
-  APP = 'notebook'
-  INTERFACE = 'hiveserver2'
-  PROPERTIES = {
-    'server_host': '',
-    'server_port': '',
-  }
-
-
 class Connectors(models.Model):
   type = models.CharField(max_length=32, db_index=True, help_text=_t('Type of connector, e.g. hive-tez, '))  # Must be in lib
 

+ 2 - 1
desktop/core/src/desktop/lib/connectors/urls.py

@@ -26,7 +26,8 @@ urlpatterns = [
   url(r'^api/instances/?$', api.connectors, name='connectors.api.connectors'),
   url(r'^api/types/?$', api.connector_types, name='connectors.api.connector_types'),
 
-  url(r'^api/instance/get/(?P<name>\w+)$', api.get_connector, name='connectors.api.get_connector'),
+  url(r'^api/instance/new/(?P<type>[\w\-]+)$', api.new_connector, name='connectors.api.new_connector'),
+  url(r'^api/instance/get/(?P<id>\d+)$', api.get_connector, name='connectors.api.get_connector'),
   url(r'^api/instance/delete/?$', api.delete_connector, name='connectors.api.delete_connector'),
   url(r'^api/instance/update/?$', api.update_connector, name='connectors.api.update_connector'),
 ]

+ 32 - 17
desktop/core/src/desktop/templates/connectors.mako

@@ -60,13 +60,13 @@ else:
         if (self.connectorsFilter()) {
           var lowerQuery = self.connectorsFilter().toLowerCase();
           var filteredConnectors = []
-          connectors.forEach(function (connectors) {
-            var _connectors = {"category": connectors.category, "values": []};
-            _connectors.values = connectors.values.filter(function (subMetricKey) {
+          connectors.forEach(function (connector) {
+            var _connector = {"category": connector.category(), "values": []};
+            _connector.values = connector.values.filter(function (subMetricKey) {
               return subMetricKey.name.toLowerCase().indexOf(lowerQuery) !== -1;
             });
-            if (_connectors.values.length > 0) {
-              filteredConnectors.push(_connectors);
+            if (_connector.values.length > 0) {
+              filteredConnectors.push(_connector);
             }
           });
           connectors = filteredConnectors;
@@ -82,12 +82,18 @@ else:
 
       self.fetchConnectors = function () {
         self.apiHelper.simpleGet('/desktop/connectors/api/instances/', {}, {successCallback: function (data) {
-          self.instances(data.connectors);
+          self.instances(ko.mapping.fromJS(data.connectors));
         }});
       };
-      self.fetchConnector = function (name) {
-        self.apiHelper.simpleGet('/desktop/connectors/api/instance/get/' + name, {successCallback: function (data) {
-          self.instance(data.connector);
+      self.newConnector = function (type) {
+        self.apiHelper.simpleGet('/desktop/connectors/api/instance/new/' + type, {}, {successCallback: function (data) {
+          self.instance(ko.mapping.fromJS(data.connector));
+          self.section('connector-page');
+        }});
+      };
+      self.fetchConnector = function (id) {
+        self.apiHelper.simpleGet('/desktop/connectors/api/instance/get/' + id, {}, {successCallback: function (data) {
+          self.instance(ko.mapping.fromJS(data.connector));
         }});
       };
       self.deleteConnector = function (connector) {
@@ -98,6 +104,7 @@ else:
       };
       self.updateConnector = function (connector) {
         self.apiHelper.simplePost('/desktop/connectors/api/instance/update', {'connector': ko.mapping.toJSON(connector)}, {successCallback: function (data) {
+          connector.id(data.id)
           self.section('connectors-page');
           self.fetchConnectors();
         }});
@@ -123,7 +130,7 @@ ${layout.menubar(section='connectors')}
 <div id="connectorsComponents" class="container-fluid">
 
   <a href="javascript:void(0)" data-bind="click: function() { section('connectors-page'); }">
-    Connectors
+    ${ _('Connectors') }
   </a>
 
   <!-- ko if: section() == 'connectors-page' -->
@@ -153,13 +160,13 @@ ${layout.menubar(section='connectors')}
       <thead>
         <tr>
           <th width="30%">${ _('Name') }</th>
-          <th>${ _('') }</th>
+          <th>${ _('Type') }</th>
         </tr>
       </thead>
       <tbody data-bind="foreach: $data">
         <tr data-bind="click: function() { $root.instance($data); $root.section('connector-page'); }">
           <td data-bind="text: name"></td>
-          <td data-bind="input: value"></td>
+          <td data-bind="input: type"></td>
         </tr>
       </tbody>
     </table>
@@ -174,9 +181,17 @@ ${layout.menubar(section='connectors')}
 
 <script type="text/html" id="connector-page">
   <div class="row-fluid">
-    <span data-bind="text: name"></span>
+    <input data-bind="value: name">
+    (<span data-bind="text: type"></span>)
     <a href="javascript:void(0)" data-bind="click: $root.updateConnector">
-      ${ _('Update') }
+      <!-- ko if: typeof id != 'undefined' -->
+        <!-- ko if: id -->
+          ${ _('Update') }
+        <!-- /ko -->
+        <!-- ko ifnot: id -->
+          ${ _('Save') }
+        <!-- /ko -->
+      <!-- /ko -->
     </a>
     <a href="javascript:void(0)" data-bind="click: $root.deleteConnector">
       ${ _('Delete') }
@@ -234,14 +249,14 @@ ${layout.menubar(section='connectors')}
                 <thead>
                   <tr>
                     <th width="30%">${ _('Name') }</th>
-                    <th>${ _('Instances') }</th>
+                    <th>${ _('Description') }</th>
                   </tr>
                 </thead>
                 <!-- ko if: $data.values -->
                 <tbody data-bind="foreach: values">
-                  <tr data-bind="click: function() { $root.instance(name); }">
+                  <tr data-bind="click: function() { $root.newConnector(type); }">
                     <td data-bind="text: name"></td>
-                    <td data-bind="text: instances.length > 0 ? instances.length : ''"></td>
+                    <td data-bind="text: description"></td>
                   </tr>
                 </tbody>
                 <!-- /ko -->