浏览代码

HUE-8758 [connectors] Remove hardcoding of connector instances

Can pick up initial ones from ini.
Then manual or API edition.
Romain 6 年之前
父节点
当前提交
5030f84bac
共有 2 个文件被更改,包括 43 次插入47 次删除
  1. 14 14
      desktop/core/src/desktop/conf.py
  2. 29 33
      desktop/core/src/desktop/lib/connectors/api.py

+ 14 - 14
desktop/core/src/desktop/conf.py

@@ -1828,30 +1828,30 @@ CONNECTORS = UnspecifiedConfigSection(
   each=ConfigSection(
   each=ConfigSection(
     help=_("Id of the connector."),
     help=_("Id of the connector."),
     members=dict(
     members=dict(
-      NAME=Config(
-          "name",
+      NICE_NAME=Config(
+          "nice_name",
           help=_("Nice name of the connector to show to the user. Same as id if not specified."),
           help=_("Nice name of the connector to show to the user. Same as id if not specified."),
           default=None,
           default=None,
           type=str,
           type=str,
       ),
       ),
-      TYPE=Config(
-          "type",
-          help=_("Type of cluster, e.g. single, direct, local ini, CM API, Dataeng, Arcus, BigQuery, Presto."),
-          default='direct',
+      DIALECT=Config(
+          "dialect",
+          help=_("The language or type of the integrated service. e.g. MySql, Hive, HDFS..."),
+          default=None,
           type=str,
           type=str,
       ),
       ),
       INTERFACE=Config(
       INTERFACE=Config(
           "interface",
           "interface",
-          help=_("Type of cluster interface"),
-          default='hive',
+          help=_("The class of connector to use to connect to the service."),
+          default=None,
           type=str,
           type=str,
       ),
       ),
-      OPTIONS=Config(
-        key='options',
-        help=_('Specific options for connecting to the server.'),
-        type=coerce_json_dict,
-        default='{}'
-      )
+      SETTINGS=Config(
+          "settings",
+          help=_("Json string of a list of name/value settings to configure the connector. e.g. '{\"name\": \"url\", \"value\": \"mysql://hue:hue@host:3306/hue\"}]'"),
+          default='{}',
+          type=coerce_json_dict,
+      ),
     )
     )
   )
   )
 )
 )

+ 29 - 33
desktop/core/src/desktop/lib/connectors/api.py

@@ -20,6 +20,7 @@ import logging
 
 
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
 
 
+from desktop.conf import has_connectors, CONNECTORS
 from desktop.lib.django_util import JsonResponse, render
 from desktop.lib.django_util import JsonResponse, render
 from desktop.lib.connectors.lib.impala import Impala
 from desktop.lib.connectors.lib.impala import Impala
 from desktop.lib.connectors.lib.hive import Hive
 from desktop.lib.connectors.lib.hive import Hive
@@ -34,7 +35,7 @@ LOG = logging.getLogger(__name__)
 CONNECTOR_TYPES = [{
 CONNECTOR_TYPES = [{
     'nice_name': connector.NAME,
     'nice_name': connector.NAME,
     'dialect': connector.TYPE,
     'dialect': connector.TYPE,
-    'interface': connector.INTERFACE,
+    'interface': connector.INTERFACE, # interfaces = ['int1', 'int2'...]
     'settings': connector.PROPERTIES,
     'settings': connector.PROPERTIES,
     'id': None,
     'id': None,
     'category': 'editor',
     'category': 'editor',
@@ -44,13 +45,14 @@ CONNECTOR_TYPES = [{
     Impala(), Hive()
     Impala(), Hive()
   ]
   ]
 ]
 ]
+
 CONNECTOR_TYPES += [
 CONNECTOR_TYPES += [
   {'nice_name': "Hive Tez", 'dialect': 'hive-tez', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Hive Tez", 'dialect': 'hive-tez', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Hive LLAP", 'dialect': 'hive-llap', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Hive LLAP", 'dialect': 'hive-llap', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Druid", 'dialect': 'sql-druid', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'druid://druid-host.com:8082/druid/v2/sql/'}], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Druid", 'dialect': 'sql-druid', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'druid://druid-host.com:8082/druid/v2/sql/'}], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Kafka SQL", 'dialect': 'kafka-sql', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Kafka SQL", 'dialect': 'kafka-sql', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "SparkSQL", 'dialect': 'spark-sql', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "SparkSQL", 'dialect': 'spark-sql', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "MySQL", 'dialect': 'sql-mysql', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'mysql://username:password@mysq-host:3306/hue'}], 'id': None, 'category': 'editor', 'description': ''},
+  {'nice_name': "MySQL", 'dialect': 'mysql', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'mysql://username:password@mysq-host:3306/hue'}], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Presto", 'dialect': 'presto', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Presto", 'dialect': 'presto', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Athena", 'dialect': 'athena', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Athena", 'dialect': 'athena', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Redshift", 'dialect': 'redshift', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
   {'nice_name': "Redshift", 'dialect': 'redshift', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
@@ -111,37 +113,31 @@ AVAILABLE_CONNECTORS = _group_category_connectors(CONNECTOR_TYPES)
 # TODO: connector groups: if we want one type (e.g. Hive) to show-up with multiple computes and the same saved query.
 # TODO: connector groups: if we want one type (e.g. Hive) to show-up with multiple computes and the same saved query.
 # TODO: type --> name, type --> SQL language, e.g. mysql
 # TODO: type --> name, type --> SQL language, e.g. mysql
 
 
-# connector_type: engine, engine_type: sql, language: hive, hive tez, hiveserver2 + endpoint
-CONNECTOR_INSTANCES = [{
-    'nice_name': 'Impala', 'name': 'impala-1',
-    'dialect': Impala().TYPE, 'interface': Impala().INTERFACE, 'settings': Impala().PROPERTIES, 'is_sql': True, 'id': 1, 'category': 'editor', 'description': ''
-  }, {
-    'nice_name': 'Hive', 'name': 'hive-1',
-    'dialect': Hive().TYPE, 'interface': Hive().INTERFACE, 'settings': Hive().PROPERTIES, 'is_sql': True, 'id': 2, 'category': 'editor', 'description': ''
-  }, {
-    'nice_name': 'Hive c5', 'name': 'hive-2',
-    'dialect': Hive().TYPE, 'interface': Hive().INTERFACE, 'settings': Hive().PROPERTIES, 'is_sql': True, 'id': 3, 'category': 'editor', 'description': ''
-  }, {
-    'nice_name': 'MySQL', 'name': 'mysql-1',
-    'dialect': 'mysql', 'interface': 'sqlalchemy', 'settings': [], 'is_sql': True, 'id': 4, 'category': 'editor', 'description': ''
-  },
-]
-
-CONNECTOR_INSTANCES[0]['settings'] = [
-  {'name': 'server_host', 'value': 'self-service-dw2-2.gce.cloudera.com'},
-  {'name': 'server_port', 'value': '21050'},
-]
-CONNECTOR_INSTANCES[1]['settings'] = [
-  {'name': 'server_host', 'value': 'self-service-dw2-1.gce.cloudera.com'},
-  {'name': 'server_port', 'value': '10000'},
-]
-CONNECTOR_INSTANCES[2]['settings'] = [
-  {'name': 'server_host', 'value': 'nightly6x-unsecure-1.vpc.cloudera.com'},
-  {'name': 'server_port', 'value': '10000'},
-]
-CONNECTOR_INSTANCES[3]['settings'] = [
-  {'name': 'url', 'value': 'mysql://hue:datasshue@romain2:3306/hue'},
-]
+# connector_type: category --> engine, is_sql --> engine_type: sql
+CONNECTOR_INSTANCES = []
+
+if has_connectors():
+  connector_config = CONNECTORS.get()
+
+  for i in connector_config:
+    connector_class = [
+      connector_type
+      for connector_type in CONNECTOR_TYPES
+          if connector_type['dialect'] == connector_config[i].DIALECT.get() and connector_type['interface'] == connector_config[i].INTERFACE.get()
+    ]
+    CONNECTOR_INSTANCES.append({
+        'nice_name': connector_config[i].NICE_NAME.get() or i,
+        'name': i,
+        'dialect': connector_config[i].DIALECT.get(),
+        'interface': connector_config[i].INTERFACE.get(),
+        'settings': connector_config[i].SETTINGS.get(),
+        # From Connector class
+        'is_sql': True,
+        'id': None,
+        'category': connector_class[0]['category'],
+        'description': connector_class[0]['description']
+      }
+    )
 
 
 
 
 def get_connector_classes(request):
 def get_connector_classes(request):