Bläddra i källkod

HUE-8758 [connectors] Add configuration of a HDFS connector

Romain 6 år sedan
förälder
incheckning
d81c27b295

+ 60 - 59
desktop/core/src/desktop/lib/connectors/api.py

@@ -32,56 +32,56 @@ LOG = logging.getLogger(__name__)
 
 # TODO: automatically load modules from lib module
 # TODO: offer to white/black list available connector classes
-CONNECTOR_TYPES = [{
+CONNECTOR_CLASSES = [{
     'nice_name': connector.NAME,
     'dialect': connector.TYPE,
     'interface': connector.INTERFACE, # interfaces = ['int1', 'int2'...]
     'settings': connector.PROPERTIES,
-    'id': None,
     'category': 'editor',
-    'description': ''
-    }
+    'description': '',
+    'properties': {'is_sql': True}
+  }
   for connector in [
     Impala(), Hive()
   ]
 ]
 
-CONNECTOR_TYPES += [
-  {'nice_name': "Hive Tez", 'dialect': 'hive-tez', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Hive LLAP", 'dialect': 'hive-llap', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Druid", 'dialect': 'sql-druid', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'druid://druid-host.com:8082/druid/v2/sql/'}], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Kafka SQL", 'dialect': 'kafka-sql', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "SparkSQL", 'dialect': 'spark-sql', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "MySQL", 'dialect': 'mysql', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'mysql://username:password@mysq-host:3306/hue'}], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Presto", 'dialect': 'presto', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Athena", 'dialect': 'athena', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Redshift", 'dialect': 'redshift', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Big Query", 'dialect': 'bigquery', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Oracle", 'dialect': 'oracle', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "SQL Database", 'dialect': 'sql-alchemy', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "SQL Database (JDBC)", 'dialect': 'sql-jdbc', 'interface': 'sqlalchemy', 'settings': [], 'id': None, 'category': 'editor', 'description': 'Deprecated: older way to connect to any database.'},
+CONNECTOR_CLASSES += [
+  {'nice_name': "Hive Tez", 'dialect': 'hive-tez', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Hive LLAP", 'dialect': 'hive-llap', 'interface': 'hiveserver2', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Druid", 'dialect': 'sql-druid', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'druid://druid-host.com:8082/druid/v2/sql/'}], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Kafka SQL", 'dialect': 'kafka-sql', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "SparkSQL", 'dialect': 'spark-sql', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "MySQL", 'dialect': 'mysql', 'interface': 'sqlalchemy', 'settings': [{'name': 'url', 'value': 'mysql://username:password@mysq-host:3306/hue'}], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Presto", 'dialect': 'presto', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Athena", 'dialect': 'athena', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Redshift", 'dialect': 'redshift', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Big Query", 'dialect': 'bigquery', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "Oracle", 'dialect': 'oracle', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "SQL Database", 'dialect': 'sql-alchemy', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': '', 'properties': {'is_sql': True}},
+  {'nice_name': "SQL Database (JDBC)", 'dialect': 'sql-jdbc', 'interface': 'sqlalchemy', 'settings': [], 'category': 'editor', 'description': 'Deprecated: older way to connect to any database.', 'properties': {'is_sql': True}},
   # solr
   # hbase
   # kafka
 
-  {'nice_name': "PySpark", 'dialect': 'pyspark', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Spark", 'dialect': 'spark', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Pig", 'dialect': 'pig', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
-  {'nice_name': "Java", 'dialect': 'java', 'settings': [], 'id': None, 'category': 'editor', 'description': ''},
+  {'nice_name': "PySpark", 'dialect': 'pyspark', 'settings': [], 'category': 'editor', 'description': '', 'properties': {}},
+  {'nice_name': "Spark", 'dialect': 'spark', 'settings': [], 'category': 'editor', 'description': '', 'properties': {}},
+  {'nice_name': "Pig", 'dialect': 'pig', 'settings': [], 'category': 'editor', 'description': '', 'properties': {}},
+  {'nice_name': "Java", 'dialect': 'java', 'settings': [], 'category': 'editor', 'description': '', 'properties': {}},
 
-  {'nice_name': "HDFS", 'dialect': 'hdfs', 'settings': [], 'id': None, 'category': 'browsers', 'description': ''},
-  {'nice_name': "YARN", 'dialect': 'yarn', 'settings': [], 'id': None, 'category': 'browsers', 'description': ''},
-  {'nice_name': "S3", 'dialect': 's3', 'settings': [], 'id': None, 'category': 'browsers', 'description': ''},
-  {'nice_name': "ADLS", 'dialect': 'adls-v1', 'settings': [], 'id': None, 'category': 'browsers', 'description': ''},
+  {'nice_name': "HDFS", 'dialect': 'hdfs', 'settings': [{'name': 'server_host', 'value': ''}], 'category': 'browsers', 'description': '', 'properties': {}},
+  {'nice_name': "YARN", 'dialect': 'yarn', 'settings': [], 'category': 'browsers', 'description': '', 'properties': {}},
+  {'nice_name': "S3", 'dialect': 's3', 'settings': [], 'category': 'browsers', 'description': '', 'properties': {}},
+  {'nice_name': "ADLS", 'dialect': 'adls-v1', 'settings': [], 'category': 'browsers', 'description': '', 'properties': {}},
 
-  {'nice_name': "Hive Metastore", 'dialect': 'hms', 'settings': [], 'id': None, 'category': 'catalogs', 'description': ''},
-  {'nice_name': "Atlas", 'dialect': 'atlas', 'settings': [], 'id': None, 'category': 'catalogs', 'description': ''},
-  {'nice_name': "Navigator", 'dialect': 'navigator', 'settings': [], 'id': None, 'category': 'catalogs', 'description': ''},
+  {'nice_name': "Hive Metastore", 'dialect': 'hms', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
+  {'nice_name': "Atlas", 'dialect': 'atlas', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
+  {'nice_name': "Navigator", 'dialect': 'navigator', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
 
-  {'nice_name': "Optimizer", 'dialect': 'optimizer', 'settings': [], 'id': None, 'category': 'optimizers', 'description': ''},
+  {'nice_name': "Optimizer", 'dialect': 'optimizer', 'settings': [], 'category': 'optimizers', 'description': '', 'properties': {}},
 
-  {'nice_name': "Oozie", 'dialect': 'oozie', 'settings': [], 'id': None, 'category': 'schedulers', 'description': ''},
-  {'nice_name': "Celery", 'dialect': 'celery', 'settings': [], 'id': None, 'category': 'schedulers', 'description': '' },
+  {'nice_name': "Oozie", 'dialect': 'oozie', 'settings': [], 'category': 'schedulers', 'description': '', 'properties': {}},
+  {'nice_name': "Celery", 'dialect': 'celery', 'settings': [], 'category': 'schedulers', 'description': '', 'properties': {}},
 ]
 
 CATEGORIES = [
@@ -90,7 +90,6 @@ CATEGORIES = [
   {"name": "Catalogs", 'type': 'catalogs', 'description': ''},
   {"name": "Optimizers", 'type': 'optimizers', 'description': ''},
   {"name": "Schedulers", 'type': 'schedulers', 'description': ''},
-  {"name": "Apps", 'type': 'apps', 'description': ''},
   {"name": "Plugins", 'type': 'plugins', 'description': ''},
 ]
 
@@ -104,16 +103,15 @@ def _group_category_connectors(connectors):
   } for category in CATEGORIES
 ]
 
-AVAILABLE_CONNECTORS = _group_category_connectors(CONNECTOR_TYPES)
+AVAILABLE_CONNECTORS = _group_category_connectors(CONNECTOR_CLASSES)
 
 
 # TODO: persist in DB
 # TODO: remove installed connectors that don't have a connector or are blacklisted
 # TODO: load back from DB and apply Category properties, e.g. defaults, interface, category, category_name...
 # TODO: connector groups: if we want one type (e.g. Hive) to show-up with multiple computes and the same saved query.
-# TODO: type --> name, type --> SQL language, e.g. mysql
 
-# connector_type: category --> engine, is_sql --> engine_type: sql
+# connector_class: category --> engine, is_sql --> engine_type: sql
 CONNECTOR_INSTANCES = None
 CONNECTOR_IDS = 1
 
@@ -139,6 +137,7 @@ def new_connector(request, dialect):
   instance = _get_connector_by_type(dialect)
 
   instance['nice_name'] = dialect.title()
+  instance['id'] = None
 
   return JsonResponse({'connector': instance})
 
@@ -164,7 +163,6 @@ def update_connector(request):
     instance['id'] = CONNECTOR_IDS
     instance['nice_name'] = instance['nice_name']
     instance['name'] = '%s-%s' % (instance['dialect'], CONNECTOR_IDS)
-    instance['is_sql'] = instance.get('interface') in ("hiveserver2", "sqlalchemy")
     CONNECTOR_IDS += 1
     CONNECTOR_INSTANCES.append(instance)
 
@@ -172,9 +170,9 @@ def update_connector(request):
 
 
 def _get_connector_by_type(dialect):
-  global CONNECTOR_TYPES
+  global CONNECTOR_CLASSES
 
-  instance = [connector for connector in CONNECTOR_TYPES if connector['dialect'] == dialect]
+  instance = [connector for connector in CONNECTOR_CLASSES if connector['dialect'] == dialect]
 
   if instance:
     return instance[0]
@@ -197,34 +195,37 @@ def delete_connector(request):
     raise PopupException(_('No connector with the name %(name)s found.') % connector)
 
 
-def _get_installed_connectors():
+def _get_installed_connectors(category=None):
   global CONNECTOR_INSTANCES
   global CONNECTOR_IDS
-  connector_config = CONNECTORS.get()
+  config_connectors = CONNECTORS.get()
 
   if CONNECTOR_INSTANCES is None:
     CONNECTOR_INSTANCES = []
-    for i in connector_config:
+    for i in config_connectors:
       connector_class = [
-        connector_type
-        for connector_type in CONNECTOR_TYPES
-            if connector_type['dialect'] == connector_config[i].DIALECT.get() and connector_type['interface'] == connector_config[i].INTERFACE.get()
+        connector_class
+        for connector_class in CONNECTOR_CLASSES
+            if connector_class['dialect'] == config_connectors[i].DIALECT.get() and connector_class['interface'] == config_connectors[i].INTERFACE.get()
       ]
-      CONNECTOR_INSTANCES.append({
-          'nice_name': connector_config[i].NICE_NAME.get() or i,
-          'name': i,
-          'dialect': connector_config[i].DIALECT.get(),
-          'interface': connector_config[i].INTERFACE.get(),
-          'settings': connector_config[i].SETTINGS.get(),
-          # From Connector class
-          'is_sql': True,
-          'id': CONNECTOR_IDS,
-          'category': connector_class[0]['category'],
-          'description': connector_class[0]['description']
-        }
-      )
+      connector = {
+        'nice_name': config_connectors[i].NICE_NAME.get() or i,
+        'name': i,
+        'dialect': config_connectors[i].DIALECT.get(),
+        'interface': config_connectors[i].INTERFACE.get(),
+        'settings': config_connectors[i].SETTINGS.get(),
+        'id': CONNECTOR_IDS,
+        'category': connector_class[0]['category'],
+        'description': connector_class[0]['description']
+      }
+      connector.update(connector_class[0]['properties'])
+      CONNECTOR_INSTANCES.append(connector)
       CONNECTOR_IDS += 1
-  return CONNECTOR_INSTANCES
+
+  if category is not None:
+    return [connector for connector in CONNECTOR_INSTANCES if category == connector['category']]
+  else:
+    return CONNECTOR_INSTANCES
 
 
 def _get_connector_by_id(id):

+ 7 - 2
desktop/libs/hadoop/src/hadoop/conf.py

@@ -21,7 +21,7 @@ import os
 
 from django.utils.translation import ugettext_lazy as _t
 
-from desktop.conf import default_ssl_validate
+from desktop.conf import default_ssl_validate, has_connectors
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_bool
 
 
@@ -54,7 +54,12 @@ UPLOAD_CHUNK_SIZE = Config(
 
 
 def has_hdfs_enabled():
-  return list(HDFS_CLUSTERS.keys())
+  if has_connectors():
+    from desktop.lib.connectors.api import _get_installed_connectors
+    return any([connector for connector in _get_installed_connectors() if connector['dialect'] == 'hdfs'])
+  else:
+    return list(HDFS_CLUSTERS.keys())
+
 
 def get_hadoop_conf_dir_default():
   """ get from environment variable HADOOP_CONF_DIR or "/etc/hadoop/conf" """

+ 8 - 8
desktop/libs/notebook/src/notebook/conf.py

@@ -68,14 +68,14 @@ def get_ordered_interpreters(user=None):
   if has_connectors():
     from desktop.lib.connectors.api import _get_installed_connectors
     reordered_interpreters = [{
-        'name': i['nice_name'],
-        'type': i['name'],
-        'dialect': i['dialect'],
-        'category': i['category'],
-        'is_sql': i['is_sql'],
-        'interface': i['interface'],
-        'options': {setting['name']: setting['value'] for setting in i['settings']}
-      } for i in _get_installed_connectors()
+        'name': connector['nice_name'],
+        'type': connector['name'],
+        'dialect': connector['dialect'],
+        'category': connector['category'],
+        'is_sql': connector['is_sql'],
+        'interface': connector['interface'],
+        'options': {setting['name']: setting['value'] for setting in connector['settings']}
+      } for connector in _get_installed_connectors(category='editor')
     ]
   else:
     reordered_interpreters = interpreters_shown_on_wheel + [i for i in user_interpreters if i not in interpreters_shown_on_wheel]