Преглед изворни кода

HUE-8330 [config] Hook-in connectors to the cluster config

Romain Rigaux пре 6 година
родитељ
комит
72ab357008

+ 5 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -82,6 +82,7 @@ def get(user, query_server=None, cluster=None):
 def get_query_server_config(name='beeswax', server=None, cluster=None):
   LOG.debug("Query cluster %s: %s" % (name, cluster))
 
+
   cluster_config = get_cluster_config(cluster)
 
   if name == 'impala':
@@ -135,12 +136,14 @@ def get_query_server_config(name='beeswax', server=None, cluster=None):
 
 
 def get_cluster_config(cluster=None):
-  if cluster and cluster.get('id') != CLUSTER_ID.get():
+  if cluster and cluster.get('connector'): # Connector interface
+    cluster_config = cluster
+  elif cluster and cluster.get('id') != CLUSTER_ID.get():
     if 'altus:dataware:k8s' in cluster['id']:
       compute_end_point = cluster['compute_end_point'][0] if type(cluster['compute_end_point']) == list else cluster['compute_end_point'] # TODO getting list from left assist
       cluster_config = {'server_host': compute_end_point, 'name': cluster['name']} # TODO get port too
     else:
-      cluster_config = Cluster(user=None).get_config(cluster['id']) # Direct cluster
+      cluster_config = Cluster(user=None).get_config(cluster['id']) # Direct cluster # Deprecated
   else:
     cluster_config = None
 

+ 41 - 21
desktop/core/src/desktop/lib/connectors/api.py

@@ -29,21 +29,23 @@ from desktop.lib.exceptions_renderable import PopupException
 LOG = logging.getLogger(__name__)
 
 
-INSTALLED_CONNECTORS = [
-  {'name': 'Impala', 'type': Impala().NAME, 'settings': Impala().PROPERTIES, 'id': 1, 'category': 'engines', 'description': ''},
-  {'name': 'Hive', 'type': Hive().NAME, 'settings': Hive().PROPERTIES, 'id': 2, 'category': 'engines', 'description': ''},
-]
-
-CONNECTOR_TYPES = [
-  {'name': connector.NAME, 'type': connector.TYPE, 'settings': connector.PROPERTIES, 'id': None, 'category': 'engines', 'description': ''}
-    for connector in [
-      Impala(),
-      Hive()
-    ]
+# TODO: automatically load modules from lib module
+# TODO: offer to white/black list available connector classes
+CONNECTOR_TYPES = [{
+    'name': connector.NAME,
+    'type': connector.TYPE,
+    'interface': connector.INTERFACE,
+    'settings': connector.PROPERTIES,
+    'id': None,
+    'category': 'engines',
+    'description': ''
+    }
+  for connector in [
+    Impala(), Hive()
+  ]
 ]
 
 CONNECTOR_TYPES += [
-  {'name': "SQL Database", 'type': 'sql-alchemy', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
   {'name': "Hive Tez", 'type': 'hive-tez', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'engines', 'description': ''},
   {'name': "Hive LLAP", 'type': 'hive-llap', 'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},], 'id': None, 'category': 'engines', 'description': ''},
   {'name': "Druid", 'type': 'druid', 'settings': [{'name': 'connection_url', 'value': 'druid://druid-host.com:8082/druid/v2/sql/'}], 'id': None, 'category': 'engines', 'description': ''},
@@ -54,6 +56,8 @@ CONNECTOR_TYPES += [
   {'name': "Redshift", 'type': 'redshift', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
   {'name': "Big Query", 'type': 'bigquery', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
   {'name': "Oracle", 'type': 'oracle', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "SQL Database", 'type': 'sql-alchemy', 'settings': {}, 'id': None, 'category': 'engines', 'description': ''},
+  {'name': "SQL Database (JDBC)", 'type': 'sql-jdbc', 'settings': {}, 'id': None, 'category': 'engines', 'description': 'Deprecated: older way to connect to any database.'},
 
   {'name': "HDFS", 'type': 'hdfs', 'settings': {}, 'id': None, 'category': 'browsers', 'description': ''},
   {'name': "YARN", 'type': 'yarn', 'settings': {}, 'id': None, 'category': 'browsers', 'description': ''},
@@ -87,16 +91,28 @@ AVAILABLE_CONNECTORS = {
   } for category in CATEGORIES]
 }
 
+# TODO: persist in DB
+# TODO: remove installed connectors that don't have a connector or are blacklisted
+# TODO: load back from DB and apply type defaults, interface...
+# TODO: connector groups: if we want one type (e.g. Hive) to show-up with multiple computes and the same saved query.
+CONFIGURED_CONNECTORS = [
+  {'name': 'Impala', 'type': Impala().TYPE + '-1', 'connector_name': Impala().TYPE, 'interface': Impala().INTERFACE, 'settings': Impala().PROPERTIES, 'id': 1},
+  {'name': 'Hive', 'type': Hive().TYPE + '-2', 'connector_name': Hive().TYPE, 'interface': Hive().INTERFACE, 'settings': Hive().PROPERTIES, 'id': 2},
+  {'name': 'Hive c5', 'type': Hive().TYPE + '-3', 'connector_name': Hive().TYPE, 'interface': Hive().INTERFACE, 'settings': Hive().PROPERTIES, 'id': 3},
+]
+
 
 def connectors(request):
   return JsonResponse({
-    'connectors': INSTALLED_CONNECTORS
+    'connectors': CONFIGURED_CONNECTORS
   })
 
 
 def new_connector(request, type):
   instance = _get_connector_by_type(type)
 
+  instance['connector_name'] = ''
+
   return JsonResponse({'connector': instance})
 
 
@@ -111,17 +127,21 @@ def update_connector(request):
   global CONNECTOR_IDS
 
   connector = json.loads(request.POST.get('connector'), '{}')
+  saved_as = False
 
   if connector.get('id'):
     instance = _get_connector_by_id(connector['id'])
     instance.update(connector)
   else:
+    saved_as = True
     instance = connector
     instance['id'] = CONNECTOR_IDS
+    instance['connector_name'] = instance['type']
+    instance['type'] = '%s-%s' % (instance['type'], CONNECTOR_IDS)
     CONNECTOR_IDS += 1
-    INSTALLED_CONNECTORS.append(instance)
+    CONFIGURED_CONNECTORS.append(instance)
 
-  return JsonResponse(instance)
+  return JsonResponse({'connector': instance, 'saved_as': saved_as})
 
 
 def _get_connector_by_type(type):
@@ -136,13 +156,13 @@ def _get_connector_by_type(type):
 
 
 def delete_connector(request):
-  global INSTALLED_CONNECTORS
+  global CONFIGURED_CONNECTORS
 
   connector = json.loads(request.POST.get('connector'), '{}')
 
-  size_before = len(INSTALLED_CONNECTORS)
-  INSTALLED_CONNECTORS = filter(lambda _connector: _connector['name'] != connector['name'], INSTALLED_CONNECTORS)
-  size_after = len(INSTALLED_CONNECTORS)
+  size_before = len(CONFIGURED_CONNECTORS)
+  CONFIGURED_CONNECTORS = filter(lambda _connector: _connector['name'] != connector['name'], CONFIGURED_CONNECTORS)
+  size_after = len(CONFIGURED_CONNECTORS)
 
   if size_before == size_after + 1:
     return JsonResponse({})
@@ -151,9 +171,9 @@ def delete_connector(request):
 
 
 def _get_connector_by_id(id):
-  global INSTALLED_CONNECTORS
+  global CONFIGURED_CONNECTORS
 
-  instance = filter(lambda connector: connector['id'] == id, INSTALLED_CONNECTORS)
+  instance = filter(lambda connector: connector['id'] == id, CONFIGURED_CONNECTORS)
 
   if instance:
     return instance[0]

+ 20 - 13
desktop/core/src/desktop/templates/connectors.mako

@@ -61,7 +61,7 @@ else:
           var lowerQuery = self.connectorsFilter().toLowerCase();
           var filteredConnectors = []
           connectors.forEach(function (connector) {
-            var _connector = {"category": connector.category(), "values": []};
+            var _connector = {"category": connector.category, "values": []};
             _connector.values = connector.values.filter(function (subMetricKey) {
               return subMetricKey.name.toLowerCase().indexOf(lowerQuery) !== -1;
             });
@@ -100,13 +100,15 @@ else:
         self.apiHelper.simplePost('/desktop/connectors/api/instance/delete', {'connector': ko.mapping.toJSON(connector)}, {successCallback: function (data) {
           self.section('connectors-page');
           self.fetchConnectors();
+          huePubSub.publish('cluster.config.refresh.config');
         }});
       };
       self.updateConnector = function (connector) {
         self.apiHelper.simplePost('/desktop/connectors/api/instance/update', {'connector': ko.mapping.toJSON(connector)}, {successCallback: function (data) {
-          connector.id(data.id)
+          connector.id(data.connector.id)
           self.section('connectors-page');
           self.fetchConnectors();
+          huePubSub.publish('cluster.config.refresh.config');
         }});
       };
       self.fetchConnectorTypes = function () {
@@ -166,7 +168,7 @@ ${layout.menubar(section='connectors')}
       <tbody data-bind="foreach: $data">
         <tr data-bind="click: function() { $root.instance($data); $root.section('connector-page'); }">
           <td data-bind="text: name"></td>
-          <td data-bind="input: type"></td>
+          <td data-bind="text: connector_name"></td>
         </tr>
       </tbody>
     </table>
@@ -182,19 +184,24 @@ ${layout.menubar(section='connectors')}
 <script type="text/html" id="connector-page">
   <div class="row-fluid">
     <input data-bind="value: name">
-    (<span data-bind="text: type"></span>)
-    <a href="javascript:void(0)" data-bind="click: $root.updateConnector">
-      <!-- ko if: typeof id != 'undefined' -->
-        <!-- ko if: id -->
+    <!-- ko if: typeof id != 'undefined' -->
+      <!-- ko if: id -->
+        (<span data-bind="text: connector_name"></span>)
+        <a href="javascript:void(0)" data-bind="click: $root.updateConnector">
           ${ _('Update') }
-        <!-- /ko -->
-        <!-- ko ifnot: id -->
+        </a>
+        <a href="javascript:void(0)" data-bind="click: $root.deleteConnector">
+          ${ _('Delete') }
+        </a>
+      <!-- /ko -->
+      <!-- ko ifnot: id -->
+        <a href="javascript:void(0)" data-bind="click: $root.updateConnector">
           ${ _('Save') }
-        <!-- /ko -->
+        </a>
       <!-- /ko -->
-    </a>
-    <a href="javascript:void(0)" data-bind="click: $root.deleteConnector">
-      ${ _('Delete') }
+    <!-- /ko -->
+    <a href="javascript:void(0)">
+      ${ _('Test connection') }
     </a>
     <table class="table table-condensed">
       <thead>

+ 3 - 3
desktop/core/src/desktop/urls.py

@@ -154,9 +154,9 @@ dynamic_patterns += [
   url(r'^desktop/api2/doc/share/?$', desktop_api2.share_document),
 
   url(r'^desktop/api2/get_config/?$', desktop_api2.get_config),
-  url(r'^desktop/api2/context/namespaces/(?P<interface>\w+)/?$', desktop_api2.get_context_namespaces),
-  url(r'^desktop/api2/context/computes/(?P<interface>\w+)/?$', desktop_api2.get_context_computes),
-  url(r'^desktop/api2/context/clusters/(?P<interface>\w+)/?$', desktop_api2.get_context_clusters),
+  url(r'^desktop/api2/context/namespaces/(?P<interface>[\w\-]+)/?$', desktop_api2.get_context_namespaces),
+  url(r'^desktop/api2/context/computes/(?P<interface>[\w\-]+)/?$', desktop_api2.get_context_computes),
+  url(r'^desktop/api2/context/clusters/(?P<interface>[\w\-]+)/?$', desktop_api2.get_context_clusters),
   url(r'^desktop/api2/user_preferences/(?P<key>\w+)?$', desktop_api2.user_preferences, name="desktop.api2.user_preferences"),
 
   url(r'^desktop/api2/doc/export/?$', desktop_api2.export_documents),

+ 31 - 9
desktop/libs/notebook/src/notebook/conf.py

@@ -21,7 +21,7 @@ from django.utils.translation import ugettext_lazy as _t
 
 
 from desktop import appmanager
-from desktop.conf import is_oozie_enabled
+from desktop.conf import is_oozie_enabled, CONNECTORS
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_json_dict, coerce_bool, coerce_csv
 
 
@@ -44,7 +44,12 @@ def check_permissions(user, interpreter):
          (interpreter in ('spark', 'pyspark', 'r', 'jar', 'py') and 'spark' not in user_apps) or \
          (interpreter in ('java', 'spark2', 'mapreduce', 'shell', 'sqoop1', 'distcp') and 'oozie' not in user_apps)
 
+
+
 def get_ordered_interpreters(user=None):
+  from desktop.lib.connectors.api import CONFIGURED_CONNECTORS
+  global CONFIGURED_CONNECTORS
+
   if not INTERPRETERS.get():
     _default_interpreters(user)
 
@@ -53,7 +58,7 @@ def get_ordered_interpreters(user=None):
 
   user_interpreters = []
   for interpreter in interpreters:
-    if (check_permissions(user, interpreter)):
+    if check_permissions(user, interpreter):
       pass # Not allowed
     else:
       user_interpreters.append(interpreter)
@@ -62,19 +67,36 @@ def get_ordered_interpreters(user=None):
   if unknown_interpreters:
     raise ValueError("Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s" % unknown_interpreters)
 
-  reordered_interpreters = interpreters_shown_on_wheel + [i for i in user_interpreters if i not in interpreters_shown_on_wheel]
+  if CONNECTORS.IS_ENABLED.get():
+    reordered_interpreters = [{
+        'name': i['name'],
+        'type': i['type'],
+        'interface': i['interface'],
+        'options': {setting['name']: setting['value'] for setting in i['settings']}
+      } for i in CONFIGURED_CONNECTORS
+    ]
+  else:
+    reordered_interpreters = interpreters_shown_on_wheel + [i for i in user_interpreters if i not in interpreters_shown_on_wheel]
+    reordered_interpreters = [{
+        'name': interpreters[i].NAME.get(),
+        'type': i,
+        'interface': interpreters[i].INTERFACE.get(),
+        'options': interpreters[i].OPTIONS.get()
+      } for i in reordered_interpreters
+    ]
 
   return [{
-      "name": interpreters[i].NAME.get(),
-      "type": i,
-      "interface": interpreters[i].INTERFACE.get(),
-      "options": interpreters[i].OPTIONS.get(),
-      "is_sql": interpreters[i].INTERFACE.get() in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy", "hms"],
-      "is_catalog": interpreters[i].INTERFACE.get() in ["hms",]
+      "name": i['name'],
+      "type": i['type'],
+      "interface": i['interface'],
+      "options": i['options'],
+      "is_sql": i['interface'] in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy", "hms"],
+      "is_catalog": i['interface'] in ["hms",]
     }
     for i in reordered_interpreters
   ]
 
+# cf. admin wizard too
 
 INTERPRETERS = UnspecifiedConfigSection(
   "interpreters",

+ 12 - 3
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -27,7 +27,7 @@ from desktop.lib import export_csvxls
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_unicode
 
-from notebook.conf import get_ordered_interpreters
+from notebook.conf import get_ordered_interpreters, CONNECTORS
 
 
 LOG = logging.getLogger(__name__)
@@ -313,13 +313,22 @@ def get_api(request, snippet):
         'is_sql': False
       }]
     else:
-      raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet)
+      raise PopupException(_('Snippet type %(type)s is not configured.') % snippet)
 
   interpreter = interpreter[0]
   interface = interpreter['interface']
 
+
+  if CONNECTORS.IS_ENABLED.get():
+    cluster = {
+      'connector': snippet['type'],
+      'id': interpreter['type'],
+    }
+    snippet['type'] = snippet['type'].split('-', 2)[0]
+    cluster.update(interpreter['options'])
+    print cluster
   # Multi cluster
-  if has_multi_cluster():
+  elif has_multi_cluster():
     cluster = json.loads(request.POST.get('cluster', '""')) # Via Catalog autocomplete API or Notebook create sessions
     if cluster == '""' or cluster == 'undefined':
       cluster = None