|
@@ -30,30 +30,23 @@ from desktop.lib.exceptions_renderable import PopupException
|
|
|
LOG = logging.getLogger(__name__)
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
-# TODO: persist in DB and migrations
|
|
|
|
|
-# TODO: connector groups: if we want one dialect (e.g. hive) to show-up with multiple/transient computes and the same saved query
|
|
|
|
|
|
|
+class Connector(models.Model):
|
|
|
|
|
+ """
|
|
|
|
|
+ Instance of a connector pointing to an external service: connection
|
|
|
|
|
+ """
|
|
|
|
|
+ name = models.CharField(default='', max_length=255)
|
|
|
|
|
+ description = models.TextField(default='')
|
|
|
|
|
+ dialect = models.CharField(max_length=32, db_index=True, help_text=_t('Type of connector, e.g. hive, mysql... '))
|
|
|
|
|
|
|
|
-CONNECTOR_INSTANCES = None
|
|
|
|
|
-CONNECTOR_IDS = 1
|
|
|
|
|
|
|
+ settings = models.TextField(default='{}')
|
|
|
|
|
+ last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
|
|
|
|
|
|
|
|
-# class Connector(models.Model):
|
|
|
|
|
- # '''
|
|
|
|
|
- # Instance of a connector pointing to an external service: connection
|
|
|
|
|
- # '''
|
|
|
|
|
-# type = models.CharField(max_length=32, db_index=True, help_text=_t('Type of connector, e.g. hive-tez, '))
|
|
|
|
|
-# name = models.CharField(default='', max_length=255)
|
|
|
|
|
-# description = models.TextField(default='')
|
|
|
|
|
-# is_valid # Must be in lib
|
|
|
|
|
|
|
+ organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
|
|
|
|
|
|
|
|
-# settings = models.TextField(default='{}')
|
|
|
|
|
-# last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
|
|
|
|
|
-
|
|
|
|
|
-# organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
|
|
|
|
|
-
|
|
|
|
|
- # class Meta:
|
|
|
|
|
- # verbose_name = _t('connector')
|
|
|
|
|
- # verbose_name_plural = _t('connectors')
|
|
|
|
|
- # unique_together = ('name', 'organization',)
|
|
|
|
|
|
|
+ class Meta:
|
|
|
|
|
+ verbose_name = _t('connector')
|
|
|
|
|
+ verbose_name_plural = _t('connectors')
|
|
|
|
|
+ unique_together = ('name', 'organization',)
|
|
|
|
|
|
|
|
|
|
|
|
|
def _group_category_connectors(connectors):
|
|
def _group_category_connectors(connectors):
|
|
@@ -65,48 +58,63 @@ def _group_category_connectors(connectors):
|
|
|
} for category in CATEGORIES
|
|
} for category in CATEGORIES
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
|
|
+
|
|
|
AVAILABLE_CONNECTORS = _group_category_connectors(CONNECTOR_TYPES)
|
|
AVAILABLE_CONNECTORS = _group_category_connectors(CONNECTOR_TYPES)
|
|
|
|
|
|
|
|
|
|
|
|
|
def _get_installed_connectors(category=None, categories=None, dialect=None, interface=None, user=None):
|
|
def _get_installed_connectors(category=None, categories=None, dialect=None, interface=None, user=None):
|
|
|
- global CONNECTOR_INSTANCES
|
|
|
|
|
- global CONNECTOR_IDS
|
|
|
|
|
- config_connectors = CONNECTORS.get()
|
|
|
|
|
-
|
|
|
|
|
- if CONNECTOR_INSTANCES is None:
|
|
|
|
|
- CONNECTOR_INSTANCES = []
|
|
|
|
|
-
|
|
|
|
|
- for i in config_connectors:
|
|
|
|
|
- connector_types = []
|
|
|
|
|
-
|
|
|
|
|
- for connector_type in CONNECTOR_TYPES:
|
|
|
|
|
- if connector_type['dialect'] == config_connectors[i].DIALECT.get():
|
|
|
|
|
- connector_types.insert(0, connector_type)
|
|
|
|
|
- elif connector_type.get('interface') == config_connectors[i].INTERFACE.get():
|
|
|
|
|
- connector_types.append(connector_type)
|
|
|
|
|
-
|
|
|
|
|
- if not connector_types:
|
|
|
|
|
- LOG.warn('Skipping connector %s as connector dialect %s or interface %s are not installed' % (
|
|
|
|
|
- i, config_connectors[i].DIALECT.get(), config_connectors[i].INTERFACE.get()
|
|
|
|
|
- )
|
|
|
|
|
|
|
+ connector = []
|
|
|
|
|
+ connector_instances = [
|
|
|
|
|
+ {
|
|
|
|
|
+ 'id': connector.id,
|
|
|
|
|
+ 'nice_name': connector.name,
|
|
|
|
|
+ 'description': connector.description,
|
|
|
|
|
+ 'dialect': connector.dialect,
|
|
|
|
|
+ 'interface': None,
|
|
|
|
|
+ 'setting': json.loads(connector.settings),
|
|
|
|
|
+ 'is_demo': False,
|
|
|
|
|
+ }
|
|
|
|
|
+ for connector in Connector.objects.all()
|
|
|
|
|
+ ] + [ # TODO move to samples? or auto
|
|
|
|
|
+ {
|
|
|
|
|
+ 'id': i,
|
|
|
|
|
+ 'nice_name': config_connectors[i].NICE_NAME.get() or i,
|
|
|
|
|
+ 'description': '',
|
|
|
|
|
+ 'dialect': config_connectors[i].DIALECT.get(),
|
|
|
|
|
+ 'interface': config_connectors[i].INTERFACE.get(),
|
|
|
|
|
+ 'settings': config_connectors[i].SETTINGS.get(),
|
|
|
|
|
+ 'is_demo': True,
|
|
|
|
|
+ }
|
|
|
|
|
+ for i in CONNECTORS.get()
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ for connector in connector_instances:
|
|
|
|
|
+ connector_types = []
|
|
|
|
|
+
|
|
|
|
|
+ for connector_type in CONNECTOR_TYPES:
|
|
|
|
|
+ if connector_type['dialect'] == connector['dialect']:
|
|
|
|
|
+ connector_types.insert(0, connector_type)
|
|
|
|
|
+ elif connector_type.get('interface') == connector['interface']:
|
|
|
|
|
+ connector_types.append(connector_type)
|
|
|
|
|
+
|
|
|
|
|
+ if not connector_types:
|
|
|
|
|
+ LOG.warn('Skipping connector %(id)s as connector dialect %(dialect)s or interface %(interface)s are not installed' % (
|
|
|
|
|
+ {'id': i, 'dialect': connector['dialect'], 'interface': connector['interface']}
|
|
|
)
|
|
)
|
|
|
- else:
|
|
|
|
|
- connector_type = connector_types[0]
|
|
|
|
|
- connector = {
|
|
|
|
|
- 'nice_name': config_connectors[i].NICE_NAME.get() or i,
|
|
|
|
|
- 'name': i,
|
|
|
|
|
- 'dialect': config_connectors[i].DIALECT.get(),
|
|
|
|
|
- 'interface': config_connectors[i].INTERFACE.get() or connector_type.get('interface'),
|
|
|
|
|
- 'settings': config_connectors[i].SETTINGS.get(),
|
|
|
|
|
- 'id': CONNECTOR_IDS,
|
|
|
|
|
- 'category': connector_type['category'],
|
|
|
|
|
- 'description': connector_type['description'],
|
|
|
|
|
- 'dialect_properties': connector_type.get('properties', {})
|
|
|
|
|
- }
|
|
|
|
|
- CONNECTOR_INSTANCES.append(connector)
|
|
|
|
|
- CONNECTOR_IDS += 1
|
|
|
|
|
-
|
|
|
|
|
- connectors = CONNECTOR_INSTANCES
|
|
|
|
|
|
|
+ )
|
|
|
|
|
+ else:
|
|
|
|
|
+ connector_type = connector_types[0]
|
|
|
|
|
+ connector.append({
|
|
|
|
|
+ 'nice_name': connector['nice_name'],
|
|
|
|
|
+ 'name': i,
|
|
|
|
|
+ 'dialect': connector['dialect'],
|
|
|
|
|
+ 'interface': connector['interface'],
|
|
|
|
|
+ 'settings': connector['settings'],
|
|
|
|
|
+ 'id': connector['id'],
|
|
|
|
|
+ 'category': connector_type['category'],
|
|
|
|
|
+ 'description': connector_type['description'],
|
|
|
|
|
+ 'dialect_properties': connector_type.get('properties', {})
|
|
|
|
|
+ })
|
|
|
|
|
|
|
|
if categories is not None:
|
|
if categories is not None:
|
|
|
connectors = [connector for connector in connectors if connector['category'] in categories]
|
|
connectors = [connector for connector in connectors if connector['category'] in categories]
|
|
@@ -124,9 +132,7 @@ def _get_installed_connectors(category=None, categories=None, dialect=None, inte
|
|
|
|
|
|
|
|
|
|
|
|
|
def _get_connector_by_id(id):
|
|
def _get_connector_by_id(id):
|
|
|
- global CONNECTOR_INSTANCES
|
|
|
|
|
-
|
|
|
|
|
- instance = [connector for connector in CONNECTOR_INSTANCES if connector['id'] == id]
|
|
|
|
|
|
|
+ instance = [connector for connector in _get_installed_connectors() if connector['id'] == id]
|
|
|
|
|
|
|
|
if instance:
|
|
if instance:
|
|
|
return instance[0]
|
|
return instance[0]
|