|
@@ -22,7 +22,7 @@ import catalogUtils from 'catalog/catalogUtils';
|
|
|
import DataCatalogEntry from 'catalog/dataCatalogEntry';
|
|
import DataCatalogEntry from 'catalog/dataCatalogEntry';
|
|
|
import GeneralDataCatalog from 'catalog/generalDataCatalog';
|
|
import GeneralDataCatalog from 'catalog/generalDataCatalog';
|
|
|
import MultiTableEntry from 'catalog/multiTableEntry';
|
|
import MultiTableEntry from 'catalog/multiTableEntry';
|
|
|
-import { fetchPopularity } from './optimizer/optimizerApiHelper';
|
|
|
|
|
|
|
+import { getOptimizer } from './optimizer/optimizer';
|
|
|
|
|
|
|
|
const STORAGE_POSTFIX = window.LOGGED_USERNAME;
|
|
const STORAGE_POSTFIX = window.LOGGED_USERNAME;
|
|
|
const DATA_CATALOG_VERSION = 5;
|
|
const DATA_CATALOG_VERSION = 5;
|
|
@@ -124,12 +124,14 @@ const mergeMultiTableEntry = function(multiTableCatalogEntry, storeEntry) {
|
|
|
class DataCatalog {
|
|
class DataCatalog {
|
|
|
/**
|
|
/**
|
|
|
* @param {string} sourceType
|
|
* @param {string} sourceType
|
|
|
|
|
+ * @param {Connector} connector
|
|
|
*
|
|
*
|
|
|
* @constructor
|
|
* @constructor
|
|
|
*/
|
|
*/
|
|
|
- constructor(sourceType) {
|
|
|
|
|
|
|
+ constructor(sourceType, connector) {
|
|
|
const self = this;
|
|
const self = this;
|
|
|
self.sourceType = sourceType;
|
|
self.sourceType = sourceType;
|
|
|
|
|
+ self.connector = connector;
|
|
|
self.entries = {};
|
|
self.entries = {};
|
|
|
self.temporaryEntries = {};
|
|
self.temporaryEntries = {};
|
|
|
self.multiTableEntries = {};
|
|
self.multiTableEntries = {};
|
|
@@ -161,8 +163,12 @@ class DataCatalog {
|
|
|
* @return {boolean}
|
|
* @return {boolean}
|
|
|
*/
|
|
*/
|
|
|
canHaveOptimizerMeta() {
|
|
canHaveOptimizerMeta() {
|
|
|
- const self = this;
|
|
|
|
|
- return HAS_OPTIMIZER && (self.sourceType === 'hive' || self.sourceType === 'impala');
|
|
|
|
|
|
|
+ return (
|
|
|
|
|
+ HAS_OPTIMIZER &&
|
|
|
|
|
+ this.connector &&
|
|
|
|
|
+ this.connector.optimizer &&
|
|
|
|
|
+ this.connector.optimizer !== 'off'
|
|
|
|
|
+ );
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -258,6 +264,7 @@ class DataCatalog {
|
|
|
* @param {Object} options
|
|
* @param {Object} options
|
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
|
|
|
+ * @param {Connector} options.connector
|
|
|
* @param {string[][]} options.paths
|
|
* @param {string[][]} options.paths
|
|
|
* @param {boolean} [options.silenceErrors] - Default true
|
|
* @param {boolean} [options.silenceErrors] - Default true
|
|
|
* @param {boolean} [options.cancellable] - Default false
|
|
* @param {boolean} [options.cancellable] - Default false
|
|
@@ -313,10 +320,11 @@ class DataCatalog {
|
|
|
const loadDeferred = $.Deferred();
|
|
const loadDeferred = $.Deferred();
|
|
|
if (pathsToLoad.length) {
|
|
if (pathsToLoad.length) {
|
|
|
cancellablePromises.push(
|
|
cancellablePromises.push(
|
|
|
- fetchPopularity({
|
|
|
|
|
- silenceErrors: options.silenceErrors,
|
|
|
|
|
- paths: pathsToLoad
|
|
|
|
|
- })
|
|
|
|
|
|
|
+ getOptimizer(options.connector)
|
|
|
|
|
+ .fetchPopularity({
|
|
|
|
|
+ silenceErrors: options.silenceErrors,
|
|
|
|
|
+ paths: pathsToLoad
|
|
|
|
|
+ })
|
|
|
.done(data => {
|
|
.done(data => {
|
|
|
const perTable = {};
|
|
const perTable = {};
|
|
|
|
|
|
|
@@ -415,6 +423,7 @@ class DataCatalog {
|
|
|
* @param {string} options.name
|
|
* @param {string} options.name
|
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
|
|
|
+ * @param {Connector} options.connector
|
|
|
*
|
|
*
|
|
|
* @param {Object[]} options.columns
|
|
* @param {Object[]} options.columns
|
|
|
* @param {string} options.columns[].name
|
|
* @param {string} options.columns[].name
|
|
@@ -715,6 +724,7 @@ class DataCatalog {
|
|
|
* @param {Object} options
|
|
* @param {Object} options
|
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
|
|
|
+ * @param {Connector} options.connector
|
|
|
* @param {string[][]} options.paths
|
|
* @param {string[][]} options.paths
|
|
|
*
|
|
*
|
|
|
* @return {Promise}
|
|
* @return {Promise}
|
|
@@ -732,7 +742,11 @@ class DataCatalog {
|
|
|
if (!cacheEnabled) {
|
|
if (!cacheEnabled) {
|
|
|
deferred
|
|
deferred
|
|
|
.resolve(
|
|
.resolve(
|
|
|
- new MultiTableEntry({ identifier: identifier, dataCatalog: self, paths: options.paths })
|
|
|
|
|
|
|
+ new MultiTableEntry({
|
|
|
|
|
+ identifier: identifier,
|
|
|
|
|
+ dataCatalog: self,
|
|
|
|
|
+ paths: options.paths
|
|
|
|
|
+ })
|
|
|
)
|
|
)
|
|
|
.promise();
|
|
.promise();
|
|
|
} else {
|
|
} else {
|
|
@@ -752,7 +766,11 @@ class DataCatalog {
|
|
|
.catch(error => {
|
|
.catch(error => {
|
|
|
console.warn(error);
|
|
console.warn(error);
|
|
|
deferred.resolve(
|
|
deferred.resolve(
|
|
|
- new MultiTableEntry({ identifier: identifier, dataCatalog: self, paths: options.paths })
|
|
|
|
|
|
|
+ new MultiTableEntry({
|
|
|
|
|
+ identifier: identifier,
|
|
|
|
|
+ dataCatalog: self,
|
|
|
|
|
+ paths: options.paths
|
|
|
|
|
+ })
|
|
|
);
|
|
);
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
@@ -795,15 +813,17 @@ const sourceBoundCatalogs = {};
|
|
|
* Helper function to get the DataCatalog instance for a given data source.
|
|
* Helper function to get the DataCatalog instance for a given data source.
|
|
|
*
|
|
*
|
|
|
* @param {string} sourceType
|
|
* @param {string} sourceType
|
|
|
|
|
+ * @param {Connector} connector
|
|
|
|
|
+ *
|
|
|
* @return {DataCatalog}
|
|
* @return {DataCatalog}
|
|
|
*/
|
|
*/
|
|
|
-const getCatalog = function(sourceType) {
|
|
|
|
|
|
|
+const getCatalog = function(sourceType, connector) {
|
|
|
if (!sourceType) {
|
|
if (!sourceType) {
|
|
|
throw new Error('getCatalog called without sourceType');
|
|
throw new Error('getCatalog called without sourceType');
|
|
|
}
|
|
}
|
|
|
return (
|
|
return (
|
|
|
sourceBoundCatalogs[sourceType] ||
|
|
sourceBoundCatalogs[sourceType] ||
|
|
|
- (sourceBoundCatalogs[sourceType] = new DataCatalog(sourceType))
|
|
|
|
|
|
|
+ (sourceBoundCatalogs[sourceType] = new DataCatalog(sourceType, connector))
|
|
|
);
|
|
);
|
|
|
};
|
|
};
|
|
|
|
|
|
|
@@ -818,6 +838,7 @@ export default {
|
|
|
* @param {string} options.sourceType
|
|
* @param {string} options.sourceType
|
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
|
|
|
+ * @param {Connector} options.connector
|
|
|
* @param {string} options.name
|
|
* @param {string} options.name
|
|
|
*
|
|
*
|
|
|
* @param {Object[]} options.columns
|
|
* @param {Object[]} options.columns
|
|
@@ -828,7 +849,7 @@ export default {
|
|
|
* @return {Object}
|
|
* @return {Object}
|
|
|
*/
|
|
*/
|
|
|
addTemporaryTable: function(options) {
|
|
addTemporaryTable: function(options) {
|
|
|
- return getCatalog(options.sourceType).addTemporaryTable(options);
|
|
|
|
|
|
|
+ return getCatalog(options.sourceType, options.connector).addTemporaryTable(options);
|
|
|
},
|
|
},
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -836,6 +857,7 @@ export default {
|
|
|
* @param {string} options.sourceType
|
|
* @param {string} options.sourceType
|
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
|
|
|
+ * @param {Connector} options.connector
|
|
|
* @param {string|string[]} options.path
|
|
* @param {string|string[]} options.path
|
|
|
* @param {Object} [options.definition] - Optional initial definition
|
|
* @param {Object} [options.definition] - Optional initial definition
|
|
|
* @param {boolean} [options.temporaryOnly] - Default: false
|
|
* @param {boolean} [options.temporaryOnly] - Default: false
|
|
@@ -843,7 +865,7 @@ export default {
|
|
|
* @return {Promise}
|
|
* @return {Promise}
|
|
|
*/
|
|
*/
|
|
|
getEntry: function(options) {
|
|
getEntry: function(options) {
|
|
|
- return getCatalog(options.sourceType).getEntry(options);
|
|
|
|
|
|
|
+ return getCatalog(options.sourceType, options.connector).getEntry(options);
|
|
|
},
|
|
},
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -851,12 +873,13 @@ export default {
|
|
|
* @param {string} options.sourceType
|
|
* @param {string} options.sourceType
|
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
|
|
|
+ * @param {Connector} options.connector
|
|
|
* @param {string[][]} options.paths
|
|
* @param {string[][]} options.paths
|
|
|
*
|
|
*
|
|
|
* @return {Promise}
|
|
* @return {Promise}
|
|
|
*/
|
|
*/
|
|
|
getMultiTableEntry: function(options) {
|
|
getMultiTableEntry: function(options) {
|
|
|
- return getCatalog(options.sourceType).getMultiTableEntry(options);
|
|
|
|
|
|
|
+ return getCatalog(options.sourceType, options.connector).getMultiTableEntry(options);
|
|
|
},
|
|
},
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -867,6 +890,7 @@ export default {
|
|
|
* @param {string} options.sourceType
|
|
* @param {string} options.sourceType
|
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
* @param {ContextNamespace} options.namespace - The context namespace
|
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
* @param {ContextCompute} options.compute - The context compute
|
|
|
|
|
+ * @param {Connector} options.connector
|
|
|
* @param {string|string[]} options.path
|
|
* @param {string|string[]} options.path
|
|
|
* @param {Object} [options.definition] - Optional initial definition of the parent entry
|
|
* @param {Object} [options.definition] - Optional initial definition of the parent entry
|
|
|
* @param {boolean} [options.silenceErrors]
|
|
* @param {boolean} [options.silenceErrors]
|
|
@@ -879,7 +903,7 @@ export default {
|
|
|
getChildren: function(options) {
|
|
getChildren: function(options) {
|
|
|
const deferred = $.Deferred();
|
|
const deferred = $.Deferred();
|
|
|
const cancellablePromises = [];
|
|
const cancellablePromises = [];
|
|
|
- getCatalog(options.sourceType)
|
|
|
|
|
|
|
+ getCatalog(options.sourceType, options.connector)
|
|
|
.getEntry(options)
|
|
.getEntry(options)
|
|
|
.done(entry => {
|
|
.done(entry => {
|
|
|
cancellablePromises.push(
|
|
cancellablePromises.push(
|
|
@@ -895,6 +919,7 @@ export default {
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
* @param {string} sourceType
|
|
* @param {string} sourceType
|
|
|
|
|
+ * @param {Connector} connector
|
|
|
*
|
|
*
|
|
|
* @return {DataCatalog}
|
|
* @return {DataCatalog}
|
|
|
*/
|
|
*/
|