|
|
@@ -122,19 +122,22 @@ var DataCatalog = (function () {
|
|
|
/**
|
|
|
* Clears the data catalog and cache for the given path and any children thereof.
|
|
|
*
|
|
|
- * @param {string} clusterId - The cluster ID
|
|
|
+ * @param {SourceContext} [sourceContext] - The source context
|
|
|
* @param {string[]} rootPath - The path to clear
|
|
|
*/
|
|
|
- DataCatalog.prototype.clearStorageCascade = function (clusterId, rootPath) {
|
|
|
+ DataCatalog.prototype.clearStorageCascade = function (sourceContext, rootPath) {
|
|
|
var self = this;
|
|
|
var deferred = $.Deferred();
|
|
|
- if (rootPath.length === 0 && !clusterId) {
|
|
|
- self.entries = {};
|
|
|
- self.store.clear().then(deferred.resolve).catch(deferred.reject);
|
|
|
- return deferred.promise();
|
|
|
+ if (!sourceContext) {
|
|
|
+ if (rootPath.length === 0) {
|
|
|
+ self.entries = {};
|
|
|
+ self.store.clear().then(deferred.resolve).catch(deferred.reject);
|
|
|
+ return deferred.promise();
|
|
|
+ }
|
|
|
+ return deferred.reject().promise();
|
|
|
}
|
|
|
|
|
|
- var keyPrefix = clusterId;
|
|
|
+ var keyPrefix = sourceContext.name;
|
|
|
if (rootPath.length) {
|
|
|
keyPrefix += '_' + rootPath.join('.');
|
|
|
}
|
|
|
@@ -174,7 +177,7 @@ var DataCatalog = (function () {
|
|
|
}
|
|
|
var deferred = $.Deferred();
|
|
|
|
|
|
- var identifier = dataCatalogEntry.clusterId;
|
|
|
+ var identifier = dataCatalogEntry.sourceContext.name;
|
|
|
if (dataCatalogEntry.path.length) {
|
|
|
identifier += '_' + dataCatalogEntry.path.join('.');
|
|
|
}
|
|
|
@@ -198,6 +201,7 @@ var DataCatalog = (function () {
|
|
|
* Loads Navigator Optimizer popularity for multiple tables in one go.
|
|
|
*
|
|
|
* @param {Object} options
|
|
|
+ * @oaram {SourceContext} options.sourceContext
|
|
|
* @param {string[][]} options.paths
|
|
|
* @param {boolean} [options.silenceErrors] - Default true
|
|
|
* @param {boolean} [options.cancellable] - Default false
|
|
|
@@ -216,7 +220,7 @@ var DataCatalog = (function () {
|
|
|
var existingPromises = [];
|
|
|
options.paths.forEach(function (path) {
|
|
|
var existingDeferred = $.Deferred();
|
|
|
- self.getEntry({ path: path }).done(function (tableEntry) {
|
|
|
+ self.getEntry({ sourceContext: options.sourceContext, path: path }).done(function (tableEntry) {
|
|
|
if (tableEntry.navOptPopularityForChildrenPromise) {
|
|
|
tableEntry.navOptPopularityForChildrenPromise.done(function (existingPopularEntries) {
|
|
|
popularEntries = popularEntries.concat(existingPopularEntries);
|
|
|
@@ -276,7 +280,7 @@ var DataCatalog = (function () {
|
|
|
|
|
|
Object.keys(perTable).forEach(function (path) {
|
|
|
var tableDeferred = $.Deferred();
|
|
|
- self.getEntry({ path: path }).done(function (entry) {
|
|
|
+ self.getEntry({ sourceContext: options.sourceContext, path: path }).done(function (entry) {
|
|
|
cancellablePromises.push(entry.trackedPromise('navOptPopularityForChildrenPromise', entry.applyNavOptResponseToChildren(perTable[path], options).done(function (entries) {
|
|
|
popularEntries = popularEntries.concat(entries);
|
|
|
tableDeferred.resolve();
|
|
|
@@ -330,21 +334,21 @@ var DataCatalog = (function () {
|
|
|
|
|
|
/**
|
|
|
* @param {Object} options
|
|
|
- * @param {string} [options.clusterId] - Default: DEFAULT_CLUSTER_ID
|
|
|
+ * @param {SourceContext} options.sourceContext
|
|
|
* @param {string|string[]} options.path
|
|
|
* @return {DataCatalogEntry}
|
|
|
*/
|
|
|
DataCatalog.prototype.getKnownEntry = function (options) {
|
|
|
var self = this;
|
|
|
var identifier = typeof options.path === 'string' ? options.path : options.path.join('.');
|
|
|
- identifier = (options.clusterId || DEFAULT_CLUSTER_ID) + (identifier ? '_' + identifier : '');
|
|
|
+ identifier = options.sourceContext.name + (identifier ? '_' + identifier : '');
|
|
|
return self.entries[identifier];
|
|
|
};
|
|
|
|
|
|
/**
|
|
|
* @param {Object} options
|
|
|
* @param {string|string[]} options.path
|
|
|
- * @param {string} [options.clusterId] - Default: DEFAULT_CLUSTER_ID
|
|
|
+ * @param {SourceContext} options.sourceContext
|
|
|
* @param {Object} [options.definition] - The initial definition if not already set on the entry
|
|
|
* @param {boolean} [options.cachedOnly] - Default: false
|
|
|
* @return {Promise}
|
|
|
@@ -352,7 +356,7 @@ var DataCatalog = (function () {
|
|
|
DataCatalog.prototype.getEntry = function (options) {
|
|
|
var self = this;
|
|
|
var identifier = typeof options.path === 'string' ? options.path : options.path.join('.');
|
|
|
- identifier = (options.clusterId || DEFAULT_CLUSTER_ID) + (identifier ? '_' + identifier : '');
|
|
|
+ identifier = options.sourceContext.name + (identifier ? '_' + identifier : '');
|
|
|
if (self.entries[identifier]) {
|
|
|
return self.entries[identifier];
|
|
|
}
|
|
|
@@ -361,11 +365,11 @@ var DataCatalog = (function () {
|
|
|
self.entries[identifier] = deferred.promise();
|
|
|
|
|
|
if (!cacheEnabled) {
|
|
|
- deferred.resolve(new DataCatalogEntry({ dataCatalog: self, clusterId: options.clusterId, path: options.path, definition: options.definition })).promise();
|
|
|
+ deferred.resolve(new DataCatalogEntry({ dataCatalog: self, sourceContext: options.sourceContext, path: options.path, definition: options.definition })).promise();
|
|
|
} else {
|
|
|
self.store.getItem(identifier).then(function (storeEntry) {
|
|
|
var definition = storeEntry ? storeEntry.definition : options.definition;
|
|
|
- var entry = new DataCatalogEntry({ dataCatalog: self, clusterId: options.clusterId, path: options.path, definition: definition });
|
|
|
+ var entry = new DataCatalogEntry({ dataCatalog: self, sourceContext: options.sourceContext, path: options.path, definition: definition });
|
|
|
if (storeEntry) {
|
|
|
mergeEntry(entry, storeEntry);
|
|
|
} else if (!options.cachedOnly && options.definition) {
|
|
|
@@ -374,7 +378,7 @@ var DataCatalog = (function () {
|
|
|
deferred.resolve(entry);
|
|
|
}).catch(function (error) {
|
|
|
console.warn(error);
|
|
|
- var entry = new DataCatalogEntry({ dataCatalog: self, clusterId: options.clusterId, path: options.path, definition: options.definition });
|
|
|
+ var entry = new DataCatalogEntry({ dataCatalog: self, sourceContext: options.sourceContext, path: options.path, definition: options.definition });
|
|
|
if (!options.cachedOnly && options.definition) {
|
|
|
entry.saveLater();
|
|
|
}
|
|
|
@@ -410,31 +414,31 @@ var DataCatalog = (function () {
|
|
|
/**
|
|
|
* Creates an identifier for the given paths with duplicates removed
|
|
|
*
|
|
|
- * @param {string} clusterId
|
|
|
+ * @param {SourceContext} sourceContext
|
|
|
* @param {string[][]} paths
|
|
|
* @return {string}
|
|
|
*/
|
|
|
- var createMultiTableIdentifier = function (clusterId, paths) {
|
|
|
+ var createMultiTableIdentifier = function (sourceContext, paths) {
|
|
|
var pathSet = {};
|
|
|
paths.forEach(function (path) {
|
|
|
pathSet[path.join('.')] = true;
|
|
|
});
|
|
|
var uniquePaths = Object.keys(pathSet);
|
|
|
uniquePaths.sort();
|
|
|
- return clusterId + '_' + uniquePaths.join(',');
|
|
|
+ return sourceContext.name + '_' + uniquePaths.join(',');
|
|
|
};
|
|
|
|
|
|
/**
|
|
|
*
|
|
|
* @param {Object} options
|
|
|
- * @param {string} [options.clusterId] - Default: DEFAULT_CLUSTER_ID
|
|
|
+ * @param {SourceContext} options.sourceContext
|
|
|
* @param {string[][]} options.paths
|
|
|
*
|
|
|
* @return {Promise}
|
|
|
*/
|
|
|
DataCatalog.prototype.getMultiTableEntry = function (options) {
|
|
|
var self = this;
|
|
|
- var identifier = createMultiTableIdentifier(options.clusterId || DEFAULT_CLUSTER_ID, options.paths);
|
|
|
+ var identifier = createMultiTableIdentifier(options.sourceContext, options.paths);
|
|
|
if (self.multiTableEntries[identifier]) {
|
|
|
return self.multiTableEntries[identifier];
|
|
|
}
|
|
|
@@ -591,7 +595,7 @@ var DataCatalog = (function () {
|
|
|
/**
|
|
|
* @param {DataCatalog} options.dataCatalog
|
|
|
* @param {string|string[]} options.path
|
|
|
- * @param {string} [options.clusterId] - Default: DEFAULT_CLUSTER_ID
|
|
|
+ * @param {SourceContext} options.sourceContext
|
|
|
* @param {Object} options.definition - Initial known metadata on creation (normally comes from the parent entry)
|
|
|
*
|
|
|
* @constructor
|
|
|
@@ -599,8 +603,7 @@ var DataCatalog = (function () {
|
|
|
function DataCatalogEntry(options) {
|
|
|
var self = this;
|
|
|
|
|
|
- // TODO: Make clusterId mandatory once implemented throughout
|
|
|
- self.clusterId = options.clusterId || DEFAULT_CLUSTER_ID;
|
|
|
+ self.sourceContext = options.sourceContext;
|
|
|
self.dataCatalog = options.dataCatalog;
|
|
|
self.path = typeof options.path === 'string' && options.path ? options.path.split('.') : options.path || [];
|
|
|
self.name = self.path.length ? self.path[self.path.length - 1] : options.dataCatalog.sourceType;
|
|
|
@@ -651,7 +654,7 @@ var DataCatalog = (function () {
|
|
|
self.childrenPromise = undefined;
|
|
|
|
|
|
if (self.path.length) {
|
|
|
- var parent = self.dataCatalog.getKnownEntry({ path: self.path.slice(0, self.path.length - 1) });
|
|
|
+ var parent = self.dataCatalog.getKnownEntry({ sourceContext: self.sourceContext, path: self.path.slice(0, self.path.length - 1) });
|
|
|
if (parent) {
|
|
|
parent.navigatorMetaForChildrenPromise = undefined;
|
|
|
parent.navOptPopularityForChildrenPromise = undefined;
|
|
|
@@ -718,7 +721,7 @@ var DataCatalog = (function () {
|
|
|
}
|
|
|
|
|
|
self.reset();
|
|
|
- var saveDeferred = options.cascade ? self.dataCatalog.clearStorageCascade(self.clusterId, self.path) : self.save();
|
|
|
+ var saveDeferred = options.cascade ? self.dataCatalog.clearStorageCascade(self.sourceContext, self.path) : self.save();
|
|
|
|
|
|
var clearPromise = $.when(invalidatePromise, saveDeferred);
|
|
|
|
|
|
@@ -796,6 +799,7 @@ var DataCatalog = (function () {
|
|
|
entities.forEach(function (entity) {
|
|
|
if (!sourceMeta.databases || ((entity.name || entity) !== '_impala_builtins')) {
|
|
|
promises.push(self.dataCatalog.getEntry({
|
|
|
+ sourceContext: self.sourceContext,
|
|
|
path: self.path.concat(entity.name || entity)
|
|
|
}).done(function (catalogEntry) {
|
|
|
if (!catalogEntry.definition || typeof catalogEntry.definition.index === 'undefined') {
|
|
|
@@ -823,7 +827,7 @@ var DataCatalog = (function () {
|
|
|
if ((self.getSourceType() === 'impala' || self.getSourceType() === 'hive') && self.isComplex()) {
|
|
|
(sourceMeta.type === 'map' ? ['key', 'value'] : ['item']).forEach(function (path) {
|
|
|
if (sourceMeta[path]) {
|
|
|
- promises.push(self.dataCatalog.getEntry({ path: self.path.concat(path) }).done(function (catalogEntry) {
|
|
|
+ promises.push(self.dataCatalog.getEntry({ sourceContext: self.sourceContext, path: self.path.concat(path) }).done(function (catalogEntry) {
|
|
|
if (!catalogEntry.definition || typeof catalogEntry.definition.index === 'undefined') {
|
|
|
var definition = sourceMeta[path];
|
|
|
definition.index = index++;
|
|
|
@@ -1736,7 +1740,7 @@ var DataCatalog = (function () {
|
|
|
}
|
|
|
};
|
|
|
|
|
|
- self.dataCatalog.getEntry({ path: self.path.slice(0, 2), definition: { type: 'table' } }).done(function (tableEntry) {
|
|
|
+ self.dataCatalog.getEntry({ sourceContext: self.sourceContext, path: self.path.slice(0, 2), definition: { type: 'table' } }).done(function (tableEntry) {
|
|
|
if (tableEntry && tableEntry.samplePromise) {
|
|
|
cancellablePromises.push(applyCancellable(tableEntry.samplePromise, options));
|
|
|
|
|
|
@@ -1801,7 +1805,7 @@ var DataCatalog = (function () {
|
|
|
return deferred.reject();
|
|
|
}
|
|
|
var cancellablePromises = [];
|
|
|
- catalogEntry.dataCatalog.getMultiTableEntry({ clusterId: catalogEntry.clusterId, paths: [ catalogEntry.path ] }).done(function (multiTableEntry) {
|
|
|
+ catalogEntry.dataCatalog.getMultiTableEntry({ sourceContext: catalogEntry.sourceContext, paths: [ catalogEntry.path ] }).done(function (multiTableEntry) {
|
|
|
cancellablePromises.push(multiTableEntry[functionName](options).done(deferred.resolve).fail(deferred.reject));
|
|
|
}).fail(deferred.reject);
|
|
|
return new CancellablePromise(deferred, undefined, cancellablePromises);
|
|
|
@@ -2144,8 +2148,8 @@ var DataCatalog = (function () {
|
|
|
/**
|
|
|
* @param {Object} options
|
|
|
* @param {string} options.sourceType
|
|
|
+ * @param {SourceContext} options.sourceContext
|
|
|
* @param {string|string[]} options.path
|
|
|
- * @param {string} [options.clusterId] - Optional cluster ID default: DEFAULT_CLUSTER_ID
|
|
|
* @param {Object} [options.definition] - Optional initial definition
|
|
|
*
|
|
|
* @return {Promise}
|
|
|
@@ -2157,7 +2161,7 @@ var DataCatalog = (function () {
|
|
|
/**
|
|
|
* @param {Object} options
|
|
|
* @param {string} options.sourceType
|
|
|
- * @param {string} [options.clusterId] - Default: DEFAULT_CLUSTER_ID
|
|
|
+ * @param {SourceContext} options.sourceContext
|
|
|
* @param {string[][]} options.paths
|
|
|
*
|
|
|
* @return {Promise}
|
|
|
@@ -2172,7 +2176,7 @@ var DataCatalog = (function () {
|
|
|
*
|
|
|
* @param {Object} options
|
|
|
* @param {string} options.sourceType
|
|
|
- * @param {string} [options.clusterId]- Default: DEFAULT_CLUSTER_ID
|
|
|
+ * @param {SourceContext} options.sourceContext
|
|
|
* @param {string|string[]} options.path
|
|
|
* @param {Object} [options.definition] - Optional initial definition of the parent entry
|
|
|
* @param {boolean} [options.silenceErrors]
|