浏览代码

[editor] Externalize the SQL references repositories

To solve an issue with dynamic imports and web components we can now pass the UDF and SQL reference repositories as properties on the Editor component.

This also splits the previous repository in two, one for UDFs and one for the rest.
Johan Ahlen 4 年之前
父节点
当前提交
d3c8aafe2a

+ 4 - 0
desktop/core/src/desktop/js/apps/editor/components/aceEditor/AceEditor.vue

@@ -22,6 +22,7 @@
     <ace-autocomplete
     <ace-autocomplete
       v-if="editor && autocompleteParser"
       v-if="editor && autocompleteParser"
       :autocomplete-parser="autocompleteParser"
       :autocomplete-parser="autocompleteParser"
+      :sql-reference-provider="sqlReferenceProvider"
       :editor="editor"
       :editor="editor"
       :editor-id="id"
       :editor-id="id"
       :executor="executor"
       :executor="executor"
@@ -54,6 +55,7 @@
   import { defer } from 'utils/hueUtils';
   import { defer } from 'utils/hueUtils';
   import I18n from 'utils/i18n';
   import I18n from 'utils/i18n';
   import { getFromLocalStorage, setInLocalStorage } from 'utils/storageUtils';
   import { getFromLocalStorage, setInLocalStorage } from 'utils/storageUtils';
+  import { SqlReferenceProvider } from 'sql/reference/types';
 
 
   // Taken from https://www.cs.tut.fi/~jkorpela/chars/spaces.html
   // Taken from https://www.cs.tut.fi/~jkorpela/chars/spaces.html
   const UNICODES_TO_REMOVE = /[\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u200B\u202F\u205F\u3000\uFEFF]/gi;
   const UNICODES_TO_REMOVE = /[\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u200B\u202F\u205F\u3000\uFEFF]/gi;
@@ -80,6 +82,8 @@
     aceOptions?: Ace.Options;
     aceOptions?: Ace.Options;
     @Prop({ required: false })
     @Prop({ required: false })
     sqlParserProvider?: SqlParserProvider;
     sqlParserProvider?: SqlParserProvider;
+    @Prop({ required: false })
+    sqlReferenceProvider?: SqlReferenceProvider;
 
 
     subTracker = new SubscriptionTracker();
     subTracker = new SubscriptionTracker();
     editor: Ace.Editor | null = null;
     editor: Ace.Editor | null = null;

+ 3 - 0
desktop/core/src/desktop/js/apps/editor/components/aceEditor/AceEditorKoBridge.vue

@@ -25,6 +25,7 @@
     :initial-cursor-position="cursorPosition"
     :initial-cursor-position="cursorPosition"
     :initial-value="value"
     :initial-value="value"
     :sql-parser-provider="sqlParserProvider"
     :sql-parser-provider="sqlParserProvider"
+    :sql-reference-provider="sqlReferenceProvider"
     @ace-created="aceCreated"
     @ace-created="aceCreated"
     @create-new-doc="createNewDoc"
     @create-new-doc="createNewDoc"
     @cursor-changed="cursorChanged"
     @cursor-changed="cursorChanged"
@@ -45,6 +46,7 @@
   import Executor from 'apps/editor/execution/executor';
   import Executor from 'apps/editor/execution/executor';
   import SubscriptionTracker from 'components/utils/SubscriptionTracker';
   import SubscriptionTracker from 'components/utils/SubscriptionTracker';
   import sqlParserRepository from 'parse/sql/sqlParserRepository';
   import sqlParserRepository from 'parse/sql/sqlParserRepository';
+  import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 
 
   @Component({
   @Component({
     components: { AceEditor }
     components: { AceEditor }
@@ -65,6 +67,7 @@
     editorId?: string;
     editorId?: string;
     initialized = false;
     initialized = false;
     sqlParserProvider = sqlParserRepository;
     sqlParserProvider = sqlParserRepository;
+    sqlReferenceProvider = sqlReferenceRepository;
     subTracker = new SubscriptionTracker();
     subTracker = new SubscriptionTracker();
     value?: string;
     value?: string;
 
 

+ 5 - 1
desktop/core/src/desktop/js/apps/editor/components/aceEditor/autocomplete/AceAutocomplete.vue

@@ -101,6 +101,7 @@
   import UdfDetailsPanel from './UdfDetailsPanel.vue';
   import UdfDetailsPanel from './UdfDetailsPanel.vue';
   import Spinner from 'components/Spinner.vue';
   import Spinner from 'components/Spinner.vue';
   import { clickOutsideDirective } from 'components/directives/clickOutsideDirective';
   import { clickOutsideDirective } from 'components/directives/clickOutsideDirective';
+  import { SqlReferenceProvider } from 'sql/reference/types';
   import hueDebug from 'utils/hueDebug';
   import hueDebug from 'utils/hueDebug';
 
 
   const aceUtil = <Ace.AceUtil>ace.require('ace/autocomplete/util');
   const aceUtil = <Ace.AceUtil>ace.require('ace/autocomplete/util');
@@ -124,6 +125,8 @@
     @Prop({ required: true })
     @Prop({ required: true })
     autocompleteParser!: AutocompleteParser;
     autocompleteParser!: AutocompleteParser;
     @Prop({ required: true })
     @Prop({ required: true })
+    sqlReferenceProvider!: SqlReferenceProvider;
+    @Prop({ required: true })
     editor!: Ace.Editor;
     editor!: Ace.Editor;
     @Prop({ required: true })
     @Prop({ required: true })
     editorId!: string;
     editorId!: string;
@@ -203,7 +206,8 @@
         executor: this.executor,
         executor: this.executor,
         editor: this.editor,
         editor: this.editor,
         temporaryOnly: this.temporaryOnly,
         temporaryOnly: this.temporaryOnly,
-        autocompleteParser: this.autocompleteParser
+        autocompleteParser: this.autocompleteParser,
+        sqlReferenceProvider: this.sqlReferenceProvider
       });
       });
 
 
       this.subTracker.addDisposable(this.autocompleter);
       this.subTracker.addDisposable(this.autocompleter);

+ 37 - 10
desktop/core/src/desktop/js/apps/editor/components/aceEditor/autocomplete/AutocompleteResults.ts

@@ -46,7 +46,7 @@ import {
 
 
 import apiHelper from 'api/apiHelper';
 import apiHelper from 'api/apiHelper';
 import dataCatalog from 'catalog/dataCatalog';
 import dataCatalog from 'catalog/dataCatalog';
-import { SetDetails, UdfDetails } from 'sql/reference/types';
+import { SetDetails, SqlReferenceProvider, UdfDetails } from 'sql/reference/types';
 import { hueWindow } from 'types/types';
 import { hueWindow } from 'types/types';
 import hueUtils from 'utils/hueUtils';
 import hueUtils from 'utils/hueUtils';
 import I18n from 'utils/i18n';
 import I18n from 'utils/i18n';
@@ -58,9 +58,8 @@ import {
   findUdf,
   findUdf,
   getArgumentDetailsForUdf,
   getArgumentDetailsForUdf,
   getUdfsWithReturnTypes,
   getUdfsWithReturnTypes,
-  getReturnTypesForUdf,
-  getSetOptions
-} from 'sql/reference/sqlReferenceRepository';
+  getReturnTypesForUdf
+} from 'sql/reference/sqlUdfRepository';
 
 
 interface ColumnReference {
 interface ColumnReference {
   type: string;
   type: string;
@@ -147,6 +146,7 @@ class AutocompleteResults {
   editor: Ace.Editor;
   editor: Ace.Editor;
   temporaryOnly: boolean;
   temporaryOnly: boolean;
   activeDatabase: string;
   activeDatabase: string;
+  sqlReferenceProvider: SqlReferenceProvider;
 
 
   parseResult!: AutocompleteParseResult;
   parseResult!: AutocompleteParseResult;
   subTracker = new SubscriptionTracker();
   subTracker = new SubscriptionTracker();
@@ -154,7 +154,13 @@ class AutocompleteResults {
   lastKnownRequests: JQueryXHR[] = [];
   lastKnownRequests: JQueryXHR[] = [];
   cancellablePromises: CancellablePromise<unknown>[] = [];
   cancellablePromises: CancellablePromise<unknown>[] = [];
 
 
-  constructor(options: { executor: Executor; editor: Ace.Editor; temporaryOnly: boolean }) {
+  constructor(options: {
+    sqlReferenceProvider: SqlReferenceProvider;
+    executor: Executor;
+    editor: Ace.Editor;
+    temporaryOnly: boolean;
+  }) {
+    this.sqlReferenceProvider = options.sqlReferenceProvider;
     this.executor = options.executor;
     this.executor = options.executor;
     this.editor = options.editor;
     this.editor = options.editor;
     this.temporaryOnly = options.temporaryOnly;
     this.temporaryOnly = options.temporaryOnly;
@@ -227,6 +233,7 @@ class AutocompleteResults {
 
 
   async adjustForUdfArgument(): Promise<void> {
   async adjustForUdfArgument(): Promise<void> {
     const foundArgumentDetails = (await getArgumentDetailsForUdf(
     const foundArgumentDetails = (await getArgumentDetailsForUdf(
+      this.sqlReferenceProvider,
       this.executor.connector(),
       this.executor.connector(),
       this.parseResult.udfArgument.name,
       this.parseResult.udfArgument.name,
       this.parseResult.udfArgument.position
       this.parseResult.udfArgument.position
@@ -414,7 +421,11 @@ class AutocompleteResults {
         });
         });
       } else if (type === 'UDFREF' && columnAlias.udfRef) {
       } else if (type === 'UDFREF' && columnAlias.udfRef) {
         try {
         try {
-          const types = await getReturnTypesForUdf(this.executor.connector(), columnAlias.udfRef);
+          const types = await getReturnTypesForUdf(
+            this.sqlReferenceProvider,
+            this.executor.connector(),
+            columnAlias.udfRef
+          );
           const resolvedType = types.length === 1 ? types[0] : 'T';
           const resolvedType = types.length === 1 ? types[0] : 'T';
           columnAliasSuggestions.push({
           columnAliasSuggestions.push({
             value: columnAlias.name,
             value: columnAlias.name,
@@ -466,7 +477,9 @@ class AutocompleteResults {
       return [];
       return [];
     }
     }
     try {
     try {
-      const setOptions = await getSetOptions(this.executor.connector());
+      const setOptions = await this.sqlReferenceProvider.getSetOptions(
+        this.executor.connector().dialect || 'generic'
+      );
       return Object.keys(setOptions).map(name => ({
       return Object.keys(setOptions).map(name => ({
         category: Category.Option,
         category: Category.Option,
         value: name,
         value: name,
@@ -494,6 +507,7 @@ class AutocompleteResults {
       const getUdfsForTypes = async (types: string[]): Promise<Suggestion[]> => {
       const getUdfsForTypes = async (types: string[]): Promise<Suggestion[]> => {
         try {
         try {
           const functionsToSuggest = await getUdfsWithReturnTypes(
           const functionsToSuggest = await getUdfsWithReturnTypes(
+            this.sqlReferenceProvider,
             this.executor.connector(),
             this.executor.connector(),
             types,
             types,
             !!this.parseResult.suggestAggregateFunctions,
             !!this.parseResult.suggestAggregateFunctions,
@@ -527,7 +541,11 @@ class AutocompleteResults {
           const colRef = await colRefPromise;
           const colRef = await colRefPromise;
           types = [colRef.type.toUpperCase()];
           types = [colRef.type.toUpperCase()];
         } else if (suggestFunctions.udfRef) {
         } else if (suggestFunctions.udfRef) {
-          types = await getReturnTypesForUdf(this.executor.connector(), suggestFunctions.udfRef);
+          types = await getReturnTypesForUdf(
+            this.sqlReferenceProvider,
+            this.executor.connector(),
+            suggestFunctions.udfRef
+          );
         }
         }
       } catch (err) {}
       } catch (err) {}
       suggestions = await getUdfsForTypes(types);
       suggestions = await getUdfsForTypes(types);
@@ -536,6 +554,7 @@ class AutocompleteResults {
 
 
       try {
       try {
         const functionsToSuggest = await getUdfsWithReturnTypes(
         const functionsToSuggest = await getUdfsWithReturnTypes(
+          this.sqlReferenceProvider,
           this.executor.connector(),
           this.executor.connector(),
           types,
           types,
           !!this.parseResult.suggestAggregateFunctions,
           !!this.parseResult.suggestAggregateFunctions,
@@ -721,7 +740,11 @@ class AutocompleteResults {
         suggestColumns.types[0] === 'UDFREF' &&
         suggestColumns.types[0] === 'UDFREF' &&
         suggestColumns.udfRef
         suggestColumns.udfRef
       ) {
       ) {
-        types = await getReturnTypesForUdf(this.executor.connector(), suggestColumns.udfRef);
+        types = await getReturnTypesForUdf(
+          this.sqlReferenceProvider,
+          this.executor.connector(),
+          suggestColumns.udfRef
+        );
       }
       }
     } catch (err) {}
     } catch (err) {}
 
 
@@ -1504,7 +1527,11 @@ class AutocompleteResults {
           clean = clean.replace(substitution.replace, substitution.with);
           clean = clean.replace(substitution.replace, substitution.with);
         });
         });
 
 
-        const foundUdfs = await findUdf(this.executor.connector(), value.aggregateFunction);
+        const foundUdfs = await findUdf(
+          this.sqlReferenceProvider,
+          this.executor.connector(),
+          value.aggregateFunction
+        );
 
 
         // TODO: Support showing multiple UDFs with the same name but different category in the autocomplete details.
         // TODO: Support showing multiple UDFs with the same name but different category in the autocomplete details.
         // For instance, trunc appears both for dates with one description and for numbers with another description.
         // For instance, trunc appears both for dates with one description and for numbers with another description.

+ 3 - 0
desktop/core/src/desktop/js/apps/editor/components/aceEditor/autocomplete/SqlAutocompleter.ts

@@ -27,6 +27,7 @@ import { ParsedSqlStatement } from 'parse/sqlStatementsParser';
 import { AutocompleteParser, AutocompleteParseResult } from 'parse/types';
 import { AutocompleteParser, AutocompleteParseResult } from 'parse/types';
 import { EditorInterpreter } from 'types/config';
 import { EditorInterpreter } from 'types/config';
 import AutocompleteResults from './AutocompleteResults';
 import AutocompleteResults from './AutocompleteResults';
+import { SqlReferenceProvider } from 'sql/reference/types';
 import huePubSub from 'utils/huePubSub';
 import huePubSub from 'utils/huePubSub';
 
 
 export default class SqlAutocompleter implements Disposable {
 export default class SqlAutocompleter implements Disposable {
@@ -51,6 +52,7 @@ export default class SqlAutocompleter implements Disposable {
     fixedPrefix?: () => string;
     fixedPrefix?: () => string;
     fixedPostfix?: () => string;
     fixedPostfix?: () => string;
     autocompleteParser: AutocompleteParser;
     autocompleteParser: AutocompleteParser;
+    sqlReferenceProvider: SqlReferenceProvider;
   }) {
   }) {
     this.editorId = options.editorId;
     this.editorId = options.editorId;
     this.editor = options.editor;
     this.editor = options.editor;
@@ -60,6 +62,7 @@ export default class SqlAutocompleter implements Disposable {
     this.autocompleteParser = options.autocompleteParser;
     this.autocompleteParser = options.autocompleteParser;
 
 
     this.autocompleteResults = new AutocompleteResults({
     this.autocompleteResults = new AutocompleteResults({
+      sqlReferenceProvider: options.sqlReferenceProvider,
       executor: options.executor,
       executor: options.executor,
       editor: this.editor,
       editor: this.editor,
       temporaryOnly: !!options.temporaryOnly
       temporaryOnly: !!options.temporaryOnly

+ 6 - 6
desktop/core/src/desktop/js/ko/components/assist/ko.assistFunctionsPanel.js

@@ -18,13 +18,13 @@ import * as ko from 'knockout';
 
 
 import componentUtils from 'ko/components/componentUtils';
 import componentUtils from 'ko/components/componentUtils';
 import { HUE_DROP_DOWN_COMPONENT } from 'ko/components/ko.dropDown';
 import { HUE_DROP_DOWN_COMPONENT } from 'ko/components/ko.dropDown';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 import {
 import {
   CLEAR_UDF_CACHE_EVENT,
   CLEAR_UDF_CACHE_EVENT,
   DESCRIBE_UDF_EVENT,
   DESCRIBE_UDF_EVENT,
   getUdfCategories,
   getUdfCategories,
-  hasUdfCategories,
   UDF_DESCRIBED_EVENT
   UDF_DESCRIBED_EVENT
-} from 'sql/reference/sqlReferenceRepository';
+} from 'sql/reference/sqlUdfRepository';
 import { CONFIG_REFRESHED_EVENT, filterEditorConnectors } from 'utils/hueConfig';
 import { CONFIG_REFRESHED_EVENT, filterEditorConnectors } from 'utils/hueConfig';
 import huePubSub from 'utils/huePubSub';
 import huePubSub from 'utils/huePubSub';
 import I18n from 'utils/i18n';
 import I18n from 'utils/i18n';
@@ -229,7 +229,7 @@ class ConnectorUdfCategories {
   async getUdfs() {
   async getUdfs() {
     this.loading(true);
     this.loading(true);
     const categories = [];
     const categories = [];
-    const functions = await getUdfCategories(this.connector);
+    const functions = await getUdfCategories(sqlReferenceRepository, this.connector);
     functions.forEach(category => {
     functions.forEach(category => {
       const koCategory = {
       const koCategory = {
         name: category.name,
         name: category.name,
@@ -294,9 +294,9 @@ class AssistFunctionsPanel {
         (this.activeConnector() && this.activeConnector().type) ||
         (this.activeConnector() && this.activeConnector().type) ||
         getFromLocalStorage('assist.function.panel.active.connector.type');
         getFromLocalStorage('assist.function.panel.active.connector.type');
 
 
-      const availableConnectorUdfs = filterEditorConnectors(hasUdfCategories).map(
-        connector => new ConnectorUdfCategories(connector)
-      );
+      const availableConnectorUdfs = filterEditorConnectors(
+        connector => connector.dialect && sqlReferenceRepository.hasUdfCategories(connector.dialect)
+      ).map(connector => new ConnectorUdfCategories(connector));
 
 
       availableConnectorUdfs.sort((a, b) =>
       availableConnectorUdfs.sort((a, b) =>
         a.connector.displayName.localeCompare(b.connector.displayName)
         a.connector.displayName.localeCompare(b.connector.displayName)

+ 3 - 2
desktop/core/src/desktop/js/ko/components/contextPopover/functionContext.js

@@ -17,7 +17,8 @@
 import * as ko from 'knockout';
 import * as ko from 'knockout';
 
 
 import I18n from 'utils/i18n';
 import I18n from 'utils/i18n';
-import { findUdf } from 'sql/reference/sqlReferenceRepository';
+import { findUdf } from 'sql/reference/sqlUdfRepository';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 
 
 const TEMPLATE_NAME = 'context-popover-function-details';
 const TEMPLATE_NAME = 'context-popover-function-details';
 
 
@@ -44,7 +45,7 @@ class FunctionContextTabs {
     this.loading = ko.observable(true);
     this.loading = ko.observable(true);
     this.hasErrors = ko.observable(false);
     this.hasErrors = ko.observable(false);
 
 
-    findUdf(connector, data.function)
+    findUdf(sqlReferenceRepository, connector, data.function)
       .then(udfs => {
       .then(udfs => {
         // TODO: Support showing multiple UDFs with the same name but different category in the context popover.
         // TODO: Support showing multiple UDFs with the same name but different category in the context popover.
         // For instance, trunc appears both for dates with one description and for numbers with another description.
         // For instance, trunc appears both for dates with one description and for numbers with another description.

+ 29 - 8
desktop/core/src/desktop/js/sql/autocompleteResults.js

@@ -32,9 +32,9 @@ import {
   findUdf,
   findUdf,
   getArgumentDetailsForUdf,
   getArgumentDetailsForUdf,
   getUdfsWithReturnTypes,
   getUdfsWithReturnTypes,
-  getReturnTypesForUdf,
-  getSetOptions
-} from './reference/sqlReferenceRepository';
+  getReturnTypesForUdf
+} from './reference/sqlUdfRepository';
+import sqlReferenceRepository from './reference/sqlReferenceRepository';
 
 
 const normalizedColors = HueColors.getNormalizedColors();
 const normalizedColors = HueColors.getNormalizedColors();
 
 
@@ -482,6 +482,7 @@ class AutocompleteResults {
 
 
   async adjustForUdfArgument() {
   async adjustForUdfArgument() {
     const foundArgumentDetails = (await getArgumentDetailsForUdf(
     const foundArgumentDetails = (await getArgumentDetailsForUdf(
+      sqlReferenceRepository,
       this.snippet.connector(),
       this.snippet.connector(),
       this.parseResult.udfArgument.name,
       this.parseResult.udfArgument.name,
       this.parseResult.udfArgument.position
       this.parseResult.udfArgument.position
@@ -670,7 +671,11 @@ class AutocompleteResults {
         });
         });
       } else if (type === 'UDFREF') {
       } else if (type === 'UDFREF') {
         try {
         try {
-          const types = await getReturnTypesForUdf(this.snippet.connector(), columnAlias.udfRef);
+          const types = await getReturnTypesForUdf(
+            sqlReferenceRepository,
+            this.snippet.connector(),
+            columnAlias.udfRef
+          );
           const resolvedType = types.length === 1 ? types[0] : 'T';
           const resolvedType = types.length === 1 ? types[0] : 'T';
           columnAliasSuggestions.push({
           columnAliasSuggestions.push({
             value: columnAlias.name,
             value: columnAlias.name,
@@ -723,7 +728,9 @@ class AutocompleteResults {
       return [];
       return [];
     }
     }
     try {
     try {
-      const setOptions = await getSetOptions(this.snippet.connector());
+      const setOptions = await sqlReferenceRepository.getSetOptions(
+        this.snippet.connector().dialect || ''
+      );
       return Object.keys(setOptions).map(name => ({
       return Object.keys(setOptions).map(name => ({
         category: CATEGORIES.OPTION,
         category: CATEGORIES.OPTION,
         value: name,
         value: name,
@@ -752,6 +759,7 @@ class AutocompleteResults {
       const getUdfsForTypes = async types => {
       const getUdfsForTypes = async types => {
         try {
         try {
           const functionsToSuggest = await getUdfsWithReturnTypes(
           const functionsToSuggest = await getUdfsWithReturnTypes(
+            sqlReferenceRepository,
             this.snippet.connector(),
             this.snippet.connector(),
             types,
             types,
             this.parseResult.suggestAggregateFunctions || false,
             this.parseResult.suggestAggregateFunctions || false,
@@ -785,7 +793,11 @@ class AutocompleteResults {
           const colRef = await colRefPromise;
           const colRef = await colRefPromise;
           types = [colRef.type.toUpperCase()];
           types = [colRef.type.toUpperCase()];
         } else {
         } else {
-          types = await getReturnTypesForUdf(this.snippet.connector(), suggestFunctions.udfRef);
+          types = await getReturnTypesForUdf(
+            sqlReferenceRepository,
+            this.snippet.connector(),
+            suggestFunctions.udfRef
+          );
         }
         }
       } catch (err) {}
       } catch (err) {}
       suggestions = await getUdfsForTypes(types);
       suggestions = await getUdfsForTypes(types);
@@ -794,6 +806,7 @@ class AutocompleteResults {
 
 
       try {
       try {
         const functionsToSuggest = await getUdfsWithReturnTypes(
         const functionsToSuggest = await getUdfsWithReturnTypes(
+          sqlReferenceRepository,
           this.snippet.connector(),
           this.snippet.connector(),
           types,
           types,
           this.parseResult.suggestAggregateFunctions || false,
           this.parseResult.suggestAggregateFunctions || false,
@@ -977,7 +990,11 @@ class AutocompleteResults {
         const colRef = await colRefPromise;
         const colRef = await colRefPromise;
         types = [colRef.type.toUpperCase()];
         types = [colRef.type.toUpperCase()];
       } else if (suggestColumns.types && suggestColumns.types[0] === 'UDFREF') {
       } else if (suggestColumns.types && suggestColumns.types[0] === 'UDFREF') {
-        types = await getReturnTypesForUdf(this.snippet.connector(), suggestColumns.udfRef);
+        types = await getReturnTypesForUdf(
+          sqlReferenceRepository,
+          this.snippet.connector(),
+          suggestColumns.udfRef
+        );
       }
       }
     } catch (err) {}
     } catch (err) {}
 
 
@@ -1771,7 +1788,11 @@ class AutocompleteResults {
           clean = clean.replace(substitution.replace, substitution.with);
           clean = clean.replace(substitution.replace, substitution.with);
         });
         });
 
 
-        const foundUdfs = await findUdf(this.snippet.connector(), value.aggregateFunction);
+        const foundUdfs = await findUdf(
+          sqlReferenceRepository,
+          this.snippet.connector(),
+          value.aggregateFunction
+        );
 
 
         // TODO: Support showing multiple UDFs with the same name but different category in the autocomplete details.
         // TODO: Support showing multiple UDFs with the same name but different category in the autocomplete details.
         // For instance, trunc appears both for dates with one description and for numbers with another description.
         // For instance, trunc appears both for dates with one description and for numbers with another description.

+ 6 - 5
desktop/core/src/desktop/js/sql/autocompleteResults.test.js

@@ -25,7 +25,8 @@ import dataCatalog from 'catalog/dataCatalog';
 import huePubSub from 'utils/huePubSub';
 import huePubSub from 'utils/huePubSub';
 import I18n from 'utils/i18n';
 import I18n from 'utils/i18n';
 import LOTS_OF_PARSE_RESULTS from './test/lotsOfParseResults';
 import LOTS_OF_PARSE_RESULTS from './test/lotsOfParseResults';
-import * as sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
+import * as sqlUdfRepository from 'sql/reference/sqlUdfRepository';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 import { sleep } from '../utils/hueUtils';
 import { sleep } from '../utils/hueUtils';
 
 
 describe('AutocompleteResults.js', () => {
 describe('AutocompleteResults.js', () => {
@@ -380,7 +381,7 @@ describe('AutocompleteResults.js', () => {
   it('should handle parse results with functions', async () => {
   it('should handle parse results with functions', async () => {
     subject.entries([]);
     subject.entries([]);
 
 
-    const spy = spyOn(sqlReferenceRepository, 'getUdfsWithReturnTypes').and.callFake(async () =>
+    const spy = spyOn(sqlUdfRepository, 'getUdfsWithReturnTypes').and.callFake(async () =>
       Promise.resolve([
       Promise.resolve([
         {
         {
           name: 'count',
           name: 'count',
@@ -413,7 +414,7 @@ describe('AutocompleteResults.js', () => {
   it('should handle parse results with udf argument keywords', async () => {
   it('should handle parse results with udf argument keywords', async () => {
     subject.entries([]);
     subject.entries([]);
 
 
-    const spy = spyOn(sqlReferenceRepository, 'getArgumentDetailsForUdf').and.callFake(async () =>
+    const spy = spyOn(sqlUdfRepository, 'getArgumentDetailsForUdf').and.callFake(async () =>
       Promise.resolve([{ type: 'T', keywords: ['a', 'b'] }])
       Promise.resolve([{ type: 'T', keywords: ['a', 'b'] }])
     );
     );
 
 
@@ -440,9 +441,9 @@ describe('AutocompleteResults.js', () => {
     subject.entries([]);
     subject.entries([]);
 
 
     const spy = spyOn(sqlReferenceRepository, 'getSetOptions').and.callFake(
     const spy = spyOn(sqlReferenceRepository, 'getSetOptions').and.callFake(
-      async connector =>
+      async dialect =>
         new Promise(resolve => {
         new Promise(resolve => {
-          expect(connector).toEqual(subject.snippet.connector());
+          expect(dialect).toEqual(subject.snippet.connector().dialect);
           resolve({
           resolve({
             OPTION_1: {
             OPTION_1: {
               description: 'Desc 1',
               description: 'Desc 1',

+ 0 - 218
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.test.ts

@@ -1,218 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { UdfArgument } from 'sql/reference/types';
-import { Connector } from 'types/config';
-import { getArgumentDetailsForUdf, isReserved } from './sqlReferenceRepository';
-import * as apiUtils from 'sql/reference/apiUtils';
-
-describe('sqlReferenceRepository.js', () => {
-  const createTestConnector = (dialect: string, id: string): Connector => ({
-    dialect: dialect,
-    id: id,
-    buttonName: '',
-    displayName: '',
-    page: '',
-    tooltip: '',
-    type: ''
-  });
-
-  const hiveConn: Connector = createTestConnector('hive', 'hive');
-  const impalaConn = createTestConnector('impala', 'impala');
-
-  jest.mock('sql/reference/impala/udfReference', () => ({
-    UDF_CATEGORIES: [
-      {
-        functions: {
-          cos: {
-            returnTypes: ['DOUBLE'],
-            arguments: [[{ type: 'DOUBLE' }]],
-            signature: 'cos(DOUBLE a)',
-            draggable: 'cos()',
-            description: ''
-          },
-          concat: {
-            returnTypes: ['STRING'],
-            arguments: [[{ type: 'STRING' }], [{ type: 'STRING', multiple: true }]],
-            signature: 'concat(STRING a, STRING b...)',
-            draggable: 'concat()',
-            description: ''
-          },
-          strleft: {
-            returnTypes: ['STRING'],
-            arguments: [[{ type: 'STRING' }], [{ type: 'INT' }]],
-            signature: 'strleft(STRING a, INT num_chars)',
-            draggable: 'strleft()',
-            description: ''
-          }
-        }
-      }
-    ]
-  }));
-
-  jest.mock('sql/reference/hive/udfReference', () => ({
-    UDF_CATEGORIES: [
-      {
-        functions: {
-          cos: {
-            returnTypes: ['DOUBLE'],
-            arguments: [[{ type: 'DECIMAL' }, { type: 'DOUBLE' }]],
-            signature: 'cos(DECIMAL|DOUBLE a)',
-            draggable: 'cos()',
-            description: ''
-          },
-          concat: {
-            returnTypes: ['STRING'],
-            arguments: [
-              [
-                { type: 'STRING', multiple: true },
-                { type: 'BINARY', multiple: true }
-              ]
-            ],
-            signature: 'concat(STRING|BINARY a, STRING|BINARY b...)',
-            draggable: 'concat()',
-            description: ''
-          },
-          reflect: {
-            returnTypes: ['T'],
-            arguments: [
-              [{ type: 'STRING' }],
-              [{ type: 'STRING' }],
-              [{ type: 'T', multiple: true, optional: true }]
-            ],
-            signature: 'reflect(class, method[, arg1[, arg2..]])',
-            draggable: 'reflect()',
-            description: ''
-          }
-        }
-      }
-    ]
-  }));
-
-  jest.spyOn(apiUtils, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
-
-  const extractType = (details: UdfArgument): string => details.type;
-
-  it('should give the expected argument types at a specific position', async () => {
-    expect((await getArgumentDetailsForUdf(hiveConn, 'cos', 1)).map(extractType)).toEqual([
-      'DECIMAL',
-      'DOUBLE'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'cos', 2)).map(extractType)).toEqual([]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'cos', 1)).map(extractType)).toEqual([
-      'DOUBLE'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'cos', 2)).map(extractType)).toEqual([]);
-
-    expect((await getArgumentDetailsForUdf(hiveConn, 'concat', 10)).map(extractType)).toEqual([
-      'STRING',
-      'BINARY'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'concat', 10)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-  });
-
-  it('should handle functions with different type of arguments', async () => {
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 1)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 2)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 3)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-  });
-
-  it('should handle functions with an infinite amount of arguments', async () => {
-    expect((await getArgumentDetailsForUdf(impalaConn, 'greatest', 1)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'greatest', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 1)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 2)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 3)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-  });
-
-  it('should not return types for arguments out of bounds', async () => {
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 1)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 2)).map(extractType)).toEqual([
-      'INT'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 3)).map(extractType)).toEqual([]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 200)).map(extractType)).toEqual(
-      []
-    );
-  });
-
-  it("should return T for any argument if the udf isn't found", async () => {
-    expect((await getArgumentDetailsForUdf(hiveConn, 'blabla', 2)).map(extractType)).toEqual(['T']);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'blabla', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'blabla', 2)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'blabla', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-  });
-
-  it("Should return generic keywords if the dialect isn't defined", async () => {
-    jest.mock('sql/reference/generic/reservedKeywords', () => ({
-      RESERVED_WORDS: new Set<string>(['GENERICRESERVED'])
-    }));
-
-    const reserved = await isReserved({ dialect: 'foo' } as Connector, 'GENERICRESERVED');
-    expect(reserved).toBeTruthy();
-
-    const notReserved = await isReserved({ dialect: 'foo' } as Connector, 'not_reserved');
-    expect(notReserved).toBeFalsy();
-  });
-
-  it('Should use custom keywords if defined for dialect', async () => {
-    jest.mock('sql/reference/calcite/reservedKeywords', () => ({
-      RESERVED_WORDS: new Set<string>(['CUSTOM'])
-    }));
-    jest.mock('sql/reference/generic/reservedKeywords', () => ({
-      RESERVED_WORDS: new Set<string>(['OTHER'])
-    }));
-
-    const reserved = await isReserved({ dialect: 'calcite' } as Connector, 'CUSTOM');
-    expect(reserved).toBeTruthy();
-
-    const notReserved = await isReserved({ dialect: 'calcite' } as Connector, 'OTHER');
-    expect(notReserved).toBeFalsy();
-  });
-});

+ 21 - 265
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.ts

@@ -14,25 +14,10 @@
 // See the License for the specific language governing permissions and
 // See the License for the specific language governing permissions and
 // limitations under the License.
 // limitations under the License.
 
 
-import {
-  SetOptions,
-  UdfArgument,
-  UdfCategory,
-  UdfCategoryFunctions,
-  UdfDetails
-} from 'sql/reference/types';
-import { Connector } from 'types/config';
-import { matchesType } from './typeUtils';
-import I18n from 'utils/i18n';
-import huePubSub from 'utils/huePubSub';
-import { clearUdfCache, getCachedUdfCategories, setCachedUdfCategories } from './apiCache';
-import { fetchDescribe, fetchUdfs } from './apiUtils';
-
-export const CLEAR_UDF_CACHE_EVENT = 'hue.clear.udf.cache';
-export const DESCRIBE_UDF_EVENT = 'hue.describe.udf';
-export const UDF_DESCRIBED_EVENT = 'hue.udf.described';
+import { SetOptions, SqlReferenceProvider, UdfCategory } from 'sql/reference/types';
 
 
 const GENERIC = 'generic';
 const GENERIC = 'generic';
+const EMPTY_KEYWORDS = new Set<string>();
 
 
 const KEYWORD_REFS: { [attr: string]: () => Promise<{ RESERVED_WORDS?: Set<string> }> } = {
 const KEYWORD_REFS: { [attr: string]: () => Promise<{ RESERVED_WORDS?: Set<string> }> } = {
   calcite: async () => import(/* webpackChunkName: "calcite-ref" */ './calcite/reservedKeywords'),
   calcite: async () => import(/* webpackChunkName: "calcite-ref" */ './calcite/reservedKeywords'),
@@ -57,261 +42,32 @@ const UDF_REFS: { [attr: string]: () => Promise<{ UDF_CATEGORIES?: UdfCategory[]
   flink: async () => import(/* webpackChunkName: "flink-ref" */ './flink/udfReference')
   flink: async () => import(/* webpackChunkName: "flink-ref" */ './flink/udfReference')
 };
 };
 
 
-const IGNORED_UDF_REGEX = /^[!=$%&*+-/<>^|~]+$/;
-
-const mergedUdfPromises: { [attr: string]: Promise<UdfCategory[]> } = {};
-
-const getMergedUdfKey = (connector: Connector, database?: string): string => {
-  let key = connector.id;
-  if (database) {
-    key += '_' + database;
-  }
-  return key;
-};
-
-export const hasUdfCategories = (connector: Connector): boolean =>
-  !!connector.dialect && typeof UDF_REFS[connector.dialect] !== 'undefined';
-
-const findUdfsToAdd = (
-  apiUdfs: UdfDetails[],
-  existingCategories: UdfCategory[]
-): UdfCategoryFunctions => {
-  const existingUdfNames = new Set();
-  existingCategories.forEach(category => {
-    Object.keys(category.functions).forEach(udfName => {
-      existingUdfNames.add(udfName.toUpperCase());
-    });
-  });
-
-  const result: UdfCategoryFunctions = {};
-
-  apiUdfs.forEach(apiUdf => {
-    if (
-      !result[apiUdf.name] &&
-      !existingUdfNames.has(apiUdf.name.toUpperCase()) &&
-      !IGNORED_UDF_REGEX.test(apiUdf.name)
-    ) {
-      result[apiUdf.name] = apiUdf;
-    }
-  });
-
-  return result;
-};
-
-const mergeWithApiUdfs = async (
-  categories: UdfCategory[],
-  connector: Connector,
-  database?: string
-) => {
-  const apiUdfs = await fetchUdfs(connector, database);
-
-  if (apiUdfs.length) {
-    const additionalUdfs = findUdfsToAdd(apiUdfs, categories);
-    if (Object.keys(additionalUdfs).length) {
-      const generalCategory = {
-        name: I18n('General'),
-        functions: additionalUdfs
-      };
-      categories.unshift(generalCategory);
-    }
-  }
-};
-
-export const getUdfCategories = async (
-  connector: Connector,
-  database?: string
-): Promise<UdfCategory[]> => {
-  const promiseKey = getMergedUdfKey(connector, database);
-  if (!mergedUdfPromises[promiseKey]) {
-    mergedUdfPromises[promiseKey] = new Promise(async resolve => {
-      const cachedCategories = await getCachedUdfCategories(connector, database);
-      if (cachedCategories) {
-        resolve(cachedCategories);
-      }
-      let categories: UdfCategory[] = [];
-      if (connector.dialect && UDF_REFS[connector.dialect]) {
-        const module = await UDF_REFS[connector.dialect]();
-        if (module.UDF_CATEGORIES) {
-          categories = module.UDF_CATEGORIES;
-          categories.forEach(category => {
-            Object.values(category.functions).forEach(udf => {
-              udf.described = true;
-            });
-          });
-        }
-      }
-      await mergeWithApiUdfs(categories, connector, database);
-      await setCachedUdfCategories(connector, database, categories);
-      resolve(categories);
-    });
-  }
-
-  return await mergedUdfPromises[promiseKey];
-};
-
-export const findUdf = async (
-  connector: Connector,
-  functionName: string
-): Promise<UdfDetails[]> => {
-  const categories = await getUdfCategories(connector);
-  const found: UdfDetails[] = [];
-  categories.forEach(category => {
-    if (category.functions[functionName]) {
-      found.push(category.functions[functionName]);
-    }
-  });
-  return found;
-};
-
-export const getReturnTypesForUdf = async (
-  connector: Connector,
-  functionName: string
-): Promise<string[]> => {
-  if (!functionName) {
-    return ['T'];
-  }
-  const udfs = await findUdf(connector, functionName);
-  if (!udfs.length) {
-    let returnTypesPresent = false;
-    const returnTypes = new Set<string>();
-    udfs.forEach(udf => {
-      if (udf.returnTypes) {
-        returnTypesPresent = true;
-        udf.returnTypes.forEach(type => returnTypes.add(type));
-      }
-    });
-    if (returnTypesPresent) {
-      return [...returnTypes];
-    }
+class SqlReferenceRepository implements SqlReferenceProvider {
+  async getReservedKeywords(dialect: string): Promise<Set<string>> {
+    const refImport = KEYWORD_REFS[dialect] || KEYWORD_REFS[GENERIC];
+    const module = await refImport();
+    return module.RESERVED_WORDS || EMPTY_KEYWORDS;
   }
   }
 
 
-  return ['T'];
-};
-
-export const getUdfsWithReturnTypes = async (
-  connector: Connector,
-  returnTypes: string[],
-  includeAggregate?: boolean,
-  includeAnalytic?: boolean
-): Promise<UdfDetails[]> => {
-  const categories = await getUdfCategories(connector);
-  const result: UdfDetails[] = [];
-  categories.forEach(category => {
-    if (
-      (!category.isAnalytic && !category.isAggregate) ||
-      (includeAggregate && category.isAggregate) ||
-      (includeAnalytic && category.isAnalytic)
-    ) {
-      Object.keys(category.functions).forEach(udfName => {
-        const udf = category.functions[udfName];
-        if (
-          !returnTypes ||
-          (connector.dialect && matchesType(connector.dialect, returnTypes, udf.returnTypes))
-        ) {
-          result.push(udf);
-        }
-      });
+  async getSetOptions(dialect: string): Promise<SetOptions> {
+    if (SET_REFS[dialect]) {
+      const module = await SET_REFS[dialect]();
+      return module.SET_OPTIONS || {};
     }
     }
-  });
-  result.sort((a, b) => a.name.localeCompare(b.name));
-  return result;
-};
-
-export const getArgumentDetailsForUdf = async (
-  connector: Connector,
-  functionName: string,
-  argumentPosition: number
-): Promise<UdfArgument[]> => {
-  const foundFunctions = await findUdf(connector, functionName);
-  if (!foundFunctions.length) {
-    return [{ type: 'T' }];
+    return {};
   }
   }
 
 
-  const possibleArguments: UdfArgument[] = [];
-  foundFunctions.forEach(foundFunction => {
-    const args = foundFunction.arguments;
-    if (argumentPosition > args.length) {
-      possibleArguments.push(...args[args.length - 1].filter(type => type.multiple));
-    } else {
-      possibleArguments.push(...args[argumentPosition - 1]);
-    }
-  });
-  return possibleArguments;
-};
-
-export const getSetOptions = async (connector: Connector): Promise<SetOptions> => {
-  if (connector.dialect && SET_REFS[connector.dialect]) {
-    const module = await SET_REFS[connector.dialect]();
-    if (module.SET_OPTIONS) {
-      return module.SET_OPTIONS;
-    }
+  async getUdfCategories(dialect: string): Promise<UdfCategory[]> {
+    const refImport = UDF_REFS[dialect] || UDF_REFS[GENERIC];
+    const module = await refImport();
+    return module.UDF_CATEGORIES || [];
   }
   }
-  return {};
-};
 
 
-export const isReserved = async (connector: Connector, word: string): Promise<boolean> => {
-  const refImport = (connector.dialect && KEYWORD_REFS[connector.dialect]) || KEYWORD_REFS[GENERIC];
-  const module = await refImport();
-  if (module.RESERVED_WORDS) {
-    return module.RESERVED_WORDS.has(word.toUpperCase());
+  hasUdfCategories(dialect: string): boolean {
+    return !!UDF_REFS[dialect];
   }
   }
+}
 
 
-  return false;
-};
-
-const findUdfInCategories = (
-  categories: UdfCategory[],
-  udfName: string
-): UdfDetails | undefined => {
-  let foundUdf = undefined;
-  categories.some(category =>
-    Object.values(category.functions).some(udf => {
-      if (udf.name === udfName) {
-        foundUdf = udf;
-        return true;
-      }
-    })
-  );
-  return foundUdf;
-};
+const sqlReferenceRepository = new SqlReferenceRepository();
 
 
-huePubSub.subscribe(
-  DESCRIBE_UDF_EVENT,
-  async (details: { connector: Connector; udfName: string; database?: string }): Promise<void> => {
-    const categories = await getUdfCategories(details.connector, details.database);
-    const foundUdf = findUdfInCategories(categories, details.udfName);
-    if (foundUdf && !foundUdf.described) {
-      const apiUdf = await fetchDescribe(details.connector, foundUdf, details.database);
-      if (apiUdf) {
-        if (apiUdf.description) {
-          foundUdf.description = apiUdf.description;
-        }
-        if (apiUdf.signature) {
-          foundUdf.signature = apiUdf.signature;
-        }
-        foundUdf.described = true;
-        await setCachedUdfCategories(details.connector, details.database, categories);
-        huePubSub.publish(UDF_DESCRIBED_EVENT, {
-          connector: details.connector,
-          database: details.database,
-          udf: foundUdf
-        });
-      }
-    }
-  }
-);
-
-huePubSub.subscribe(
-  CLEAR_UDF_CACHE_EVENT,
-  async (details: { connector: Connector; callback: () => void }) => {
-    await clearUdfCache(details.connector);
-    Object.keys(mergedUdfPromises).forEach(key => {
-      if (key === details.connector.id || key.indexOf(details.connector.id + '_') === 0) {
-        delete mergedUdfPromises[key];
-      }
-    });
-    if (details.callback) {
-      details.callback();
-    }
-  }
-);
+export default sqlReferenceRepository;

+ 242 - 0
desktop/core/src/desktop/js/sql/reference/sqlUdfRepository.test.ts

@@ -0,0 +1,242 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { UdfArgument } from 'sql/reference/types';
+import { Connector } from 'types/config';
+import { getArgumentDetailsForUdf } from './sqlUdfRepository';
+import sqlReferenceRepository from './sqlReferenceRepository';
+import * as apiUtils from 'sql/reference/apiUtils';
+
+describe('sqlUdfRepository.ts', () => {
+  const createTestConnector = (dialect: string, id: string): Connector => ({
+    dialect: dialect,
+    id: id,
+    buttonName: '',
+    displayName: '',
+    page: '',
+    tooltip: '',
+    type: ''
+  });
+
+  const hiveConn: Connector = createTestConnector('hive', 'hive');
+  const impalaConn = createTestConnector('impala', 'impala');
+
+  jest.mock('sql/reference/impala/udfReference', () => ({
+    UDF_CATEGORIES: [
+      {
+        functions: {
+          cos: {
+            returnTypes: ['DOUBLE'],
+            arguments: [[{ type: 'DOUBLE' }]],
+            signature: 'cos(DOUBLE a)',
+            draggable: 'cos()',
+            description: ''
+          },
+          concat: {
+            returnTypes: ['STRING'],
+            arguments: [[{ type: 'STRING' }], [{ type: 'STRING', multiple: true }]],
+            signature: 'concat(STRING a, STRING b...)',
+            draggable: 'concat()',
+            description: ''
+          },
+          strleft: {
+            returnTypes: ['STRING'],
+            arguments: [[{ type: 'STRING' }], [{ type: 'INT' }]],
+            signature: 'strleft(STRING a, INT num_chars)',
+            draggable: 'strleft()',
+            description: ''
+          }
+        }
+      }
+    ]
+  }));
+
+  jest.mock('sql/reference/hive/udfReference', () => ({
+    UDF_CATEGORIES: [
+      {
+        functions: {
+          cos: {
+            returnTypes: ['DOUBLE'],
+            arguments: [[{ type: 'DECIMAL' }, { type: 'DOUBLE' }]],
+            signature: 'cos(DECIMAL|DOUBLE a)',
+            draggable: 'cos()',
+            description: ''
+          },
+          concat: {
+            returnTypes: ['STRING'],
+            arguments: [
+              [
+                { type: 'STRING', multiple: true },
+                { type: 'BINARY', multiple: true }
+              ]
+            ],
+            signature: 'concat(STRING|BINARY a, STRING|BINARY b...)',
+            draggable: 'concat()',
+            description: ''
+          },
+          reflect: {
+            returnTypes: ['T'],
+            arguments: [
+              [{ type: 'STRING' }],
+              [{ type: 'STRING' }],
+              [{ type: 'T', multiple: true, optional: true }]
+            ],
+            signature: 'reflect(class, method[, arg1[, arg2..]])',
+            draggable: 'reflect()',
+            description: ''
+          }
+        }
+      }
+    ]
+  }));
+
+  jest.spyOn(apiUtils, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+
+  const extractType = (details: UdfArgument): string => details.type;
+
+  it('should give the expected argument types at a specific position', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'cos', 1)).map(extractType)
+    ).toEqual(['DECIMAL', 'DOUBLE']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'cos', 2)).map(extractType)
+    ).toEqual([]);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'cos', 1)).map(
+        extractType
+      )
+    ).toEqual(['DOUBLE']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'cos', 2)).map(
+        extractType
+      )
+    ).toEqual([]);
+
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'concat', 10)).map(
+        extractType
+      )
+    ).toEqual(['STRING', 'BINARY']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'concat', 10)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+  });
+
+  it('should handle functions with different type of arguments', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 1)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 2)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 3)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+  });
+
+  it('should handle functions with an infinite amount of arguments', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'greatest', 1)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'greatest', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 1)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 2)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 3)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+  });
+
+  it('should not return types for arguments out of bounds', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 1)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 2)).map(
+        extractType
+      )
+    ).toEqual(['INT']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 3)).map(
+        extractType
+      )
+    ).toEqual([]);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 200)).map(
+        extractType
+      )
+    ).toEqual([]);
+  });
+
+  it("should return T for any argument if the udf isn't found", async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'blabla', 2)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'blabla', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'blabla', 2)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'blabla', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+  });
+});

+ 280 - 0
desktop/core/src/desktop/js/sql/reference/sqlUdfRepository.ts

@@ -0,0 +1,280 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  SqlReferenceProvider,
+  UdfArgument,
+  UdfCategory,
+  UdfCategoryFunctions,
+  UdfDetails
+} from 'sql/reference/types';
+import { Connector } from 'types/config';
+import { matchesType } from './typeUtils';
+import I18n from 'utils/i18n';
+import huePubSub from 'utils/huePubSub';
+import { clearUdfCache, getCachedUdfCategories, setCachedUdfCategories } from './apiCache';
+import { fetchDescribe, fetchUdfs } from './apiUtils';
+
+export const CLEAR_UDF_CACHE_EVENT = 'hue.clear.udf.cache';
+export const DESCRIBE_UDF_EVENT = 'hue.describe.udf';
+export const UDF_DESCRIBED_EVENT = 'hue.udf.described';
+
+const IGNORED_UDF_REGEX = /^[!=$%&*+-/<>^|~]+$/;
+
+const mergedUdfPromises: { [attr: string]: Promise<UdfCategory[]> } = {};
+
+const getMergedUdfKey = (connector: Connector, database?: string): string => {
+  let key = connector.id;
+  if (database) {
+    key += '_' + database;
+  }
+  return key;
+};
+
+const findUdfsToAdd = (
+  apiUdfs: UdfDetails[],
+  existingCategories: UdfCategory[]
+): UdfCategoryFunctions => {
+  const existingUdfNames = new Set();
+  existingCategories.forEach(category => {
+    Object.keys(category.functions).forEach(udfName => {
+      existingUdfNames.add(udfName.toUpperCase());
+    });
+  });
+
+  const result: UdfCategoryFunctions = {};
+
+  apiUdfs.forEach(apiUdf => {
+    if (
+      !result[apiUdf.name] &&
+      !existingUdfNames.has(apiUdf.name.toUpperCase()) &&
+      !IGNORED_UDF_REGEX.test(apiUdf.name)
+    ) {
+      result[apiUdf.name] = apiUdf;
+    }
+  });
+
+  return result;
+};
+
+const mergeWithApiUdfs = async (
+  categories: UdfCategory[],
+  connector: Connector,
+  database?: string
+) => {
+  const apiUdfs = await fetchUdfs(connector, database);
+
+  if (apiUdfs.length) {
+    const additionalUdfs = findUdfsToAdd(apiUdfs, categories);
+    if (Object.keys(additionalUdfs).length) {
+      const generalCategory = {
+        name: I18n('General'),
+        functions: additionalUdfs
+      };
+      categories.unshift(generalCategory);
+    }
+  }
+};
+
+export const getUdfCategories = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  database?: string
+): Promise<UdfCategory[]> => {
+  const promiseKey = getMergedUdfKey(connector, database);
+  if (!mergedUdfPromises[promiseKey]) {
+    mergedUdfPromises[promiseKey] = new Promise(async resolve => {
+      const cachedCategories = await getCachedUdfCategories(connector, database);
+      if (cachedCategories) {
+        resolve(cachedCategories);
+      }
+      let categories: UdfCategory[] = [];
+      if (connector.dialect && sqlReferenceProvider.hasUdfCategories(connector.dialect)) {
+        categories = await sqlReferenceProvider.getUdfCategories(connector.dialect);
+        categories.forEach(category => {
+          Object.values(category.functions).forEach(udf => {
+            udf.described = true;
+          });
+        });
+      }
+      await mergeWithApiUdfs(categories, connector, database);
+      await setCachedUdfCategories(connector, database, categories);
+      resolve(categories);
+    });
+  }
+
+  return await mergedUdfPromises[promiseKey];
+};
+
+export const findUdf = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  functionName: string
+): Promise<UdfDetails[]> => {
+  const categories = await getUdfCategories(sqlReferenceProvider, connector);
+  const found: UdfDetails[] = [];
+  categories.forEach(category => {
+    if (category.functions[functionName]) {
+      found.push(category.functions[functionName]);
+    }
+  });
+  return found;
+};
+
+export const getReturnTypesForUdf = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  functionName: string
+): Promise<string[]> => {
+  if (!functionName) {
+    return ['T'];
+  }
+  const udfs = await findUdf(sqlReferenceProvider, connector, functionName);
+  if (!udfs.length) {
+    let returnTypesPresent = false;
+    const returnTypes = new Set<string>();
+    udfs.forEach(udf => {
+      if (udf.returnTypes) {
+        returnTypesPresent = true;
+        udf.returnTypes.forEach(type => returnTypes.add(type));
+      }
+    });
+    if (returnTypesPresent) {
+      return [...returnTypes];
+    }
+  }
+
+  return ['T'];
+};
+
+export const getUdfsWithReturnTypes = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  returnTypes: string[],
+  includeAggregate?: boolean,
+  includeAnalytic?: boolean
+): Promise<UdfDetails[]> => {
+  const categories = await getUdfCategories(sqlReferenceProvider, connector);
+  const result: UdfDetails[] = [];
+  categories.forEach(category => {
+    if (
+      (!category.isAnalytic && !category.isAggregate) ||
+      (includeAggregate && category.isAggregate) ||
+      (includeAnalytic && category.isAnalytic)
+    ) {
+      Object.keys(category.functions).forEach(udfName => {
+        const udf = category.functions[udfName];
+        if (
+          !returnTypes ||
+          (connector.dialect && matchesType(connector.dialect, returnTypes, udf.returnTypes))
+        ) {
+          result.push(udf);
+        }
+      });
+    }
+  });
+  result.sort((a, b) => a.name.localeCompare(b.name));
+  return result;
+};
+
+export const getArgumentDetailsForUdf = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  functionName: string,
+  argumentPosition: number
+): Promise<UdfArgument[]> => {
+  const foundFunctions = await findUdf(sqlReferenceProvider, connector, functionName);
+  if (!foundFunctions.length) {
+    return [{ type: 'T' }];
+  }
+
+  const possibleArguments: UdfArgument[] = [];
+  foundFunctions.forEach(foundFunction => {
+    const args = foundFunction.arguments;
+    if (argumentPosition > args.length) {
+      possibleArguments.push(...args[args.length - 1].filter(type => type.multiple));
+    } else {
+      possibleArguments.push(...args[argumentPosition - 1]);
+    }
+  });
+  return possibleArguments;
+};
+
+const findUdfInCategories = (
+  categories: UdfCategory[],
+  udfName: string
+): UdfDetails | undefined => {
+  let foundUdf = undefined;
+  categories.some(category =>
+    Object.values(category.functions).some(udf => {
+      if (udf.name === udfName) {
+        foundUdf = udf;
+        return true;
+      }
+    })
+  );
+  return foundUdf;
+};
+
+huePubSub.subscribe(
+  DESCRIBE_UDF_EVENT,
+  async (details: {
+    sqlReferenceProvider: SqlReferenceProvider;
+    connector: Connector;
+    udfName: string;
+    database?: string;
+  }): Promise<void> => {
+    const categories = await getUdfCategories(
+      details.sqlReferenceProvider,
+      details.connector,
+      details.database
+    );
+    const foundUdf = findUdfInCategories(categories, details.udfName);
+    if (foundUdf && !foundUdf.described) {
+      const apiUdf = await fetchDescribe(details.connector, foundUdf, details.database);
+      if (apiUdf) {
+        if (apiUdf.description) {
+          foundUdf.description = apiUdf.description;
+        }
+        if (apiUdf.signature) {
+          foundUdf.signature = apiUdf.signature;
+        }
+        foundUdf.described = true;
+        await setCachedUdfCategories(details.connector, details.database, categories);
+        huePubSub.publish(UDF_DESCRIBED_EVENT, {
+          connector: details.connector,
+          database: details.database,
+          udf: foundUdf
+        });
+      }
+    }
+  }
+);
+
+huePubSub.subscribe(
+  CLEAR_UDF_CACHE_EVENT,
+  async (details: { connector: Connector; callback: () => void }) => {
+    await clearUdfCache(details.connector);
+    Object.keys(mergedUdfPromises).forEach(key => {
+      if (key === details.connector.id || key.indexOf(details.connector.id + '_') === 0) {
+        delete mergedUdfPromises[key];
+      }
+    });
+    if (details.callback) {
+      details.callback();
+    }
+  }
+);

+ 7 - 0
desktop/core/src/desktop/js/sql/reference/types.ts

@@ -56,3 +56,10 @@ export interface SetDetails {
 export interface TypeConversion {
 export interface TypeConversion {
   [attr: string]: { [attr: string]: boolean };
   [attr: string]: { [attr: string]: boolean };
 }
 }
+
+export interface SqlReferenceProvider {
+  getReservedKeywords(dialect: string): Promise<Set<string>>;
+  getSetOptions(dialect: string): Promise<SetOptions>;
+  getUdfCategories(dialect: string): Promise<UdfCategory[]>;
+  hasUdfCategories(dialect: string): boolean;
+}

+ 5 - 2
desktop/core/src/desktop/js/sql/sqlUtils.ts

@@ -20,7 +20,7 @@ import DataCatalogEntry from 'catalog/DataCatalogEntry';
 
 
 import dataCatalog from 'catalog/dataCatalog';
 import dataCatalog from 'catalog/dataCatalog';
 import { IdentifierChainEntry, ParsedLocation, ParsedTable } from 'parse/types';
 import { IdentifierChainEntry, ParsedLocation, ParsedTable } from 'parse/types';
-import { isReserved } from 'sql/reference/sqlReferenceRepository';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 import {
 import {
   CommentDetails,
   CommentDetails,
   Suggestion
   Suggestion
@@ -266,7 +266,10 @@ export default {
     if (identifier.indexOf(quoteChar) === 0) {
     if (identifier.indexOf(quoteChar) === 0) {
       return identifier;
       return identifier;
     }
     }
-    if (await isReserved(connector, identifier)) {
+    const reservedKeywords = await sqlReferenceRepository.getReservedKeywords(
+      connector.dialect || 'generic'
+    );
+    if (reservedKeywords.has(identifier.toUpperCase())) {
       return quoteChar + identifier + quoteChar;
       return quoteChar + identifier + quoteChar;
     }
     }