Selaa lähdekoodia

[editor] Externalize the SQL references repositories

To solve an issue with dynamic imports and web components we can now pass the UDF and SQL reference repositories as properties on the Editor component.

This also splits the previous repository in two, one for UDFs and one for the rest.
Johan Ahlen 4 vuotta sitten
vanhempi
commit
d3c8aafe2a

+ 4 - 0
desktop/core/src/desktop/js/apps/editor/components/aceEditor/AceEditor.vue

@@ -22,6 +22,7 @@
     <ace-autocomplete
       v-if="editor && autocompleteParser"
       :autocomplete-parser="autocompleteParser"
+      :sql-reference-provider="sqlReferenceProvider"
       :editor="editor"
       :editor-id="id"
       :executor="executor"
@@ -54,6 +55,7 @@
   import { defer } from 'utils/hueUtils';
   import I18n from 'utils/i18n';
   import { getFromLocalStorage, setInLocalStorage } from 'utils/storageUtils';
+  import { SqlReferenceProvider } from 'sql/reference/types';
 
   // Taken from https://www.cs.tut.fi/~jkorpela/chars/spaces.html
   const UNICODES_TO_REMOVE = /[\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u200B\u202F\u205F\u3000\uFEFF]/gi;
@@ -80,6 +82,8 @@
     aceOptions?: Ace.Options;
     @Prop({ required: false })
     sqlParserProvider?: SqlParserProvider;
+    @Prop({ required: false })
+    sqlReferenceProvider?: SqlReferenceProvider;
 
     subTracker = new SubscriptionTracker();
     editor: Ace.Editor | null = null;

+ 3 - 0
desktop/core/src/desktop/js/apps/editor/components/aceEditor/AceEditorKoBridge.vue

@@ -25,6 +25,7 @@
     :initial-cursor-position="cursorPosition"
     :initial-value="value"
     :sql-parser-provider="sqlParserProvider"
+    :sql-reference-provider="sqlReferenceProvider"
     @ace-created="aceCreated"
     @create-new-doc="createNewDoc"
     @cursor-changed="cursorChanged"
@@ -45,6 +46,7 @@
   import Executor from 'apps/editor/execution/executor';
   import SubscriptionTracker from 'components/utils/SubscriptionTracker';
   import sqlParserRepository from 'parse/sql/sqlParserRepository';
+  import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 
   @Component({
     components: { AceEditor }
@@ -65,6 +67,7 @@
     editorId?: string;
     initialized = false;
     sqlParserProvider = sqlParserRepository;
+    sqlReferenceProvider = sqlReferenceRepository;
     subTracker = new SubscriptionTracker();
     value?: string;
 

+ 5 - 1
desktop/core/src/desktop/js/apps/editor/components/aceEditor/autocomplete/AceAutocomplete.vue

@@ -101,6 +101,7 @@
   import UdfDetailsPanel from './UdfDetailsPanel.vue';
   import Spinner from 'components/Spinner.vue';
   import { clickOutsideDirective } from 'components/directives/clickOutsideDirective';
+  import { SqlReferenceProvider } from 'sql/reference/types';
   import hueDebug from 'utils/hueDebug';
 
   const aceUtil = <Ace.AceUtil>ace.require('ace/autocomplete/util');
@@ -124,6 +125,8 @@
     @Prop({ required: true })
     autocompleteParser!: AutocompleteParser;
     @Prop({ required: true })
+    sqlReferenceProvider!: SqlReferenceProvider;
+    @Prop({ required: true })
     editor!: Ace.Editor;
     @Prop({ required: true })
     editorId!: string;
@@ -203,7 +206,8 @@
         executor: this.executor,
         editor: this.editor,
         temporaryOnly: this.temporaryOnly,
-        autocompleteParser: this.autocompleteParser
+        autocompleteParser: this.autocompleteParser,
+        sqlReferenceProvider: this.sqlReferenceProvider
       });
 
       this.subTracker.addDisposable(this.autocompleter);

+ 37 - 10
desktop/core/src/desktop/js/apps/editor/components/aceEditor/autocomplete/AutocompleteResults.ts

@@ -46,7 +46,7 @@ import {
 
 import apiHelper from 'api/apiHelper';
 import dataCatalog from 'catalog/dataCatalog';
-import { SetDetails, UdfDetails } from 'sql/reference/types';
+import { SetDetails, SqlReferenceProvider, UdfDetails } from 'sql/reference/types';
 import { hueWindow } from 'types/types';
 import hueUtils from 'utils/hueUtils';
 import I18n from 'utils/i18n';
@@ -58,9 +58,8 @@ import {
   findUdf,
   getArgumentDetailsForUdf,
   getUdfsWithReturnTypes,
-  getReturnTypesForUdf,
-  getSetOptions
-} from 'sql/reference/sqlReferenceRepository';
+  getReturnTypesForUdf
+} from 'sql/reference/sqlUdfRepository';
 
 interface ColumnReference {
   type: string;
@@ -147,6 +146,7 @@ class AutocompleteResults {
   editor: Ace.Editor;
   temporaryOnly: boolean;
   activeDatabase: string;
+  sqlReferenceProvider: SqlReferenceProvider;
 
   parseResult!: AutocompleteParseResult;
   subTracker = new SubscriptionTracker();
@@ -154,7 +154,13 @@ class AutocompleteResults {
   lastKnownRequests: JQueryXHR[] = [];
   cancellablePromises: CancellablePromise<unknown>[] = [];
 
-  constructor(options: { executor: Executor; editor: Ace.Editor; temporaryOnly: boolean }) {
+  constructor(options: {
+    sqlReferenceProvider: SqlReferenceProvider;
+    executor: Executor;
+    editor: Ace.Editor;
+    temporaryOnly: boolean;
+  }) {
+    this.sqlReferenceProvider = options.sqlReferenceProvider;
     this.executor = options.executor;
     this.editor = options.editor;
     this.temporaryOnly = options.temporaryOnly;
@@ -227,6 +233,7 @@ class AutocompleteResults {
 
   async adjustForUdfArgument(): Promise<void> {
     const foundArgumentDetails = (await getArgumentDetailsForUdf(
+      this.sqlReferenceProvider,
       this.executor.connector(),
       this.parseResult.udfArgument.name,
       this.parseResult.udfArgument.position
@@ -414,7 +421,11 @@ class AutocompleteResults {
         });
       } else if (type === 'UDFREF' && columnAlias.udfRef) {
         try {
-          const types = await getReturnTypesForUdf(this.executor.connector(), columnAlias.udfRef);
+          const types = await getReturnTypesForUdf(
+            this.sqlReferenceProvider,
+            this.executor.connector(),
+            columnAlias.udfRef
+          );
           const resolvedType = types.length === 1 ? types[0] : 'T';
           columnAliasSuggestions.push({
             value: columnAlias.name,
@@ -466,7 +477,9 @@ class AutocompleteResults {
       return [];
     }
     try {
-      const setOptions = await getSetOptions(this.executor.connector());
+      const setOptions = await this.sqlReferenceProvider.getSetOptions(
+        this.executor.connector().dialect || 'generic'
+      );
       return Object.keys(setOptions).map(name => ({
         category: Category.Option,
         value: name,
@@ -494,6 +507,7 @@ class AutocompleteResults {
       const getUdfsForTypes = async (types: string[]): Promise<Suggestion[]> => {
         try {
           const functionsToSuggest = await getUdfsWithReturnTypes(
+            this.sqlReferenceProvider,
             this.executor.connector(),
             types,
             !!this.parseResult.suggestAggregateFunctions,
@@ -527,7 +541,11 @@ class AutocompleteResults {
           const colRef = await colRefPromise;
           types = [colRef.type.toUpperCase()];
         } else if (suggestFunctions.udfRef) {
-          types = await getReturnTypesForUdf(this.executor.connector(), suggestFunctions.udfRef);
+          types = await getReturnTypesForUdf(
+            this.sqlReferenceProvider,
+            this.executor.connector(),
+            suggestFunctions.udfRef
+          );
         }
       } catch (err) {}
       suggestions = await getUdfsForTypes(types);
@@ -536,6 +554,7 @@ class AutocompleteResults {
 
       try {
         const functionsToSuggest = await getUdfsWithReturnTypes(
+          this.sqlReferenceProvider,
           this.executor.connector(),
           types,
           !!this.parseResult.suggestAggregateFunctions,
@@ -721,7 +740,11 @@ class AutocompleteResults {
         suggestColumns.types[0] === 'UDFREF' &&
         suggestColumns.udfRef
       ) {
-        types = await getReturnTypesForUdf(this.executor.connector(), suggestColumns.udfRef);
+        types = await getReturnTypesForUdf(
+          this.sqlReferenceProvider,
+          this.executor.connector(),
+          suggestColumns.udfRef
+        );
       }
     } catch (err) {}
 
@@ -1504,7 +1527,11 @@ class AutocompleteResults {
           clean = clean.replace(substitution.replace, substitution.with);
         });
 
-        const foundUdfs = await findUdf(this.executor.connector(), value.aggregateFunction);
+        const foundUdfs = await findUdf(
+          this.sqlReferenceProvider,
+          this.executor.connector(),
+          value.aggregateFunction
+        );
 
         // TODO: Support showing multiple UDFs with the same name but different category in the autocomplete details.
         // For instance, trunc appears both for dates with one description and for numbers with another description.

+ 3 - 0
desktop/core/src/desktop/js/apps/editor/components/aceEditor/autocomplete/SqlAutocompleter.ts

@@ -27,6 +27,7 @@ import { ParsedSqlStatement } from 'parse/sqlStatementsParser';
 import { AutocompleteParser, AutocompleteParseResult } from 'parse/types';
 import { EditorInterpreter } from 'types/config';
 import AutocompleteResults from './AutocompleteResults';
+import { SqlReferenceProvider } from 'sql/reference/types';
 import huePubSub from 'utils/huePubSub';
 
 export default class SqlAutocompleter implements Disposable {
@@ -51,6 +52,7 @@ export default class SqlAutocompleter implements Disposable {
     fixedPrefix?: () => string;
     fixedPostfix?: () => string;
     autocompleteParser: AutocompleteParser;
+    sqlReferenceProvider: SqlReferenceProvider;
   }) {
     this.editorId = options.editorId;
     this.editor = options.editor;
@@ -60,6 +62,7 @@ export default class SqlAutocompleter implements Disposable {
     this.autocompleteParser = options.autocompleteParser;
 
     this.autocompleteResults = new AutocompleteResults({
+      sqlReferenceProvider: options.sqlReferenceProvider,
       executor: options.executor,
       editor: this.editor,
       temporaryOnly: !!options.temporaryOnly

+ 6 - 6
desktop/core/src/desktop/js/ko/components/assist/ko.assistFunctionsPanel.js

@@ -18,13 +18,13 @@ import * as ko from 'knockout';
 
 import componentUtils from 'ko/components/componentUtils';
 import { HUE_DROP_DOWN_COMPONENT } from 'ko/components/ko.dropDown';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 import {
   CLEAR_UDF_CACHE_EVENT,
   DESCRIBE_UDF_EVENT,
   getUdfCategories,
-  hasUdfCategories,
   UDF_DESCRIBED_EVENT
-} from 'sql/reference/sqlReferenceRepository';
+} from 'sql/reference/sqlUdfRepository';
 import { CONFIG_REFRESHED_EVENT, filterEditorConnectors } from 'utils/hueConfig';
 import huePubSub from 'utils/huePubSub';
 import I18n from 'utils/i18n';
@@ -229,7 +229,7 @@ class ConnectorUdfCategories {
   async getUdfs() {
     this.loading(true);
     const categories = [];
-    const functions = await getUdfCategories(this.connector);
+    const functions = await getUdfCategories(sqlReferenceRepository, this.connector);
     functions.forEach(category => {
       const koCategory = {
         name: category.name,
@@ -294,9 +294,9 @@ class AssistFunctionsPanel {
         (this.activeConnector() && this.activeConnector().type) ||
         getFromLocalStorage('assist.function.panel.active.connector.type');
 
-      const availableConnectorUdfs = filterEditorConnectors(hasUdfCategories).map(
-        connector => new ConnectorUdfCategories(connector)
-      );
+      const availableConnectorUdfs = filterEditorConnectors(
+        connector => connector.dialect && sqlReferenceRepository.hasUdfCategories(connector.dialect)
+      ).map(connector => new ConnectorUdfCategories(connector));
 
       availableConnectorUdfs.sort((a, b) =>
         a.connector.displayName.localeCompare(b.connector.displayName)

+ 3 - 2
desktop/core/src/desktop/js/ko/components/contextPopover/functionContext.js

@@ -17,7 +17,8 @@
 import * as ko from 'knockout';
 
 import I18n from 'utils/i18n';
-import { findUdf } from 'sql/reference/sqlReferenceRepository';
+import { findUdf } from 'sql/reference/sqlUdfRepository';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 
 const TEMPLATE_NAME = 'context-popover-function-details';
 
@@ -44,7 +45,7 @@ class FunctionContextTabs {
     this.loading = ko.observable(true);
     this.hasErrors = ko.observable(false);
 
-    findUdf(connector, data.function)
+    findUdf(sqlReferenceRepository, connector, data.function)
       .then(udfs => {
         // TODO: Support showing multiple UDFs with the same name but different category in the context popover.
         // For instance, trunc appears both for dates with one description and for numbers with another description.

+ 29 - 8
desktop/core/src/desktop/js/sql/autocompleteResults.js

@@ -32,9 +32,9 @@ import {
   findUdf,
   getArgumentDetailsForUdf,
   getUdfsWithReturnTypes,
-  getReturnTypesForUdf,
-  getSetOptions
-} from './reference/sqlReferenceRepository';
+  getReturnTypesForUdf
+} from './reference/sqlUdfRepository';
+import sqlReferenceRepository from './reference/sqlReferenceRepository';
 
 const normalizedColors = HueColors.getNormalizedColors();
 
@@ -482,6 +482,7 @@ class AutocompleteResults {
 
   async adjustForUdfArgument() {
     const foundArgumentDetails = (await getArgumentDetailsForUdf(
+      sqlReferenceRepository,
       this.snippet.connector(),
       this.parseResult.udfArgument.name,
       this.parseResult.udfArgument.position
@@ -670,7 +671,11 @@ class AutocompleteResults {
         });
       } else if (type === 'UDFREF') {
         try {
-          const types = await getReturnTypesForUdf(this.snippet.connector(), columnAlias.udfRef);
+          const types = await getReturnTypesForUdf(
+            sqlReferenceRepository,
+            this.snippet.connector(),
+            columnAlias.udfRef
+          );
           const resolvedType = types.length === 1 ? types[0] : 'T';
           columnAliasSuggestions.push({
             value: columnAlias.name,
@@ -723,7 +728,9 @@ class AutocompleteResults {
       return [];
     }
     try {
-      const setOptions = await getSetOptions(this.snippet.connector());
+      const setOptions = await sqlReferenceRepository.getSetOptions(
+        this.snippet.connector().dialect || ''
+      );
       return Object.keys(setOptions).map(name => ({
         category: CATEGORIES.OPTION,
         value: name,
@@ -752,6 +759,7 @@ class AutocompleteResults {
       const getUdfsForTypes = async types => {
         try {
           const functionsToSuggest = await getUdfsWithReturnTypes(
+            sqlReferenceRepository,
             this.snippet.connector(),
             types,
             this.parseResult.suggestAggregateFunctions || false,
@@ -785,7 +793,11 @@ class AutocompleteResults {
           const colRef = await colRefPromise;
           types = [colRef.type.toUpperCase()];
         } else {
-          types = await getReturnTypesForUdf(this.snippet.connector(), suggestFunctions.udfRef);
+          types = await getReturnTypesForUdf(
+            sqlReferenceRepository,
+            this.snippet.connector(),
+            suggestFunctions.udfRef
+          );
         }
       } catch (err) {}
       suggestions = await getUdfsForTypes(types);
@@ -794,6 +806,7 @@ class AutocompleteResults {
 
       try {
         const functionsToSuggest = await getUdfsWithReturnTypes(
+          sqlReferenceRepository,
           this.snippet.connector(),
           types,
           this.parseResult.suggestAggregateFunctions || false,
@@ -977,7 +990,11 @@ class AutocompleteResults {
         const colRef = await colRefPromise;
         types = [colRef.type.toUpperCase()];
       } else if (suggestColumns.types && suggestColumns.types[0] === 'UDFREF') {
-        types = await getReturnTypesForUdf(this.snippet.connector(), suggestColumns.udfRef);
+        types = await getReturnTypesForUdf(
+          sqlReferenceRepository,
+          this.snippet.connector(),
+          suggestColumns.udfRef
+        );
       }
     } catch (err) {}
 
@@ -1771,7 +1788,11 @@ class AutocompleteResults {
           clean = clean.replace(substitution.replace, substitution.with);
         });
 
-        const foundUdfs = await findUdf(this.snippet.connector(), value.aggregateFunction);
+        const foundUdfs = await findUdf(
+          sqlReferenceRepository,
+          this.snippet.connector(),
+          value.aggregateFunction
+        );
 
         // TODO: Support showing multiple UDFs with the same name but different category in the autocomplete details.
         // For instance, trunc appears both for dates with one description and for numbers with another description.

+ 6 - 5
desktop/core/src/desktop/js/sql/autocompleteResults.test.js

@@ -25,7 +25,8 @@ import dataCatalog from 'catalog/dataCatalog';
 import huePubSub from 'utils/huePubSub';
 import I18n from 'utils/i18n';
 import LOTS_OF_PARSE_RESULTS from './test/lotsOfParseResults';
-import * as sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
+import * as sqlUdfRepository from 'sql/reference/sqlUdfRepository';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 import { sleep } from '../utils/hueUtils';
 
 describe('AutocompleteResults.js', () => {
@@ -380,7 +381,7 @@ describe('AutocompleteResults.js', () => {
   it('should handle parse results with functions', async () => {
     subject.entries([]);
 
-    const spy = spyOn(sqlReferenceRepository, 'getUdfsWithReturnTypes').and.callFake(async () =>
+    const spy = spyOn(sqlUdfRepository, 'getUdfsWithReturnTypes').and.callFake(async () =>
       Promise.resolve([
         {
           name: 'count',
@@ -413,7 +414,7 @@ describe('AutocompleteResults.js', () => {
   it('should handle parse results with udf argument keywords', async () => {
     subject.entries([]);
 
-    const spy = spyOn(sqlReferenceRepository, 'getArgumentDetailsForUdf').and.callFake(async () =>
+    const spy = spyOn(sqlUdfRepository, 'getArgumentDetailsForUdf').and.callFake(async () =>
       Promise.resolve([{ type: 'T', keywords: ['a', 'b'] }])
     );
 
@@ -440,9 +441,9 @@ describe('AutocompleteResults.js', () => {
     subject.entries([]);
 
     const spy = spyOn(sqlReferenceRepository, 'getSetOptions').and.callFake(
-      async connector =>
+      async dialect =>
         new Promise(resolve => {
-          expect(connector).toEqual(subject.snippet.connector());
+          expect(dialect).toEqual(subject.snippet.connector().dialect);
           resolve({
             OPTION_1: {
               description: 'Desc 1',

+ 0 - 218
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.test.ts

@@ -1,218 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { UdfArgument } from 'sql/reference/types';
-import { Connector } from 'types/config';
-import { getArgumentDetailsForUdf, isReserved } from './sqlReferenceRepository';
-import * as apiUtils from 'sql/reference/apiUtils';
-
-describe('sqlReferenceRepository.js', () => {
-  const createTestConnector = (dialect: string, id: string): Connector => ({
-    dialect: dialect,
-    id: id,
-    buttonName: '',
-    displayName: '',
-    page: '',
-    tooltip: '',
-    type: ''
-  });
-
-  const hiveConn: Connector = createTestConnector('hive', 'hive');
-  const impalaConn = createTestConnector('impala', 'impala');
-
-  jest.mock('sql/reference/impala/udfReference', () => ({
-    UDF_CATEGORIES: [
-      {
-        functions: {
-          cos: {
-            returnTypes: ['DOUBLE'],
-            arguments: [[{ type: 'DOUBLE' }]],
-            signature: 'cos(DOUBLE a)',
-            draggable: 'cos()',
-            description: ''
-          },
-          concat: {
-            returnTypes: ['STRING'],
-            arguments: [[{ type: 'STRING' }], [{ type: 'STRING', multiple: true }]],
-            signature: 'concat(STRING a, STRING b...)',
-            draggable: 'concat()',
-            description: ''
-          },
-          strleft: {
-            returnTypes: ['STRING'],
-            arguments: [[{ type: 'STRING' }], [{ type: 'INT' }]],
-            signature: 'strleft(STRING a, INT num_chars)',
-            draggable: 'strleft()',
-            description: ''
-          }
-        }
-      }
-    ]
-  }));
-
-  jest.mock('sql/reference/hive/udfReference', () => ({
-    UDF_CATEGORIES: [
-      {
-        functions: {
-          cos: {
-            returnTypes: ['DOUBLE'],
-            arguments: [[{ type: 'DECIMAL' }, { type: 'DOUBLE' }]],
-            signature: 'cos(DECIMAL|DOUBLE a)',
-            draggable: 'cos()',
-            description: ''
-          },
-          concat: {
-            returnTypes: ['STRING'],
-            arguments: [
-              [
-                { type: 'STRING', multiple: true },
-                { type: 'BINARY', multiple: true }
-              ]
-            ],
-            signature: 'concat(STRING|BINARY a, STRING|BINARY b...)',
-            draggable: 'concat()',
-            description: ''
-          },
-          reflect: {
-            returnTypes: ['T'],
-            arguments: [
-              [{ type: 'STRING' }],
-              [{ type: 'STRING' }],
-              [{ type: 'T', multiple: true, optional: true }]
-            ],
-            signature: 'reflect(class, method[, arg1[, arg2..]])',
-            draggable: 'reflect()',
-            description: ''
-          }
-        }
-      }
-    ]
-  }));
-
-  jest.spyOn(apiUtils, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
-
-  const extractType = (details: UdfArgument): string => details.type;
-
-  it('should give the expected argument types at a specific position', async () => {
-    expect((await getArgumentDetailsForUdf(hiveConn, 'cos', 1)).map(extractType)).toEqual([
-      'DECIMAL',
-      'DOUBLE'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'cos', 2)).map(extractType)).toEqual([]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'cos', 1)).map(extractType)).toEqual([
-      'DOUBLE'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'cos', 2)).map(extractType)).toEqual([]);
-
-    expect((await getArgumentDetailsForUdf(hiveConn, 'concat', 10)).map(extractType)).toEqual([
-      'STRING',
-      'BINARY'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'concat', 10)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-  });
-
-  it('should handle functions with different type of arguments', async () => {
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 1)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 2)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 3)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'reflect', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-  });
-
-  it('should handle functions with an infinite amount of arguments', async () => {
-    expect((await getArgumentDetailsForUdf(impalaConn, 'greatest', 1)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'greatest', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 1)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 2)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 3)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'strleft', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-  });
-
-  it('should not return types for arguments out of bounds', async () => {
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 1)).map(extractType)).toEqual([
-      'STRING'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 2)).map(extractType)).toEqual([
-      'INT'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 3)).map(extractType)).toEqual([]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'strleft', 200)).map(extractType)).toEqual(
-      []
-    );
-  });
-
-  it("should return T for any argument if the udf isn't found", async () => {
-    expect((await getArgumentDetailsForUdf(hiveConn, 'blabla', 2)).map(extractType)).toEqual(['T']);
-    expect((await getArgumentDetailsForUdf(hiveConn, 'blabla', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'blabla', 2)).map(extractType)).toEqual([
-      'T'
-    ]);
-    expect((await getArgumentDetailsForUdf(impalaConn, 'blabla', 200)).map(extractType)).toEqual([
-      'T'
-    ]);
-  });
-
-  it("Should return generic keywords if the dialect isn't defined", async () => {
-    jest.mock('sql/reference/generic/reservedKeywords', () => ({
-      RESERVED_WORDS: new Set<string>(['GENERICRESERVED'])
-    }));
-
-    const reserved = await isReserved({ dialect: 'foo' } as Connector, 'GENERICRESERVED');
-    expect(reserved).toBeTruthy();
-
-    const notReserved = await isReserved({ dialect: 'foo' } as Connector, 'not_reserved');
-    expect(notReserved).toBeFalsy();
-  });
-
-  it('Should use custom keywords if defined for dialect', async () => {
-    jest.mock('sql/reference/calcite/reservedKeywords', () => ({
-      RESERVED_WORDS: new Set<string>(['CUSTOM'])
-    }));
-    jest.mock('sql/reference/generic/reservedKeywords', () => ({
-      RESERVED_WORDS: new Set<string>(['OTHER'])
-    }));
-
-    const reserved = await isReserved({ dialect: 'calcite' } as Connector, 'CUSTOM');
-    expect(reserved).toBeTruthy();
-
-    const notReserved = await isReserved({ dialect: 'calcite' } as Connector, 'OTHER');
-    expect(notReserved).toBeFalsy();
-  });
-});

+ 21 - 265
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.ts

@@ -14,25 +14,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import {
-  SetOptions,
-  UdfArgument,
-  UdfCategory,
-  UdfCategoryFunctions,
-  UdfDetails
-} from 'sql/reference/types';
-import { Connector } from 'types/config';
-import { matchesType } from './typeUtils';
-import I18n from 'utils/i18n';
-import huePubSub from 'utils/huePubSub';
-import { clearUdfCache, getCachedUdfCategories, setCachedUdfCategories } from './apiCache';
-import { fetchDescribe, fetchUdfs } from './apiUtils';
-
-export const CLEAR_UDF_CACHE_EVENT = 'hue.clear.udf.cache';
-export const DESCRIBE_UDF_EVENT = 'hue.describe.udf';
-export const UDF_DESCRIBED_EVENT = 'hue.udf.described';
+import { SetOptions, SqlReferenceProvider, UdfCategory } from 'sql/reference/types';
 
 const GENERIC = 'generic';
+const EMPTY_KEYWORDS = new Set<string>();
 
 const KEYWORD_REFS: { [attr: string]: () => Promise<{ RESERVED_WORDS?: Set<string> }> } = {
   calcite: async () => import(/* webpackChunkName: "calcite-ref" */ './calcite/reservedKeywords'),
@@ -57,261 +42,32 @@ const UDF_REFS: { [attr: string]: () => Promise<{ UDF_CATEGORIES?: UdfCategory[]
   flink: async () => import(/* webpackChunkName: "flink-ref" */ './flink/udfReference')
 };
 
-const IGNORED_UDF_REGEX = /^[!=$%&*+-/<>^|~]+$/;
-
-const mergedUdfPromises: { [attr: string]: Promise<UdfCategory[]> } = {};
-
-const getMergedUdfKey = (connector: Connector, database?: string): string => {
-  let key = connector.id;
-  if (database) {
-    key += '_' + database;
-  }
-  return key;
-};
-
-export const hasUdfCategories = (connector: Connector): boolean =>
-  !!connector.dialect && typeof UDF_REFS[connector.dialect] !== 'undefined';
-
-const findUdfsToAdd = (
-  apiUdfs: UdfDetails[],
-  existingCategories: UdfCategory[]
-): UdfCategoryFunctions => {
-  const existingUdfNames = new Set();
-  existingCategories.forEach(category => {
-    Object.keys(category.functions).forEach(udfName => {
-      existingUdfNames.add(udfName.toUpperCase());
-    });
-  });
-
-  const result: UdfCategoryFunctions = {};
-
-  apiUdfs.forEach(apiUdf => {
-    if (
-      !result[apiUdf.name] &&
-      !existingUdfNames.has(apiUdf.name.toUpperCase()) &&
-      !IGNORED_UDF_REGEX.test(apiUdf.name)
-    ) {
-      result[apiUdf.name] = apiUdf;
-    }
-  });
-
-  return result;
-};
-
-const mergeWithApiUdfs = async (
-  categories: UdfCategory[],
-  connector: Connector,
-  database?: string
-) => {
-  const apiUdfs = await fetchUdfs(connector, database);
-
-  if (apiUdfs.length) {
-    const additionalUdfs = findUdfsToAdd(apiUdfs, categories);
-    if (Object.keys(additionalUdfs).length) {
-      const generalCategory = {
-        name: I18n('General'),
-        functions: additionalUdfs
-      };
-      categories.unshift(generalCategory);
-    }
-  }
-};
-
-export const getUdfCategories = async (
-  connector: Connector,
-  database?: string
-): Promise<UdfCategory[]> => {
-  const promiseKey = getMergedUdfKey(connector, database);
-  if (!mergedUdfPromises[promiseKey]) {
-    mergedUdfPromises[promiseKey] = new Promise(async resolve => {
-      const cachedCategories = await getCachedUdfCategories(connector, database);
-      if (cachedCategories) {
-        resolve(cachedCategories);
-      }
-      let categories: UdfCategory[] = [];
-      if (connector.dialect && UDF_REFS[connector.dialect]) {
-        const module = await UDF_REFS[connector.dialect]();
-        if (module.UDF_CATEGORIES) {
-          categories = module.UDF_CATEGORIES;
-          categories.forEach(category => {
-            Object.values(category.functions).forEach(udf => {
-              udf.described = true;
-            });
-          });
-        }
-      }
-      await mergeWithApiUdfs(categories, connector, database);
-      await setCachedUdfCategories(connector, database, categories);
-      resolve(categories);
-    });
-  }
-
-  return await mergedUdfPromises[promiseKey];
-};
-
-export const findUdf = async (
-  connector: Connector,
-  functionName: string
-): Promise<UdfDetails[]> => {
-  const categories = await getUdfCategories(connector);
-  const found: UdfDetails[] = [];
-  categories.forEach(category => {
-    if (category.functions[functionName]) {
-      found.push(category.functions[functionName]);
-    }
-  });
-  return found;
-};
-
-export const getReturnTypesForUdf = async (
-  connector: Connector,
-  functionName: string
-): Promise<string[]> => {
-  if (!functionName) {
-    return ['T'];
-  }
-  const udfs = await findUdf(connector, functionName);
-  if (!udfs.length) {
-    let returnTypesPresent = false;
-    const returnTypes = new Set<string>();
-    udfs.forEach(udf => {
-      if (udf.returnTypes) {
-        returnTypesPresent = true;
-        udf.returnTypes.forEach(type => returnTypes.add(type));
-      }
-    });
-    if (returnTypesPresent) {
-      return [...returnTypes];
-    }
+class SqlReferenceRepository implements SqlReferenceProvider {
+  async getReservedKeywords(dialect: string): Promise<Set<string>> {
+    const refImport = KEYWORD_REFS[dialect] || KEYWORD_REFS[GENERIC];
+    const module = await refImport();
+    return module.RESERVED_WORDS || EMPTY_KEYWORDS;
   }
 
-  return ['T'];
-};
-
-export const getUdfsWithReturnTypes = async (
-  connector: Connector,
-  returnTypes: string[],
-  includeAggregate?: boolean,
-  includeAnalytic?: boolean
-): Promise<UdfDetails[]> => {
-  const categories = await getUdfCategories(connector);
-  const result: UdfDetails[] = [];
-  categories.forEach(category => {
-    if (
-      (!category.isAnalytic && !category.isAggregate) ||
-      (includeAggregate && category.isAggregate) ||
-      (includeAnalytic && category.isAnalytic)
-    ) {
-      Object.keys(category.functions).forEach(udfName => {
-        const udf = category.functions[udfName];
-        if (
-          !returnTypes ||
-          (connector.dialect && matchesType(connector.dialect, returnTypes, udf.returnTypes))
-        ) {
-          result.push(udf);
-        }
-      });
+  async getSetOptions(dialect: string): Promise<SetOptions> {
+    if (SET_REFS[dialect]) {
+      const module = await SET_REFS[dialect]();
+      return module.SET_OPTIONS || {};
     }
-  });
-  result.sort((a, b) => a.name.localeCompare(b.name));
-  return result;
-};
-
-export const getArgumentDetailsForUdf = async (
-  connector: Connector,
-  functionName: string,
-  argumentPosition: number
-): Promise<UdfArgument[]> => {
-  const foundFunctions = await findUdf(connector, functionName);
-  if (!foundFunctions.length) {
-    return [{ type: 'T' }];
+    return {};
   }
 
-  const possibleArguments: UdfArgument[] = [];
-  foundFunctions.forEach(foundFunction => {
-    const args = foundFunction.arguments;
-    if (argumentPosition > args.length) {
-      possibleArguments.push(...args[args.length - 1].filter(type => type.multiple));
-    } else {
-      possibleArguments.push(...args[argumentPosition - 1]);
-    }
-  });
-  return possibleArguments;
-};
-
-export const getSetOptions = async (connector: Connector): Promise<SetOptions> => {
-  if (connector.dialect && SET_REFS[connector.dialect]) {
-    const module = await SET_REFS[connector.dialect]();
-    if (module.SET_OPTIONS) {
-      return module.SET_OPTIONS;
-    }
+  async getUdfCategories(dialect: string): Promise<UdfCategory[]> {
+    const refImport = UDF_REFS[dialect] || UDF_REFS[GENERIC];
+    const module = await refImport();
+    return module.UDF_CATEGORIES || [];
   }
-  return {};
-};
 
-export const isReserved = async (connector: Connector, word: string): Promise<boolean> => {
-  const refImport = (connector.dialect && KEYWORD_REFS[connector.dialect]) || KEYWORD_REFS[GENERIC];
-  const module = await refImport();
-  if (module.RESERVED_WORDS) {
-    return module.RESERVED_WORDS.has(word.toUpperCase());
+  hasUdfCategories(dialect: string): boolean {
+    return !!UDF_REFS[dialect];
   }
+}
 
-  return false;
-};
-
-const findUdfInCategories = (
-  categories: UdfCategory[],
-  udfName: string
-): UdfDetails | undefined => {
-  let foundUdf = undefined;
-  categories.some(category =>
-    Object.values(category.functions).some(udf => {
-      if (udf.name === udfName) {
-        foundUdf = udf;
-        return true;
-      }
-    })
-  );
-  return foundUdf;
-};
+const sqlReferenceRepository = new SqlReferenceRepository();
 
-huePubSub.subscribe(
-  DESCRIBE_UDF_EVENT,
-  async (details: { connector: Connector; udfName: string; database?: string }): Promise<void> => {
-    const categories = await getUdfCategories(details.connector, details.database);
-    const foundUdf = findUdfInCategories(categories, details.udfName);
-    if (foundUdf && !foundUdf.described) {
-      const apiUdf = await fetchDescribe(details.connector, foundUdf, details.database);
-      if (apiUdf) {
-        if (apiUdf.description) {
-          foundUdf.description = apiUdf.description;
-        }
-        if (apiUdf.signature) {
-          foundUdf.signature = apiUdf.signature;
-        }
-        foundUdf.described = true;
-        await setCachedUdfCategories(details.connector, details.database, categories);
-        huePubSub.publish(UDF_DESCRIBED_EVENT, {
-          connector: details.connector,
-          database: details.database,
-          udf: foundUdf
-        });
-      }
-    }
-  }
-);
-
-huePubSub.subscribe(
-  CLEAR_UDF_CACHE_EVENT,
-  async (details: { connector: Connector; callback: () => void }) => {
-    await clearUdfCache(details.connector);
-    Object.keys(mergedUdfPromises).forEach(key => {
-      if (key === details.connector.id || key.indexOf(details.connector.id + '_') === 0) {
-        delete mergedUdfPromises[key];
-      }
-    });
-    if (details.callback) {
-      details.callback();
-    }
-  }
-);
+export default sqlReferenceRepository;

+ 242 - 0
desktop/core/src/desktop/js/sql/reference/sqlUdfRepository.test.ts

@@ -0,0 +1,242 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { UdfArgument } from 'sql/reference/types';
+import { Connector } from 'types/config';
+import { getArgumentDetailsForUdf } from './sqlUdfRepository';
+import sqlReferenceRepository from './sqlReferenceRepository';
+import * as apiUtils from 'sql/reference/apiUtils';
+
+describe('sqlUdfRepository.ts', () => {
+  const createTestConnector = (dialect: string, id: string): Connector => ({
+    dialect: dialect,
+    id: id,
+    buttonName: '',
+    displayName: '',
+    page: '',
+    tooltip: '',
+    type: ''
+  });
+
+  const hiveConn: Connector = createTestConnector('hive', 'hive');
+  const impalaConn = createTestConnector('impala', 'impala');
+
+  jest.mock('sql/reference/impala/udfReference', () => ({
+    UDF_CATEGORIES: [
+      {
+        functions: {
+          cos: {
+            returnTypes: ['DOUBLE'],
+            arguments: [[{ type: 'DOUBLE' }]],
+            signature: 'cos(DOUBLE a)',
+            draggable: 'cos()',
+            description: ''
+          },
+          concat: {
+            returnTypes: ['STRING'],
+            arguments: [[{ type: 'STRING' }], [{ type: 'STRING', multiple: true }]],
+            signature: 'concat(STRING a, STRING b...)',
+            draggable: 'concat()',
+            description: ''
+          },
+          strleft: {
+            returnTypes: ['STRING'],
+            arguments: [[{ type: 'STRING' }], [{ type: 'INT' }]],
+            signature: 'strleft(STRING a, INT num_chars)',
+            draggable: 'strleft()',
+            description: ''
+          }
+        }
+      }
+    ]
+  }));
+
+  jest.mock('sql/reference/hive/udfReference', () => ({
+    UDF_CATEGORIES: [
+      {
+        functions: {
+          cos: {
+            returnTypes: ['DOUBLE'],
+            arguments: [[{ type: 'DECIMAL' }, { type: 'DOUBLE' }]],
+            signature: 'cos(DECIMAL|DOUBLE a)',
+            draggable: 'cos()',
+            description: ''
+          },
+          concat: {
+            returnTypes: ['STRING'],
+            arguments: [
+              [
+                { type: 'STRING', multiple: true },
+                { type: 'BINARY', multiple: true }
+              ]
+            ],
+            signature: 'concat(STRING|BINARY a, STRING|BINARY b...)',
+            draggable: 'concat()',
+            description: ''
+          },
+          reflect: {
+            returnTypes: ['T'],
+            arguments: [
+              [{ type: 'STRING' }],
+              [{ type: 'STRING' }],
+              [{ type: 'T', multiple: true, optional: true }]
+            ],
+            signature: 'reflect(class, method[, arg1[, arg2..]])',
+            draggable: 'reflect()',
+            description: ''
+          }
+        }
+      }
+    ]
+  }));
+
+  jest.spyOn(apiUtils, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+
+  const extractType = (details: UdfArgument): string => details.type;
+
+  it('should give the expected argument types at a specific position', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'cos', 1)).map(extractType)
+    ).toEqual(['DECIMAL', 'DOUBLE']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'cos', 2)).map(extractType)
+    ).toEqual([]);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'cos', 1)).map(
+        extractType
+      )
+    ).toEqual(['DOUBLE']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'cos', 2)).map(
+        extractType
+      )
+    ).toEqual([]);
+
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'concat', 10)).map(
+        extractType
+      )
+    ).toEqual(['STRING', 'BINARY']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'concat', 10)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+  });
+
+  it('should handle functions with different type of arguments', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 1)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 2)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 3)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'reflect', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+  });
+
+  it('should handle functions with an infinite amount of arguments', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'greatest', 1)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'greatest', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 1)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 2)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 3)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'strleft', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+  });
+
+  it('should not return types for arguments out of bounds', async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 1)).map(
+        extractType
+      )
+    ).toEqual(['STRING']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 2)).map(
+        extractType
+      )
+    ).toEqual(['INT']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 3)).map(
+        extractType
+      )
+    ).toEqual([]);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'strleft', 200)).map(
+        extractType
+      )
+    ).toEqual([]);
+  });
+
+  it("should return T for any argument if the udf isn't found", async () => {
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'blabla', 2)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, hiveConn, 'blabla', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'blabla', 2)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+    expect(
+      (await getArgumentDetailsForUdf(sqlReferenceRepository, impalaConn, 'blabla', 200)).map(
+        extractType
+      )
+    ).toEqual(['T']);
+  });
+});

+ 280 - 0
desktop/core/src/desktop/js/sql/reference/sqlUdfRepository.ts

@@ -0,0 +1,280 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  SqlReferenceProvider,
+  UdfArgument,
+  UdfCategory,
+  UdfCategoryFunctions,
+  UdfDetails
+} from 'sql/reference/types';
+import { Connector } from 'types/config';
+import { matchesType } from './typeUtils';
+import I18n from 'utils/i18n';
+import huePubSub from 'utils/huePubSub';
+import { clearUdfCache, getCachedUdfCategories, setCachedUdfCategories } from './apiCache';
+import { fetchDescribe, fetchUdfs } from './apiUtils';
+
+export const CLEAR_UDF_CACHE_EVENT = 'hue.clear.udf.cache';
+export const DESCRIBE_UDF_EVENT = 'hue.describe.udf';
+export const UDF_DESCRIBED_EVENT = 'hue.udf.described';
+
+const IGNORED_UDF_REGEX = /^[!=$%&*+-/<>^|~]+$/;
+
+const mergedUdfPromises: { [attr: string]: Promise<UdfCategory[]> } = {};
+
+const getMergedUdfKey = (connector: Connector, database?: string): string => {
+  let key = connector.id;
+  if (database) {
+    key += '_' + database;
+  }
+  return key;
+};
+
+const findUdfsToAdd = (
+  apiUdfs: UdfDetails[],
+  existingCategories: UdfCategory[]
+): UdfCategoryFunctions => {
+  const existingUdfNames = new Set();
+  existingCategories.forEach(category => {
+    Object.keys(category.functions).forEach(udfName => {
+      existingUdfNames.add(udfName.toUpperCase());
+    });
+  });
+
+  const result: UdfCategoryFunctions = {};
+
+  apiUdfs.forEach(apiUdf => {
+    if (
+      !result[apiUdf.name] &&
+      !existingUdfNames.has(apiUdf.name.toUpperCase()) &&
+      !IGNORED_UDF_REGEX.test(apiUdf.name)
+    ) {
+      result[apiUdf.name] = apiUdf;
+    }
+  });
+
+  return result;
+};
+
+const mergeWithApiUdfs = async (
+  categories: UdfCategory[],
+  connector: Connector,
+  database?: string
+) => {
+  const apiUdfs = await fetchUdfs(connector, database);
+
+  if (apiUdfs.length) {
+    const additionalUdfs = findUdfsToAdd(apiUdfs, categories);
+    if (Object.keys(additionalUdfs).length) {
+      const generalCategory = {
+        name: I18n('General'),
+        functions: additionalUdfs
+      };
+      categories.unshift(generalCategory);
+    }
+  }
+};
+
+export const getUdfCategories = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  database?: string
+): Promise<UdfCategory[]> => {
+  const promiseKey = getMergedUdfKey(connector, database);
+  if (!mergedUdfPromises[promiseKey]) {
+    mergedUdfPromises[promiseKey] = new Promise(async resolve => {
+      const cachedCategories = await getCachedUdfCategories(connector, database);
+      if (cachedCategories) {
+        resolve(cachedCategories);
+      }
+      let categories: UdfCategory[] = [];
+      if (connector.dialect && sqlReferenceProvider.hasUdfCategories(connector.dialect)) {
+        categories = await sqlReferenceProvider.getUdfCategories(connector.dialect);
+        categories.forEach(category => {
+          Object.values(category.functions).forEach(udf => {
+            udf.described = true;
+          });
+        });
+      }
+      await mergeWithApiUdfs(categories, connector, database);
+      await setCachedUdfCategories(connector, database, categories);
+      resolve(categories);
+    });
+  }
+
+  return await mergedUdfPromises[promiseKey];
+};
+
+export const findUdf = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  functionName: string
+): Promise<UdfDetails[]> => {
+  const categories = await getUdfCategories(sqlReferenceProvider, connector);
+  const found: UdfDetails[] = [];
+  categories.forEach(category => {
+    if (category.functions[functionName]) {
+      found.push(category.functions[functionName]);
+    }
+  });
+  return found;
+};
+
+export const getReturnTypesForUdf = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  functionName: string
+): Promise<string[]> => {
+  if (!functionName) {
+    return ['T'];
+  }
+  const udfs = await findUdf(sqlReferenceProvider, connector, functionName);
+  if (!udfs.length) {
+    let returnTypesPresent = false;
+    const returnTypes = new Set<string>();
+    udfs.forEach(udf => {
+      if (udf.returnTypes) {
+        returnTypesPresent = true;
+        udf.returnTypes.forEach(type => returnTypes.add(type));
+      }
+    });
+    if (returnTypesPresent) {
+      return [...returnTypes];
+    }
+  }
+
+  return ['T'];
+};
+
+export const getUdfsWithReturnTypes = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  returnTypes: string[],
+  includeAggregate?: boolean,
+  includeAnalytic?: boolean
+): Promise<UdfDetails[]> => {
+  const categories = await getUdfCategories(sqlReferenceProvider, connector);
+  const result: UdfDetails[] = [];
+  categories.forEach(category => {
+    if (
+      (!category.isAnalytic && !category.isAggregate) ||
+      (includeAggregate && category.isAggregate) ||
+      (includeAnalytic && category.isAnalytic)
+    ) {
+      Object.keys(category.functions).forEach(udfName => {
+        const udf = category.functions[udfName];
+        if (
+          !returnTypes ||
+          (connector.dialect && matchesType(connector.dialect, returnTypes, udf.returnTypes))
+        ) {
+          result.push(udf);
+        }
+      });
+    }
+  });
+  result.sort((a, b) => a.name.localeCompare(b.name));
+  return result;
+};
+
+export const getArgumentDetailsForUdf = async (
+  sqlReferenceProvider: SqlReferenceProvider,
+  connector: Connector,
+  functionName: string,
+  argumentPosition: number
+): Promise<UdfArgument[]> => {
+  const foundFunctions = await findUdf(sqlReferenceProvider, connector, functionName);
+  if (!foundFunctions.length) {
+    return [{ type: 'T' }];
+  }
+
+  const possibleArguments: UdfArgument[] = [];
+  foundFunctions.forEach(foundFunction => {
+    const args = foundFunction.arguments;
+    if (argumentPosition > args.length) {
+      possibleArguments.push(...args[args.length - 1].filter(type => type.multiple));
+    } else {
+      possibleArguments.push(...args[argumentPosition - 1]);
+    }
+  });
+  return possibleArguments;
+};
+
+const findUdfInCategories = (
+  categories: UdfCategory[],
+  udfName: string
+): UdfDetails | undefined => {
+  let foundUdf = undefined;
+  categories.some(category =>
+    Object.values(category.functions).some(udf => {
+      if (udf.name === udfName) {
+        foundUdf = udf;
+        return true;
+      }
+    })
+  );
+  return foundUdf;
+};
+
+huePubSub.subscribe(
+  DESCRIBE_UDF_EVENT,
+  async (details: {
+    sqlReferenceProvider: SqlReferenceProvider;
+    connector: Connector;
+    udfName: string;
+    database?: string;
+  }): Promise<void> => {
+    const categories = await getUdfCategories(
+      details.sqlReferenceProvider,
+      details.connector,
+      details.database
+    );
+    const foundUdf = findUdfInCategories(categories, details.udfName);
+    if (foundUdf && !foundUdf.described) {
+      const apiUdf = await fetchDescribe(details.connector, foundUdf, details.database);
+      if (apiUdf) {
+        if (apiUdf.description) {
+          foundUdf.description = apiUdf.description;
+        }
+        if (apiUdf.signature) {
+          foundUdf.signature = apiUdf.signature;
+        }
+        foundUdf.described = true;
+        await setCachedUdfCategories(details.connector, details.database, categories);
+        huePubSub.publish(UDF_DESCRIBED_EVENT, {
+          connector: details.connector,
+          database: details.database,
+          udf: foundUdf
+        });
+      }
+    }
+  }
+);
+
+huePubSub.subscribe(
+  CLEAR_UDF_CACHE_EVENT,
+  async (details: { connector: Connector; callback: () => void }) => {
+    await clearUdfCache(details.connector);
+    Object.keys(mergedUdfPromises).forEach(key => {
+      if (key === details.connector.id || key.indexOf(details.connector.id + '_') === 0) {
+        delete mergedUdfPromises[key];
+      }
+    });
+    if (details.callback) {
+      details.callback();
+    }
+  }
+);

+ 7 - 0
desktop/core/src/desktop/js/sql/reference/types.ts

@@ -56,3 +56,10 @@ export interface SetDetails {
 export interface TypeConversion {
   [attr: string]: { [attr: string]: boolean };
 }
+
+export interface SqlReferenceProvider {
+  getReservedKeywords(dialect: string): Promise<Set<string>>;
+  getSetOptions(dialect: string): Promise<SetOptions>;
+  getUdfCategories(dialect: string): Promise<UdfCategory[]>;
+  hasUdfCategories(dialect: string): boolean;
+}

+ 5 - 2
desktop/core/src/desktop/js/sql/sqlUtils.ts

@@ -20,7 +20,7 @@ import DataCatalogEntry from 'catalog/DataCatalogEntry';
 
 import dataCatalog from 'catalog/dataCatalog';
 import { IdentifierChainEntry, ParsedLocation, ParsedTable } from 'parse/types';
-import { isReserved } from 'sql/reference/sqlReferenceRepository';
+import sqlReferenceRepository from 'sql/reference/sqlReferenceRepository';
 import {
   CommentDetails,
   Suggestion
@@ -266,7 +266,10 @@ export default {
     if (identifier.indexOf(quoteChar) === 0) {
       return identifier;
     }
-    if (await isReserved(connector, identifier)) {
+    const reservedKeywords = await sqlReferenceRepository.getReservedKeywords(
+      connector.dialect || 'generic'
+    );
+    if (reservedKeywords.has(identifier.toUpperCase())) {
       return quoteChar + identifier + quoteChar;
     }