Преглед на файлове

HUE-7738 [editor] Extract argument and return types from the UDF API response

Johan Ahlen преди 5 години
родител
ревизия
24ee8960f9

+ 49 - 124
desktop/core/src/desktop/js/api/apiHelper.js

@@ -22,7 +22,6 @@ import {
   cancelActiveRequest,
   simpleGet,
   simplePost,
-  simplePostAsync,
   successResponseIsError
 } from './apiUtils';
 import apiQueueManager from 'api/apiQueueManager';
@@ -31,52 +30,7 @@ import hueDebug from 'utils/hueDebug';
 import huePubSub from 'utils/huePubSub';
 import hueUtils from 'utils/hueUtils';
 import { EXECUTION_STATUS } from 'apps/notebook2/execution/executable';
-
-const AUTOCOMPLETE_API_PREFIX = '/notebook/api/autocomplete/';
-const SAMPLE_API_PREFIX = '/notebook/api/sample/';
-const EXECUTE_API_PREFIX = '/notebook/api/execute/';
-const DOCUMENTS_API = '/desktop/api2/doc/';
-const DOCUMENTS_SEARCH_API = '/desktop/api2/docs/';
-const GET_HUE_CONFIG_URL = '/desktop/api2/get_hue_config';
-const FETCH_CONFIG = '/desktop/api2/get_config/';
-const HDFS_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('/');
-const ADLS_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('adl:/');
-const ABFS_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('ABFS://');
-const GIT_API_PREFIX = '/desktop/api/vcs/contents/';
-const S3_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('S3A://');
-const IMPALA_INVALIDATE_API = '/impala/api/invalidate';
-const CONFIG_SAVE_API = '/desktop/api/configurations/save/';
-const CONFIG_APPS_API = '/desktop/api/configurations';
-const SOLR_COLLECTIONS_API = '/indexer/api/indexes/list/';
-const SOLR_FIELDS_API = '/indexer/api/index/list/';
-const DASHBOARD_TERMS_API = '/dashboard/get_terms';
-const DASHBOARD_STATS_API = '/dashboard/get_stats';
-const FORMAT_SQL_API = '/notebook/api/format';
-const GIST_API = '/desktop/api2/gist/';
-const TOPO_URL = '/desktop/topo/';
-
-const SEARCH_API = '/desktop/api/search/entities';
-const INTERACTIVE_SEARCH_API = '/desktop/api/search/entities_interactive';
-
-const HBASE_API_PREFIX = '/hbase/api/';
-const SAVE_TO_FILE = '/filebrowser/save';
-
-const NAV_URLS = {
-  ADD_TAGS: '/metadata/api/catalog/add_tags',
-  DELETE_TAGS: '/metadata/api/catalog/delete_tags',
-  FIND_ENTITY: '/metadata/api/catalog/find_entity',
-  LIST_TAGS: '/metadata/api/catalog/list_tags',
-  UPDATE_PROPERTIES: '/metadata/api/catalog/update_properties'
-};
-
-const NAV_OPT_URLS = {
-  TOP_AGGS: '/metadata/api/optimizer/top_aggs',
-  TOP_COLUMNS: '/metadata/api/optimizer/top_columns',
-  TOP_FILTERS: '/metadata/api/optimizer/top_filters',
-  TOP_JOINS: '/metadata/api/optimizer/top_joins',
-  TOP_TABLES: '/metadata/api/optimizer/top_tables',
-  TABLE_DETAILS: '/metadata/api/optimizer/table_details'
-};
+import * as URLS from './urls';
 
 export const LINK_SHARING_PERMS = {
   READ: 'read',
@@ -297,7 +251,7 @@ class ApiHelper {
    */
   saveSnippetToFile(data, options) {
     $.post(
-      SAVE_TO_FILE,
+      URLS.SAVE_TO_FILE_API,
       data,
       result => {
         if (typeof options.successCallback !== 'undefined') {
@@ -345,7 +299,7 @@ class ApiHelper {
    * @param {boolean} [options.silenceErrors]
    */
   fetchTopo(options) {
-    const url = TOPO_URL + options.location;
+    const url = URLS.TOPO_URL + options.location;
     return simpleGet(url, undefined, options);
   }
 
@@ -361,13 +315,13 @@ class ApiHelper {
   fetchStoragePreview(options) {
     let url;
     if (options.type === 's3') {
-      url = S3_API_PREFIX;
+      url = URLS.S3_API_PREFIX;
     } else if (options.type === 'adls') {
-      url = ADLS_API_PREFIX;
+      url = URLS.ADLS_API_PREFIX;
     } else if (options.type === 'abfs') {
-      url = ABFS_API_PREFIX;
+      url = URLS.ABFS_API_PREFIX;
     } else {
-      url = HDFS_API_PREFIX;
+      url = URLS.HDFS_API_PREFIX;
     }
 
     const clonedPath = options.path.concat();
@@ -416,7 +370,7 @@ class ApiHelper {
       options.pathParts.shift();
     }
     let url =
-      HDFS_API_PREFIX +
+      URLS.HDFS_API_PREFIX +
       encodeURI(options.pathParts.join('/')) +
       '?format=json&sortby=name&descending=false&pagesize=' +
       (options.pageSize || 500) +
@@ -483,7 +437,7 @@ class ApiHelper {
   fetchAdlsPath(options) {
     options.pathParts.shift();
     let url =
-      ADLS_API_PREFIX +
+      URLS.ADLS_API_PREFIX +
       encodeURI(options.pathParts.join('/')) +
       '?format=json&sortby=name&descending=false&pagesize=' +
       (options.pageSize || 500) +
@@ -549,7 +503,7 @@ class ApiHelper {
    */
   fetchAbfsPath(options) {
     let url =
-      ABFS_API_PREFIX +
+      URLS.ABFS_API_PREFIX +
       encodeURI(options.pathParts.join('/')) +
       '?format=json&sortby=name&descending=false&pagesize=' +
       (options.pageSize || 500) +
@@ -612,7 +566,7 @@ class ApiHelper {
    */
   fetchGitContents(options) {
     const url =
-      GIT_API_PREFIX +
+      URLS.GIT_API_PREFIX +
       '?path=' +
       encodeURI(options.pathParts.join('/')) +
       '&fileType=' +
@@ -676,7 +630,7 @@ class ApiHelper {
   fetchS3Path(options) {
     options.pathParts.shift(); // remove the trailing /
     let url =
-      S3_API_PREFIX +
+      URLS.S3_API_PREFIX +
       encodeURI(options.pathParts.join('/')) +
       '?format=json&sortby=name&descending=false&pagesize=' +
       (options.pageSize || 500) +
@@ -762,7 +716,7 @@ class ApiHelper {
     }
     $.ajax({
       dataType: 'json',
-      url: DASHBOARD_TERMS_API,
+      url: URLS.DASHBOARD_TERMS_API,
       type: 'POST',
       data: {
         collection: ko.mapping.toJSON({
@@ -807,7 +761,7 @@ class ApiHelper {
     }
     $.ajax({
       dataType: 'json',
-      url: DASHBOARD_STATS_API,
+      url: URLS.DASHBOARD_STATS_API,
       type: 'POST',
       data: {
         collection: ko.mapping.toJSON({
@@ -854,7 +808,7 @@ class ApiHelper {
     if (options.parent.name !== '') {
       suffix = 'getTableList/' + options.parent.name;
     }
-    const url = HBASE_API_PREFIX + suffix;
+    const url = URLS.HBASE_API_PREFIX + suffix;
     const fetchFunction = storeInCache => {
       if (options.timeout === 0) {
         assistErrorCallback(options)({ status: -1 });
@@ -951,12 +905,12 @@ class ApiHelper {
    * @param {boolean} [options.silenceErrors]
    */
   fetchConfigurations(options) {
-    simpleGet(CONFIG_APPS_API, {}, options);
+    simpleGet(URLS.CONFIG_APPS_API, {}, options);
   }
 
   saveGlobalConfiguration(options) {
     simplePost(
-      CONFIG_APPS_API,
+      URLS.CONFIG_APPS_API,
       {
         configuration: ko.mapping.toJSON(options.configuration)
       },
@@ -978,7 +932,7 @@ class ApiHelper {
    */
   saveConfiguration(options) {
     simplePost(
-      CONFIG_SAVE_API,
+      URLS.CONFIG_SAVE_API,
       {
         app: options.app,
         properties: ko.mapping.toJSON(options.properties),
@@ -1007,11 +961,11 @@ class ApiHelper {
       id += options.type;
     }
 
-    let promise = this.queueManager.getQueued(DOCUMENTS_API, id);
+    let promise = this.queueManager.getQueued(URLS.DOCUMENTS_API, id);
     const firstInQueue = typeof promise === 'undefined';
     if (firstInQueue) {
       promise = $.Deferred();
-      this.queueManager.addToQueue(promise, DOCUMENTS_API, id);
+      this.queueManager.addToQueue(promise, URLS.DOCUMENTS_API, id);
     }
 
     promise.done(options.successCallback).fail(assistErrorCallback(options));
@@ -1029,7 +983,7 @@ class ApiHelper {
     }
 
     $.ajax({
-      url: DOCUMENTS_API,
+      url: URLS.DOCUMENTS_API,
       data: data,
       traditional: true,
       success: data => {
@@ -1056,7 +1010,7 @@ class ApiHelper {
    */
   searchDocuments(options) {
     return $.ajax({
-      url: DOCUMENTS_SEARCH_API,
+      url: URLS.DOCUMENTS_SEARCH_API,
       data: {
         uuid: options.uuid,
         text: options.query,
@@ -1087,7 +1041,7 @@ class ApiHelper {
   fetchDocument(options) {
     const deferred = $.Deferred();
     const request = $.ajax({
-      url: DOCUMENTS_API,
+      url: URLS.DOCUMENTS_API,
       data: {
         uuid: options.uuid,
         data: !!options.fetchContents,
@@ -1162,7 +1116,7 @@ class ApiHelper {
    */
   createDocumentsFolder(options) {
     simplePost(
-      DOCUMENTS_API + 'mkdir',
+      URLS.DOCUMENTS_API + 'mkdir',
       {
         parent_uuid: ko.mapping.toJSON(options.parentUuid),
         name: ko.mapping.toJSON(options.name)
@@ -1182,7 +1136,7 @@ class ApiHelper {
    */
   updateDocument(options) {
     simplePost(
-      DOCUMENTS_API + 'update',
+      URLS.DOCUMENTS_API + 'update',
       {
         uuid: ko.mapping.toJSON(options.uuid),
         name: options.name
@@ -1202,7 +1156,7 @@ class ApiHelper {
    */
   uploadDocument(options) {
     $.ajax({
-      url: DOCUMENTS_API + 'import',
+      url: URLS.DOCUMENTS_API + 'import',
       type: 'POST',
       success: data => {
         if (!successResponseIsError(data)) {
@@ -1237,7 +1191,7 @@ class ApiHelper {
    */
   moveDocument(options) {
     simplePost(
-      DOCUMENTS_API + 'move',
+      URLS.DOCUMENTS_API + 'move',
       {
         source_doc_uuid: ko.mapping.toJSON(options.sourceId),
         destination_doc_uuid: ko.mapping.toJSON(options.destinationId)
@@ -1257,7 +1211,7 @@ class ApiHelper {
    */
   deleteDocument(options) {
     simplePost(
-      DOCUMENTS_API + 'delete',
+      URLS.DOCUMENTS_API + 'delete',
       {
         uuid: ko.mapping.toJSON(options.uuid),
         skip_trash: ko.mapping.toJSON(options.skipTrash || false)
@@ -1276,7 +1230,7 @@ class ApiHelper {
    */
   copyDocument(options) {
     simplePost(
-      DOCUMENTS_API + 'copy',
+      URLS.DOCUMENTS_API + 'copy',
       {
         uuid: ko.mapping.toJSON(options.uuid)
       },
@@ -1294,7 +1248,7 @@ class ApiHelper {
    */
   restoreDocument(options) {
     simplePost(
-      DOCUMENTS_API + 'restore',
+      URLS.DOCUMENTS_API + 'restore',
       {
         uuids: ko.mapping.toJSON(options.uuids)
       },
@@ -1317,7 +1271,7 @@ class ApiHelper {
     if (options.clearAll) {
       $.totalStorage(cacheIdentifier, {});
     } else {
-      let url = AUTOCOMPLETE_API_PREFIX;
+      let url = URLS.AUTOCOMPLETE_API_PREFIX;
       if (options.databaseName) {
         url += options.databaseName;
       }
@@ -1360,7 +1314,7 @@ class ApiHelper {
         data.table = options.path[1];
       }
 
-      const request = simplePost(IMPALA_INVALIDATE_API, data, options)
+      const request = simplePost(URLS.IMPALA_INVALIDATE_API, data, options)
         .done(deferred.resolve)
         .fail(deferred.reject);
 
@@ -1370,36 +1324,6 @@ class ApiHelper {
     return deferred.resolve().promise();
   }
 
-  /**
-   * @param {Object} options
-   * @param {Connector} options.connector
-   * @param {string} [options.database]
-   * @param {boolean} [options.silenceErrors]
-   *
-   * @return {Promise}
-   */
-  async fetchUdfs(options) {
-    let url = AUTOCOMPLETE_API_PREFIX;
-    if (options.database) {
-      url += '/' + options.database;
-    }
-
-    const data = {
-      notebook: {},
-      snippet: JSON.stringify({
-        type: options.connector.id
-      }),
-      operation: 'functions'
-    };
-
-    try {
-      const response = await simplePostAsync(url, data, options);
-      return (response && response.functions) || [];
-    } catch (err) {
-      return [];
-    }
-  }
-
   /**
    * @param {Object} options
    * @param {string} options.sourceType
@@ -1419,7 +1343,8 @@ class ApiHelper {
 
     const request = $.ajax({
       type: 'POST',
-      url: AUTOCOMPLETE_API_PREFIX + (isQuery ? options.path.slice(1) : options.path).join('/'),
+      url:
+        URLS.AUTOCOMPLETE_API_PREFIX + (isQuery ? options.path.slice(1) : options.path).join('/'),
       data: {
         notebook: {},
         snippet: ko.mapping.toJSON({
@@ -1898,7 +1823,7 @@ class ApiHelper {
    */
   async executeStatement(options) {
     const executable = options.executable;
-    const url = EXECUTE_API_PREFIX + executable.executor.connector().dialect;
+    const url = URLS.EXECUTE_API_PREFIX + executable.executor.connector().dialect;
 
     const promise = new Promise(async (resolve, reject) => {
       let data = {};
@@ -1995,7 +1920,7 @@ class ApiHelper {
       description: options.description
     };
     return new Promise((resolve, reject) => {
-      simplePost(GIST_API + 'create', data, options)
+      simplePost(URLS.GIST_API + 'create', data, options)
         .done(response => {
           resolve(response.link);
         })
@@ -2230,7 +2155,7 @@ class ApiHelper {
     };
 
     simplePost(
-      SAMPLE_API_PREFIX + options.path.join('/'),
+      URLS.SAMPLE_API_PREFIX + options.path.join('/'),
       {
         notebook: {},
         snippet: JSON.stringify({
@@ -2333,7 +2258,7 @@ class ApiHelper {
    */
   fetchNavigatorMetadata(options) {
     const deferred = $.Deferred();
-    let url = NAV_URLS.FIND_ENTITY;
+    let url = URLS.NAV_API.FIND_ENTITY;
 
     if (options.path.length === 1) {
       url += '?type=database&name=' + options.path[0];
@@ -2402,7 +2327,7 @@ class ApiHelper {
     if (options.deletedCustomMetadataKeys) {
       data.deletedCustomMetadataKeys = ko.mapping.toJSON(options.deletedCustomMetadataKeys);
     }
-    return simplePost(NAV_URLS.UPDATE_PROPERTIES, data, options);
+    return simplePost(URLS.NAV_API.UPDATE_PROPERTIES, data, options);
   }
 
   /**
@@ -2416,7 +2341,7 @@ class ApiHelper {
   fetchAllNavigatorTags(options) {
     const deferred = $.Deferred();
 
-    const request = simplePost(NAV_URLS.LIST_TAGS, undefined, {
+    const request = simplePost(URLS.NAV_API.LIST_TAGS, undefined, {
       silenceErrors: options.silenceErrors,
       successCallback: data => {
         if (data && data.tags) {
@@ -2432,14 +2357,14 @@ class ApiHelper {
   }
 
   addNavTags(entityId, tags) {
-    return simplePost(NAV_URLS.ADD_TAGS, {
+    return simplePost(URLS.NAV_API.ADD_TAGS, {
       id: ko.mapping.toJSON(entityId),
       tags: ko.mapping.toJSON(tags)
     });
   }
 
   deleteNavTags(entityId, tags) {
-    return simplePost(NAV_URLS.DELETE_TAGS, {
+    return simplePost(URLS.NAV_API.DELETE_TAGS, {
       id: ko.mapping.toJSON(entityId),
       tags: ko.mapping.toJSON(tags)
     });
@@ -2589,7 +2514,7 @@ class ApiHelper {
 
   async fetchHueConfigAsync(options) {
     return new Promise((resolve, reject) => {
-      $.get(GET_HUE_CONFIG_URL)
+      $.get(URLS.GET_HUE_CONFIG_API)
         .done(response => {
           if (!response && response.status === -1) {
             reject(response.message);
@@ -2602,12 +2527,12 @@ class ApiHelper {
   }
 
   getClusterConfig(data) {
-    return $.post(FETCH_CONFIG, data);
+    return $.post(URLS.FETCH_CONFIG_API, data);
   }
 
   fetchHueDocsInteractive(query) {
     const deferred = $.Deferred();
-    const request = $.post(INTERACTIVE_SEARCH_API, {
+    const request = $.post(URLS.INTERACTIVE_SEARCH_API, {
       query_s: ko.mapping.toJSON(query),
       limit: 50,
       sources: '["documents"]'
@@ -2625,7 +2550,7 @@ class ApiHelper {
 
   fetchNavEntitiesInteractive(options) {
     const deferred = $.Deferred();
-    const request = $.post(INTERACTIVE_SEARCH_API, {
+    const request = $.post(URLS.INTERACTIVE_SEARCH_API, {
       query_s: ko.mapping.toJSON(options.query),
       field_facets: ko.mapping.toJSON(options.facets || []),
       limit: 50,
@@ -2646,7 +2571,7 @@ class ApiHelper {
     const deferred = $.Deferred();
 
     const request = simplePost(
-      SEARCH_API,
+      URLS.SEARCH_API,
       {
         query_s: ko.mapping.toJSON(options.query),
         limit: options.limit || 100,
@@ -2673,7 +2598,7 @@ class ApiHelper {
     const deferred = $.Deferred();
 
     const request = simplePost(
-      FORMAT_SQL_API,
+      URLS.FORMAT_SQL_API,
       {
         statements: options.statements
       },
@@ -2700,7 +2625,7 @@ class ApiHelper {
     const deferred = $.Deferred();
 
     const request = simplePost(
-      GIST_API + 'create',
+      URLS.GIST_API + 'create',
       {
         statement: options.statement,
         doc_type: options.doc_type,

+ 64 - 0
desktop/core/src/desktop/js/api/urls.js

@@ -0,0 +1,64 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export const AUTOCOMPLETE_API_PREFIX = '/notebook/api/autocomplete/';
+export const SAMPLE_API_PREFIX = '/notebook/api/sample/';
+export const EXECUTE_API_PREFIX = '/notebook/api/execute/';
+export const DOCUMENTS_API = '/desktop/api2/doc/';
+export const DOCUMENTS_SEARCH_API = '/desktop/api2/docs/';
+export const GET_HUE_CONFIG_API = '/desktop/api2/get_hue_config';
+export const FETCH_CONFIG_API = '/desktop/api2/get_config/';
+export const HDFS_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('/');
+export const ADLS_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('adl:/');
+export const ABFS_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('ABFS://');
+export const GIT_API_PREFIX = '/desktop/api/vcs/contents/';
+export const S3_API_PREFIX = '/filebrowser/view=' + encodeURIComponent('S3A://');
+export const IMPALA_INVALIDATE_API = '/impala/api/invalidate';
+export const CONFIG_SAVE_API = '/desktop/api/configurations/save/';
+export const CONFIG_APPS_API = '/desktop/api/configurations';
+export const SOLR_COLLECTIONS_API = '/indexer/api/indexes/list/';
+export const SOLR_FIELDS_API = '/indexer/api/index/list/';
+export const DASHBOARD_TERMS_API = '/dashboard/get_terms';
+export const DASHBOARD_STATS_API = '/dashboard/get_stats';
+export const FORMAT_SQL_API = '/notebook/api/format';
+export const GIST_API = '/desktop/api2/gist/';
+export const TOPO_URL = '/desktop/topo/';
+
+export const SEARCH_API = '/desktop/api/search/entities';
+export const INTERACTIVE_SEARCH_API = '/desktop/api/search/entities_interactive';
+
+export const HBASE_API_PREFIX = '/hbase/api/';
+export const SAVE_TO_FILE_API = '/filebrowser/save';
+
+export const NAV_API = {
+  ADD_TAGS: '/metadata/api/catalog/add_tags',
+  DELETE_TAGS: '/metadata/api/catalog/delete_tags',
+  FIND_ENTITY: '/metadata/api/catalog/find_entity',
+  LIST_TAGS: '/metadata/api/catalog/list_tags',
+  UPDATE_PROPERTIES: '/metadata/api/catalog/update_properties'
+};
+
+export const OPTIMIZER_API = {
+  COMPATIBILITY: '/notebook/api/optimizer/statement/compatibility',
+  RISK: '/notebook/api/optimizer/statement/risk',
+  SIMILARITY: '/notebook/api/optimizer/statement/similarity',
+  TOP_AGGS: '/metadata/api/optimizer/top_aggs',
+  TOP_COLUMNS: '/metadata/api/optimizer/top_columns',
+  TOP_FILTERS: '/metadata/api/optimizer/top_filters',
+  TOP_JOINS: '/metadata/api/optimizer/top_joins',
+  TOP_TABLES: '/metadata/api/optimizer/top_tables',
+  TABLE_DETAILS: '/metadata/api/optimizer/table_details'
+};

+ 11 - 22
desktop/core/src/desktop/js/catalog/optimizer/apiStrategy.js

@@ -19,20 +19,9 @@ import * as ko from 'knockout';
 
 import CancellablePromise from 'api/cancellablePromise';
 import { simplePost } from 'api/apiUtils';
+import { OPTIMIZER_API } from 'api/urls';
 import BaseStrategy from './baseStrategy';
 
-const OPTIMIZER_URLS = {
-  COMPATIBILITY: '/notebook/api/optimizer/statement/compatibility',
-  RISK: '/notebook/api/optimizer/statement/risk',
-  SIMILARITY: '/notebook/api/optimizer/statement/similarity',
-  TOP_AGGS: '/metadata/api/optimizer/top_aggs',
-  TOP_COLUMNS: '/metadata/api/optimizer/top_columns',
-  TOP_FILTERS: '/metadata/api/optimizer/top_filters',
-  TOP_JOINS: '/metadata/api/optimizer/top_joins',
-  TOP_TABLES: '/metadata/api/optimizer/top_tables',
-  TABLE_DETAILS: '/metadata/api/optimizer/table_details'
-};
-
 /**
  * Fetches the popularity for various aspects of the given tables
  *
@@ -65,7 +54,7 @@ const genericOptimizerMultiTableFetch = (options, url) => {
 
 export default class ApiStrategy extends BaseStrategy {
   analyzeCompatibility(options) {
-    return simplePost(OPTIMIZER_URLS.COMPATIBILITY, {
+    return simplePost(OPTIMIZER_API.COMPATIBILITY, {
       notebook: options.notebookJson,
       snippet: options.snippetJson,
       sourcePlatform: options.sourcePlatform,
@@ -74,14 +63,14 @@ export default class ApiStrategy extends BaseStrategy {
   }
 
   analyzeRisk(options) {
-    return simplePost(OPTIMIZER_URLS.RISK, {
+    return simplePost(OPTIMIZER_API.RISK, {
       notebook: options.notebookJson,
       snippet: options.snippetJson
     });
   }
 
   analyzeSimilarity(options) {
-    return simplePost(OPTIMIZER_URLS.SIMILARITY, {
+    return simplePost(OPTIMIZER_API.SIMILARITY, {
       notebook: options.notebookJson,
       snippet: options.snippetJson,
       sourcePlatform: options.sourcePlatform
@@ -93,12 +82,12 @@ export default class ApiStrategy extends BaseStrategy {
     let url, data;
 
     if (options.paths.length === 1 && options.paths[0].length === 1) {
-      url = OPTIMIZER_URLS.TOP_TABLES;
+      url = OPTIMIZER_API.TOP_TABLES;
       data = {
         database: options.paths[0][0]
       };
     } else {
-      url = OPTIMIZER_URLS.TOP_COLUMNS;
+      url = OPTIMIZER_API.TOP_COLUMNS;
       const dbTables = [];
       options.paths.forEach(path => {
         dbTables.push(path.join('.'));
@@ -121,26 +110,26 @@ export default class ApiStrategy extends BaseStrategy {
   }
 
   fetchTopAggs(options) {
-    return genericOptimizerMultiTableFetch(options, OPTIMIZER_URLS.TOP_AGGS);
+    return genericOptimizerMultiTableFetch(options, OPTIMIZER_API.TOP_AGGS);
   }
 
   fetchTopColumns(options) {
-    return genericOptimizerMultiTableFetch(options, OPTIMIZER_URLS.TOP_COLUMNS);
+    return genericOptimizerMultiTableFetch(options, OPTIMIZER_API.TOP_COLUMNS);
   }
 
   fetchTopFilters(options) {
-    return genericOptimizerMultiTableFetch(options, OPTIMIZER_URLS.TOP_FILTERS);
+    return genericOptimizerMultiTableFetch(options, OPTIMIZER_API.TOP_FILTERS);
   }
 
   fetchTopJoins(options) {
-    return genericOptimizerMultiTableFetch(options, OPTIMIZER_URLS.TOP_JOINS);
+    return genericOptimizerMultiTableFetch(options, OPTIMIZER_API.TOP_JOINS);
   }
 
   fetchOptimizerMeta(options) {
     const deferred = $.Deferred();
 
     const request = simplePost(
-      OPTIMIZER_URLS.TABLE_DETAILS,
+      OPTIMIZER_API.TABLE_DETAILS,
       {
         databaseName: options.path[0],
         tableName: options.path[1]

+ 3 - 3
desktop/core/src/desktop/js/ko/components/assist/ko.assistFunctionsPanel.js

@@ -182,7 +182,7 @@ class ConnectorUdfCategories {
     if (this.initialized) {
       return;
     }
-    await this.fetchUdfs();
+    await this.getUdfs();
     this.initialized = true;
   }
 
@@ -190,11 +190,11 @@ class ConnectorUdfCategories {
     this.loading(true);
     huePubSub.publish(CLEAR_UDF_CACHE_EVENT, {
       connector: this.connector,
-      callback: this.fetchUdfs.bind(this)
+      callback: this.getUdfs.bind(this)
     });
   }
 
-  async fetchUdfs() {
+  async getUdfs() {
     this.loading(true);
     const categories = [];
     const functions = await getUdfCategories(this.connector);

+ 2 - 1
desktop/core/src/desktop/js/ko/components/assist/ko.assistFunctionsPanel.test.js

@@ -17,12 +17,13 @@ import $ from 'jquery';
 import * as ko from 'knockout';
 
 import ApiHelper from 'api/apiHelper';
+import * as apiUtils from 'sql/reference/apiUtils';
 import AssistFunctionsPanel from './ko.assistFunctionsPanel';
 import { refreshConfig } from 'utils/hueConfig';
 import { sleep } from 'utils/hueUtils';
 
 describe('ko.assistFunctionsPanel.js', () => {
-  jest.spyOn(ApiHelper, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+  jest.spyOn(apiUtils, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
 
   it('should handle cluster config updates', async () => {
     const spy = jest.spyOn(ApiHelper, 'getClusterConfig').mockImplementation(() =>

+ 2 - 1
desktop/core/src/desktop/js/sql/autocompleteResults.test.js

@@ -17,6 +17,7 @@
 import $ from 'jquery';
 
 import ApiHelper from 'api/apiHelper';
+import * as apiUtils from 'sql/reference/apiUtils';
 import AutocompleteResults from './autocompleteResults';
 import dataCatalog from 'catalog/dataCatalog';
 import huePubSub from 'utils/huePubSub';
@@ -229,7 +230,7 @@ describe('AutocompleteResults.js', () => {
     return deferred.promise();
   });
 
-  jest.spyOn(ApiHelper, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+  jest.spyOn(apiUtils, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
 
   const subject = new AutocompleteResults({
     snippet: {

+ 0 - 0
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepositoryCache.js → desktop/core/src/desktop/js/sql/reference/apiCache.js


+ 136 - 0
desktop/core/src/desktop/js/sql/reference/apiUtils.js

@@ -0,0 +1,136 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { simplePostAsync } from 'api/apiUtils';
+import { AUTOCOMPLETE_API_PREFIX } from 'api/urls';
+import I18n from 'utils/i18n';
+
+const FUNCTIONS_OPERATION = 'functions';
+const DEFAULT_DESCRIPTION = I18n('No description available.');
+const DEFAULT_RETURN_TYPE = ['T'];
+const DEFAULT_ARGUMENTS = [[{ type: 'T', multiple: true }]];
+
+const SIGNATURE_REGEX = /([a-z]+(?:\.{3})?)/gi;
+const TYPE_REGEX = /(?<type>[a-z]+)(?<multiple>\.{3})?/i;
+
+const stripPrecision = typeString => typeString.replace(/\(\*(,\*)?\)/g, '');
+
+// TODO: Extend with arguments etc reported by the API
+export const adaptApiUdf = apiUdf => {
+  const signature = apiUdf.name + '()';
+  return {
+    returnTypes: apiUdf.returnTypes || DEFAULT_RETURN_TYPE,
+    arguments: apiUdf.arguments || DEFAULT_ARGUMENTS,
+    signature: signature,
+    draggable: signature,
+    description: DEFAULT_DESCRIPTION
+  };
+};
+
+export const extractArgumentTypes = apiUdf => {
+  if (apiUdf.signature) {
+    const cleanSignature = stripPrecision(apiUdf.signature);
+    if (cleanSignature === '()') {
+      return [];
+    }
+    const match = cleanSignature.match(SIGNATURE_REGEX);
+    if (match) {
+      return match.map(argString => {
+        const typeMatch = argString.match(TYPE_REGEX);
+        const arg = { type: typeMatch.groups.type };
+        if (typeMatch.groups.multiple) {
+          arg.multiple = true;
+        }
+        return [arg];
+      });
+    }
+  }
+  return DEFAULT_ARGUMENTS;
+};
+
+export const mergeArgumentTypes = (target, additional) => {
+  for (let i = 0; i < target.length; i++) {
+    if (i >= additional.length) {
+      break;
+    }
+    if (target[i].type === 'T') {
+      continue;
+    }
+    if (additional[i].type === 'T') {
+      target[i] = additional[i];
+      continue;
+    }
+    target[i].push(...additional[i]);
+  }
+};
+
+export const adaptApiFunctions = functions => {
+  const udfs = [];
+  const adapted = {};
+  functions.forEach(apiUdf => {
+    apiUdf.arguments = extractArgumentTypes(apiUdf);
+    apiUdf.returnTypes = apiUdf.return_type ? [stripPrecision(apiUdf.return_type)] : ['T'];
+    if (adapted[apiUdf.name]) {
+      const adaptedUdf = adapted[apiUdf.name];
+      mergeArgumentTypes(adaptedUdf.arguments, apiUdf.arguments);
+      if (adaptedUdf.returnTypes[0] !== 'T') {
+        if (apiUdf.returnTypes[0] === 'T') {
+          adaptedUdf.returnTypes = ['T'];
+        } else if (adaptedUdf.returnTypes[0] !== apiUdf.returnTypes[0]) {
+          adaptedUdf.returnTypes.push(...apiUdf.returnTypes);
+        }
+      }
+    } else {
+      adapted[apiUdf.name] = apiUdf;
+      udfs.push(apiUdf);
+    }
+  });
+  return udfs;
+};
+
+/**
+ * @param {Object} options
+ * @param {Connector} options.connector
+ * @param {string} [options.database]
+ * @param {boolean} [options.silenceErrors]
+ *
+ * @return {Promise}
+ */
+export const fetchUdfs = async options => {
+  let url = AUTOCOMPLETE_API_PREFIX;
+  if (options.database) {
+    url += '/' + options.database;
+  }
+
+  const data = {
+    notebook: {},
+    snippet: JSON.stringify({
+      type: options.connector.id
+    }),
+    operation: FUNCTIONS_OPERATION
+  };
+
+  try {
+    const response = await simplePostAsync(url, data, options);
+
+    if (response && response.functions) {
+      return adaptApiFunctions(response.functions);
+    }
+    return (response && response.functions) || [];
+  } catch (err) {
+    return [];
+  }
+};

+ 162 - 0
desktop/core/src/desktop/js/sql/reference/apiUtils.test.js

@@ -0,0 +1,162 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { adaptApiFunctions, extractArgumentTypes, mergeArgumentTypes } from './apiUtils';
+
+describe('apiUtils.js', () => {
+  it('should return the default signature when not defined', () => {
+    const result = extractArgumentTypes({});
+    expect(JSON.stringify(result)).toEqual(JSON.stringify([[{ type: 'T', multiple: true }]]));
+  });
+
+  it('should extract empty argument types from empty signature', () => {
+    const result = extractArgumentTypes({ signature: '()' });
+    expect(result.length).toEqual(0);
+  });
+
+  it('should extract single argument type from signature', () => {
+    const result = extractArgumentTypes({ signature: '(INT)' });
+    expect(JSON.stringify(result)).toEqual(JSON.stringify([[{ type: 'INT' }]]));
+  });
+
+  it('should extract multiple argument types from signature', () => {
+    const result = extractArgumentTypes({ signature: '(INT, BIGINT, TINYINT)' });
+    expect(JSON.stringify(result)).toEqual(
+      JSON.stringify([[{ type: 'INT' }], [{ type: 'BIGINT' }], [{ type: 'TINYINT' }]])
+    );
+  });
+
+  it('should ignore precision in the signature', () => {
+    const result = extractArgumentTypes({ signature: '(DECIMAL(*,*))' });
+    expect(JSON.stringify(result)).toEqual(JSON.stringify([[{ type: 'DECIMAL' }]]));
+  });
+
+  it('should support repetitive argument types from signature', () => {
+    const result = extractArgumentTypes({ signature: '(INT, BIGINT...)' });
+    expect(JSON.stringify(result)).toEqual(
+      JSON.stringify([[{ type: 'INT' }], [{ type: 'BIGINT', multiple: true }]])
+    );
+  });
+
+  it('should support repetitive argument types with precision from signature', () => {
+    const result = extractArgumentTypes({ signature: '(INT, CHAR(*)...)' });
+    expect(JSON.stringify(result)).toEqual(
+      JSON.stringify([[{ type: 'INT' }], [{ type: 'CHAR', multiple: true }]])
+    );
+  });
+
+  it('should merge types', () => {
+    const target = [[{ type: 'INT' }]];
+    const additional = [[{ type: 'DOUBLE' }]];
+
+    mergeArgumentTypes(target, additional);
+
+    expect(JSON.stringify(target)).toEqual(JSON.stringify([[{ type: 'INT' }, { type: 'DOUBLE' }]]));
+  });
+
+  it('should add arguments where missing', () => {
+    const apiFunctions = [
+      {
+        name: 'cos'
+      },
+      {
+        name: 'sin'
+      }
+    ];
+
+    const result = adaptApiFunctions(apiFunctions);
+
+    expect(result.length).toEqual(2);
+    expect(result[0].arguments.length).toEqual(1);
+    expect(result[0].arguments[0].length).toEqual(1);
+    expect(result[0].arguments[0][0].type).toEqual('T');
+    expect(result[1].arguments.length).toEqual(1);
+    expect(result[1].arguments[0].length).toEqual(1);
+    expect(result[1].arguments[0][0].type).toEqual('T');
+  });
+
+  it('should add returnTypes where missing', () => {
+    const apiFunctions = [
+      {
+        name: 'cos'
+      },
+      {
+        name: 'sin'
+      }
+    ];
+
+    const result = adaptApiFunctions(apiFunctions);
+
+    expect(result.length).toEqual(2);
+    expect(result[0].returnTypes.length).toEqual(1);
+    expect(result[0].returnTypes[0]).toEqual('T');
+    expect(result[1].returnTypes.length).toEqual(1);
+    expect(result[1].returnTypes[0]).toEqual('T');
+  });
+
+  it('should merge same udf with multiple argument types', () => {
+    const apiFunctions = [
+      {
+        is_builtin: 'BUILTIN',
+        is_persistent: 'true',
+        name: 'casttochar',
+        return_type: 'CHAR(*)',
+        signature: '(CHAR(*))'
+      },
+      {
+        is_builtin: 'BUILTIN',
+        is_persistent: 'true',
+        name: 'casttochar',
+        return_type: 'CHAR(*)',
+        signature: '(INT)'
+      }
+    ];
+
+    const result = adaptApiFunctions(apiFunctions);
+
+    expect(result.length).toEqual(1);
+    expect(result[0].arguments.length).toEqual(1);
+    expect(result[0].arguments[0].length).toEqual(2);
+    expect(result[0].arguments[0][0].type).toEqual('CHAR');
+    expect(result[0].arguments[0][1].type).toEqual('INT');
+  });
+
+  it('should merge same udf with multiple return types', () => {
+    const apiFunctions = [
+      {
+        is_builtin: 'BUILTIN',
+        is_persistent: 'true',
+        name: 'casttochar',
+        return_type: 'CHAR(*)',
+        signature: '(CHAR(*))'
+      },
+      {
+        is_builtin: 'BUILTIN',
+        is_persistent: 'true',
+        name: 'casttochar',
+        return_type: 'VARCHAR(*)',
+        signature: '(INT)'
+      }
+    ];
+
+    const result = adaptApiFunctions(apiFunctions);
+
+    expect(result.length).toEqual(1);
+    expect(result[0].returnTypes.length).toEqual(2);
+    expect(result[0].returnTypes[0]).toEqual('CHAR');
+    expect(result[0].returnTypes[1]).toEqual('VARCHAR');
+  });
+});

+ 3 - 18
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.js

@@ -14,11 +14,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import ApiHelper from 'api/apiHelper';
 import { matchesType } from './typeUtils';
 import I18n from 'utils/i18n';
 import huePubSub from 'utils/huePubSub';
-import { clearUdfCache, getCachedApiUdfs, setCachedApiUdfs } from './sqlReferenceRepositoryCache';
+import { clearUdfCache, getCachedApiUdfs, setCachedApiUdfs } from './apiCache';
+import { adaptApiUdf, fetchUdfs } from './apiUtils';
 
 export const CLEAR_UDF_CACHE_EVENT = 'hue.clear.udf.cache';
 
@@ -33,9 +33,6 @@ const UDF_REFS = {
   pig: async () => import(/* webpackChunkName: "pig-ref" */ './pig/udfReference')
 };
 
-const DEFAULT_DESCRIPTION = I18n('No description available.');
-const DEFAULT_RETURN_TYPE = ['T'];
-const DEFAULT_ARGUMENTS = [[{ type: 'T', multiple: true }]];
 const IGNORED_UDF_REGEX = /^[!=$%&*+-/<>^|~]+$/;
 
 const mergedUdfPromises = {};
@@ -50,18 +47,6 @@ const getMergedUdfKey = (connector, database) => {
 
 export const hasUdfCategories = connector => typeof UDF_REFS[connector.dialect] !== 'undefined';
 
-// TODO: Extend with arguments etc reported by the API
-const adaptApiUdf = apiUdf => {
-  const signature = apiUdf.name + '()';
-  return {
-    returnTypes: DEFAULT_RETURN_TYPE,
-    arguments: DEFAULT_ARGUMENTS,
-    signature: signature,
-    draggable: signature,
-    description: DEFAULT_DESCRIPTION
-  };
-};
-
 const findUdfsToAdd = (apiUdfs, existingCategories) => {
   const existingUdfNames = new Set();
   existingCategories.forEach(category => {
@@ -89,7 +74,7 @@ const findUdfsToAdd = (apiUdfs, existingCategories) => {
 const mergeWithApiUdfs = async (categories, connector, database) => {
   let apiUdfs = await getCachedApiUdfs(connector, database);
   if (!apiUdfs) {
-    apiUdfs = await ApiHelper.fetchUdfs({
+    apiUdfs = await fetchUdfs({
       connector: connector,
       database: database,
       silenceErrors: true

+ 2 - 2
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.test.js

@@ -15,7 +15,7 @@
 // limitations under the License.
 
 import { getArgumentTypesForUdf } from './sqlReferenceRepository';
-import ApiHelper from 'api/apiHelper';
+import * as apiUtils from 'sql/reference/apiUtils';
 
 describe('sqlReferenceRepository.js', () => {
   const hiveConn = { dialect: 'hive', id: 'hive' };
@@ -85,7 +85,7 @@ describe('sqlReferenceRepository.js', () => {
     ]
   }));
 
-  jest.spyOn(ApiHelper, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+  jest.spyOn(apiUtils, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
 
   it('should give the expected argument types at a specific position', async () => {
     expect(await getArgumentTypesForUdf(hiveConn, 'cos', 1)).toEqual(['DECIMAL', 'DOUBLE']);