Bläddra i källkod

HUE-7738 [editor] Add result from the autocomplete functions API to the udf repository

The additional functions will now show up in the "General" category in the assist UDF panel as well as the autocomplete results.
Johan Ahlen 5 år sedan
förälder
incheckning
ee4e5ff207

+ 31 - 0
desktop/core/src/desktop/js/api/apiHelper.js

@@ -22,6 +22,7 @@ import {
   cancelActiveRequest,
   simpleGet,
   simplePost,
+  simplePostAsync,
   successResponseIsError
 } from './apiUtils';
 import apiQueueManager from 'api/apiQueueManager';
@@ -1369,6 +1370,36 @@ class ApiHelper {
     return deferred.resolve().promise();
   }
 
+  /**
+   * @param {Object} options
+   * @param {Connector} options.connector
+   * @param {string} [options.database]
+   * @param {boolean} [options.silenceErrors]
+   *
+   * @return {Promise}
+   */
+  async fetchUdfs(options) {
+    let url = AUTOCOMPLETE_API_PREFIX;
+    if (options.database) {
+      url += '/' + options.database;
+    }
+
+    const data = {
+      notebook: {},
+      snippet: JSON.stringify({
+        type: options.connector.id
+      }),
+      operation: 'functions'
+    };
+
+    try {
+      const response = await simplePostAsync(url, data, options);
+      return (response && response.functions) || [];
+    } catch (err) {
+      return [];
+    }
+  }
+
   /**
    * @param {Object} options
    * @param {string} options.sourceType

+ 19 - 1
desktop/core/src/desktop/js/api/apiUtils.js

@@ -148,13 +148,31 @@ export const simplePost = (url, data, options) => {
     return request.readyState;
   };
 
-  promise.abort = function() {
+  promise.abort = () => {
     request.abort();
   };
 
+  promise.cancel = promise.abort;
+
   return promise;
 };
 
+/**
+ * @param {string} url
+ * @param {Object} data
+ * @param {Object} [options]
+ * @param {boolean} [options.silenceErrors]
+ * @param {string} [options.dataType] - Default: Intelligent Guess (xml, json, script, text, html)
+ *
+ * @return {Promise}
+ */
+export const simplePostAsync = async (url, data, options) =>
+  new Promise((resolve, reject) => {
+    simplePost(url, data, options)
+      .done(resolve)
+      .fail(reject);
+  });
+
 export const cancelActiveRequest = request => {
   if (typeof request !== 'undefined' && request !== null) {
     const readyState = request.getReadyState ? request.getReadyState() : request.readyState;

+ 5 - 3
desktop/core/src/desktop/js/ko/components/assist/ko.assistFunctionsPanel.test.js

@@ -16,14 +16,16 @@
 import $ from 'jquery';
 import * as ko from 'knockout';
 
+import ApiHelper from 'api/apiHelper';
 import AssistFunctionsPanel from './ko.assistFunctionsPanel';
-import apiHelper from 'api/apiHelper';
 import { refreshConfig } from 'utils/hueConfig';
 import { sleep } from 'utils/hueUtils';
 
 describe('ko.assistFunctionsPanel.js', () => {
+  jest.spyOn(ApiHelper, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+
   it('should handle cluster config updates', async () => {
-    const spy = jest.spyOn(apiHelper, 'getClusterConfig').mockImplementation(() =>
+    const spy = jest.spyOn(ApiHelper, 'getClusterConfig').mockImplementation(() =>
       $.Deferred()
         .resolve({
           status: 0,
@@ -57,7 +59,7 @@ describe('ko.assistFunctionsPanel.js', () => {
 
     spy.mockRestore();
 
-    const changeSpy = jest.spyOn(apiHelper, 'getClusterConfig').mockImplementation(() =>
+    const changeSpy = jest.spyOn(ApiHelper, 'getClusterConfig').mockImplementation(() =>
       $.Deferred()
         .resolve({
           status: 0,

+ 2 - 0
desktop/core/src/desktop/js/sql/autocompleteResults.test.js

@@ -229,6 +229,8 @@ describe('AutocompleteResults.js', () => {
     return deferred.promise();
   });
 
+  jest.spyOn(ApiHelper, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+
   const subject = new AutocompleteResults({
     snippet: {
       autocompleteSettings: {

+ 94 - 14
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.js

@@ -14,7 +14,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import ApiHelper from 'api/apiHelper';
 import { matchesType } from './typeUtils';
+import I18n from 'utils/i18n';
 
 const SET_REFS = {
   impala: async () => import(/* webpackChunkName: "impala-ref" */ './impala/setReference')
@@ -27,27 +29,95 @@ const UDF_REFS = {
   pig: async () => import(/* webpackChunkName: "pig-ref" */ './pig/udfReference')
 };
 
-export const getSetOptions = async connector => {
-  if (SET_REFS[connector.dialect]) {
-    const module = await SET_REFS[connector.dialect]();
-    if (module.SET_OPTIONS) {
-      return module.SET_OPTIONS;
-    }
+const DEFAULT_DESCRIPTION = I18n('No description available.');
+const DEFAULT_RETURN_TYPE = ['T'];
+const DEFAULT_ARGUMENTS = [[{ type: 'T', multiple: true }]];
+const IGNORED_UDF_REGEX = /^[!=$%&*+-/<>^|~]+$/;
+
+const mergedUdfPromises = {};
+
+const getMergedUdfKey = (connector, database) => {
+  let key = connector.id;
+  if (database) {
+    key += '_' + database;
   }
-  return {};
+  return key;
 };
 
 export const hasUdfCategories = connector => typeof UDF_REFS[connector.dialect] !== 'undefined';
 
-export const getUdfCategories = async connector => {
-  if (UDF_REFS[connector.dialect]) {
-    const module = await UDF_REFS[connector.dialect]();
-    if (module.UDF_CATEGORIES) {
-      return module.UDF_CATEGORIES;
+// TODO: Extend with arguments etc reported by the API
+const adaptApiUdf = apiUdf => {
+  const signature = apiUdf.name + '()';
+  return {
+    returnTypes: DEFAULT_RETURN_TYPE,
+    arguments: DEFAULT_ARGUMENTS,
+    signature: signature,
+    draggable: signature,
+    description: DEFAULT_DESCRIPTION
+  };
+};
+
+const findUdfsToAdd = (apiUdfs, existingCategories) => {
+  const existingUdfNames = new Set();
+  existingCategories.forEach(category => {
+    Object.keys(category.functions).forEach(udfName => {
+      existingUdfNames.add(udfName.toUpperCase());
+    });
+  });
+
+  const result = {};
+
+  apiUdfs.forEach(apiUdf => {
+    if (
+      !result[apiUdf.name] &&
+      !existingUdfNames.has(apiUdf.name.toUpperCase()) &&
+      !IGNORED_UDF_REGEX.test(apiUdf.name)
+    ) {
+      result[apiUdf.name] = adaptApiUdf(apiUdf);
+    }
+  });
+
+  return result;
+};
+
+const mergeWithApiUdfs = async (categories, connector, database) => {
+  const apiUdfs = await ApiHelper.fetchUdfs({
+    connector: connector,
+    database: database,
+    silenceErrors: true
+  });
+
+  if (apiUdfs.length) {
+    const additionalUdfs = findUdfsToAdd(apiUdfs, categories);
+    if (Object.keys(additionalUdfs).length) {
+      const generalCategory = {
+        name: I18n('General'),
+        functions: additionalUdfs
+      };
+      categories.unshift(generalCategory);
     }
   }
-  // TODO: Fetch from API and diff/merge
-  return [];
+};
+
+export const getUdfCategories = async (connector, database) => {
+  const promiseKey = getMergedUdfKey(connector, database);
+  if (!mergedUdfPromises[promiseKey]) {
+    mergedUdfPromises[promiseKey] = new Promise(async resolve => {
+      let categories = [];
+      if (UDF_REFS[connector.dialect]) {
+        const module = await UDF_REFS[connector.dialect]();
+        if (module.UDF_CATEGORIES) {
+          categories = module.UDF_CATEGORIES;
+        }
+      }
+      await mergeWithApiUdfs(categories, connector, database);
+
+      resolve(categories);
+    });
+  }
+
+  return await mergedUdfPromises[promiseKey];
 };
 
 export const findUdf = async (connector, functionName) => {
@@ -123,3 +193,13 @@ export const getArgumentTypesForUdf = async (connector, functionName, argumentPo
     })
     .sort();
 };
+
+export const getSetOptions = async connector => {
+  if (SET_REFS[connector.dialect]) {
+    const module = await SET_REFS[connector.dialect]();
+    if (module.SET_OPTIONS) {
+      return module.SET_OPTIONS;
+    }
+  }
+  return {};
+};

+ 5 - 2
desktop/core/src/desktop/js/sql/reference/sqlReferenceRepository.test.js

@@ -15,10 +15,11 @@
 // limitations under the License.
 
 import { getArgumentTypesForUdf } from './sqlReferenceRepository';
+import ApiHelper from 'api/apiHelper';
 
 describe('sqlReferenceRepository.js', () => {
-  const hiveConn = { dialect: 'hive' };
-  const impalaConn = { dialect: 'impala' };
+  const hiveConn = { dialect: 'hive', id: 'hive' };
+  const impalaConn = { dialect: 'impala', id: 'impala' };
 
   jest.mock('sql/reference/impala/udfReference', () => ({
     UDF_CATEGORIES: [
@@ -84,6 +85,8 @@ describe('sqlReferenceRepository.js', () => {
     ]
   }));
 
+  jest.spyOn(ApiHelper, 'fetchUdfs').mockImplementation(() => Promise.resolve([]));
+
   it('should give the expected argument types at a specific position', async () => {
     expect(await getArgumentTypesForUdf(hiveConn, 'cos', 1)).toEqual(['DECIMAL', 'DOUBLE']);
     expect(await getArgumentTypesForUdf(hiveConn, 'cos', 2)).toEqual([]);