Prechádzať zdrojové kódy

HUE-8758 [connectors] Consolidate common connector logic into hueConfig

Johan Ahlen 5 rokov pred
rodič
commit
3547be28b5

+ 11 - 1
desktop/core/src/desktop/js/apps/notebook2/editorViewModel.js

@@ -28,7 +28,7 @@ import {
   ACTIVE_SNIPPET_DIALECT_CHANGED_EVENT,
   GET_ACTIVE_SNIPPET_DIALECT_EVENT
 } from 'apps/notebook2/events';
-import { CONFIG_REFRESHED_EVENT, GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
+import { CONFIG_REFRESHED_EVENT, GET_KNOWN_CONFIG_EVENT, findConnector } from 'utils/hueConfig';
 
 class EditorViewModel {
   constructor(editorId, notebooks, options, CoordinatorEditorViewModel, RunningCoordinatorModel) {
@@ -309,6 +309,16 @@ class EditorViewModel {
   }
 
   async newNotebook(editorType, callback, queryTab) {
+    const connector = await findConnector(connector => connector.type === editorType);
+    if (!connector) {
+      console.warn('No connector found for type ' + editorType);
+    } else {
+      huePubSub.publish(ACTIVE_SNIPPET_DIALECT_CHANGED_EVENT, {
+        dialect: connector.dialect,
+        isSqlDialect: connector.is_sql
+      });
+    }
+
     return new Promise((resolve, reject) => {
       $.post('/notebook/api/create_notebook', {
         type: editorType,

+ 8 - 12
desktop/core/src/desktop/js/apps/notebook2/snippet.js

@@ -43,7 +43,7 @@ import {
 } from 'ko/bindings/ace/aceLocationHandler';
 import { EXECUTE_ACTIVE_EXECUTABLE_EVENT } from 'apps/notebook2/components/ko.executableActions';
 import { UPDATE_HISTORY_EVENT } from 'apps/notebook2/components/ko.queryHistory';
-import { GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
+import { GET_KNOWN_CONFIG_EVENT, findConnector } from 'utils/hueConfig';
 import { cancelActiveRequest } from 'api/apiUtils';
 import { getOptimizer } from 'catalog/optimizer/optimizer';
 
@@ -1044,17 +1044,13 @@ export default class Snippet {
     huePubSub.publish(REFRESH_STATEMENT_LOCATIONS_EVENT, this);
   }
 
-  changeDialect(dialect) {
-    huePubSub.publish(GET_KNOWN_CONFIG_EVENT, config => {
-      if (config && config.app_config && config.app_config.editor) {
-        const foundConnector = config.app_config.editor.interpreters.find(
-          connector => connector.dialect === dialect
-        );
-        if (foundConnector) {
-          this.connector(foundConnector);
-        }
-      }
-    });
+  async changeDialect(dialect) {
+    const connector = await findConnector(connector => connector.dialect === dialect);
+    if (!connector) {
+      throw new Error('No connector found for dialect ' + dialect);
+    }
+    // TODO: Switch changeDialect to changeType
+    this.connector(connector);
   }
 
   updateFromExecutable(executable) {

+ 16 - 25
desktop/core/src/desktop/js/ko/components/assist/ko.assistDbPanel.js

@@ -23,7 +23,7 @@ import componentUtils from 'ko/components/componentUtils';
 import dataCatalog from 'catalog/dataCatalog';
 import huePubSub from 'utils/huePubSub';
 import I18n from 'utils/i18n';
-import { CONFIG_REFRESHED_EVENT, GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
+import { CONFIG_REFRESHED_EVENT, filterConnectors, GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
 
 const ASSIST_TABLE_TEMPLATES = `
   <script type="text/html" id="assist-no-database-entries">
@@ -851,31 +851,22 @@ class AssistDbPanel {
         return;
       }
 
-      const updateFromConfig = config => {
+      const updateFromConfig = async config => {
         const sources = [];
-        if (
-          config &&
-          config.app_config &&
-          config.app_config.editor &&
-          config.app_config.editor.interpreters
-        ) {
-          const interpreters = config.app_config.editor.interpreters;
-          interpreters.forEach(interpreter => {
-            if (interpreter.is_sql) {
-              const source =
-                this.sourceIndex[interpreter.type] ||
-                new AssistDbSource({
-                  i18n: this.i18n,
-                  type: interpreter.type,
-                  name: interpreter.name,
-                  connector: interpreter,
-                  nonSqlType: false,
-                  navigationSettings: navigationSettings
-                });
-              sources.push(source);
-            }
-          });
-        }
+        const connectors = await filterConnectors(connector => connector.is_sql);
+        connectors.forEach(connector => {
+          const source =
+            this.sourceIndex[connector.type] ||
+            new AssistDbSource({
+              i18n: this.i18n,
+              type: connector.type, // TODO: Remove redundant
+              name: connector.name, // TODO: Remove redundant
+              connector: connector,
+              nonSqlType: false,
+              navigationSettings: navigationSettings
+            });
+          sources.push(source);
+        });
         this.sourceIndex = {};
         sources.forEach(source => {
           this.sourceIndex[source.sourceType] = source;

+ 28 - 26
desktop/core/src/desktop/js/ko/components/assist/ko.assistFunctionsPanel.js

@@ -21,7 +21,7 @@ import componentUtils from 'ko/components/componentUtils';
 import huePubSub from 'utils/huePubSub';
 import { PigFunctions, SqlFunctions } from 'sql/sqlFunctions';
 import I18n from 'utils/i18n';
-import { GET_KNOWN_CONFIG_EVENT, CONFIG_REFRESHED_EVENT } from 'utils/hueConfig';
+import { CONFIG_REFRESHED_EVENT, filterConnectors } from 'utils/hueConfig';
 import {
   ACTIVE_SNIPPET_DIALECT_CHANGED_EVENT,
   GET_ACTIVE_SNIPPET_DIALECT_EVENT
@@ -179,41 +179,43 @@ class AssistFunctionsPanel {
       }
     );
 
-    const configUpdated = config => {
+    const configUpdated = async () => {
       const lastActiveDialect =
         this.activeDialect() ||
         apiHelper.getFromTotalStorage('assist', 'function.panel.active.dialect');
-      if (config.app_config && config.app_config.editor && config.app_config.editor.interpreters) {
-        const dialectIndex = {};
-        config.app_config.editor.interpreters.forEach(interpreter => {
-          if (
-            interpreter.dialect === 'hive' ||
-            interpreter.dialect === 'impala' ||
-            interpreter.dialect === 'pig'
-          ) {
-            dialectIndex[interpreter.dialect] = true;
-          }
-        });
-        this.availableDialects(Object.keys(dialectIndex).sort());
 
-        this.availableDialects().forEach(dialect => {
-          this.initFunctions(dialect);
-        });
+      const uniqueDialects = {};
 
-        if (lastActiveDialect && dialectIndex[lastActiveDialect]) {
-          this.activeDialect(lastActiveDialect);
-        } else {
-          this.activeDialect(
-            this.availableDialects().length ? this.availableDialects()[0] : undefined
-          );
-        }
+      const configuredDialects = (await filterConnectors(connector => {
+        const isMatch =
+          !uniqueDialects[connector.dialect] &&
+          (connector.dialect === 'hive' ||
+            connector.dialect === 'impala' ||
+            connector.dialect === 'pig');
+        uniqueDialects[connector.dialect] = true;
+        return isMatch;
+      })).map(connector => connector.dialect);
+      configuredDialects.sort();
+      this.availableDialects(configuredDialects);
+
+      this.availableDialects().forEach(dialect => {
+        this.initFunctions(dialect);
+      });
+
+      if (
+        lastActiveDialect &&
+        this.availableDialects().find(dialect => dialect === lastActiveDialect)
+      ) {
+        this.activeDialect(lastActiveDialect);
       } else {
-        this.availableDialects([]);
+        this.activeDialect(
+          this.availableDialects().length ? this.availableDialects()[0] : undefined
+        );
       }
       huePubSub.publish(GET_ACTIVE_SNIPPET_DIALECT_EVENT, updateDialect);
     };
 
-    huePubSub.publish(GET_KNOWN_CONFIG_EVENT, configUpdated);
+    configUpdated();
     const configSub = huePubSub.subscribe(CONFIG_REFRESHED_EVENT, configUpdated);
 
     this.disposals.push(() => {

+ 31 - 27
desktop/core/src/desktop/js/ko/components/assist/ko.assistFunctionsPanel.test.js

@@ -13,16 +13,19 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+import $ from 'jquery';
 
-import huePubSub from 'utils/huePubSub';
 import AssistFunctionsPanel from './ko.assistFunctionsPanel';
-import { CONFIG_REFRESHED_EVENT, GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
+import apiHelper from 'api/apiHelper';
+import { refreshConfig } from 'utils/hueConfig';
+import { sleep } from 'utils/hueUtils';
 
 describe('ko.assistFunctionsPanel.js', () => {
-  it('should handle cluster config updates', () => {
-    const spy = jest.spyOn(huePubSub, 'publish').mockImplementation((topic, cb) => {
-      if (topic === GET_KNOWN_CONFIG_EVENT && cb) {
-        cb({
+  it('should handle cluster config updates', async () => {
+    const spy = jest.spyOn(apiHelper, 'getClusterConfig').mockImplementation(() =>
+      $.Deferred()
+        .resolve({
+          status: 0,
           app_config: {
             editor: {
               interpreters: [
@@ -33,38 +36,39 @@ describe('ko.assistFunctionsPanel.js', () => {
               ]
             }
           }
-        });
-      }
-    });
+        })
+        .promise()
+    );
+    await refreshConfig();
     const subject = new AssistFunctionsPanel();
+    await sleep(0);
 
     expect(spy).toHaveBeenCalled();
     expect(subject.availableDialects()).toEqual(['impala', 'pig']);
 
     spy.mockRestore();
 
-    huePubSub.publish(CONFIG_REFRESHED_EVENT, {
-      app_config: {
-        editor: {
-          interpreters: [{ dialect: 'pig' }]
-        }
-      }
-    });
+    const changeSpy = jest.spyOn(apiHelper, 'getClusterConfig').mockImplementation(() =>
+      $.Deferred()
+        .resolve({
+          status: 0,
+          app_config: {
+            editor: {
+              interpreters: [{ dialect: 'pig' }]
+            }
+          }
+        })
+        .promise()
+    );
+    await refreshConfig();
+    expect(changeSpy).toHaveBeenCalled();
+    changeSpy.mockRestore();
+
+    await sleep(0);
 
     expect(subject.availableDialects()).toEqual(['pig']);
     expect(subject.activeDialect()).toEqual('pig');
 
-    huePubSub.publish(CONFIG_REFRESHED_EVENT, {
-      app_config: {
-        editor: {
-          interpreters: [{ dialect: 'banana' }]
-        }
-      }
-    });
-
-    expect(subject.availableDialects()).toEqual([]);
-    expect(subject.activeDialect()).toBeFalsy();
-
     subject.dispose();
   });
 });

+ 17 - 19
desktop/core/src/desktop/js/ko/components/assist/ko.assistLangRefPanel.js

@@ -20,7 +20,7 @@ import * as ko from 'knockout';
 import componentUtils from 'ko/components/componentUtils';
 import huePubSub from 'utils/huePubSub';
 import I18n from 'utils/i18n';
-import { GET_KNOWN_CONFIG_EVENT, CONFIG_REFRESHED_EVENT } from 'utils/hueConfig';
+import { GET_KNOWN_CONFIG_EVENT, CONFIG_REFRESHED_EVENT, filterConnectors } from 'utils/hueConfig';
 import { simpleGet } from 'api/apiUtils';
 import {
   ACTIVE_SNIPPET_DIALECT_CHANGED_EVENT,
@@ -156,26 +156,24 @@ class AssistLangRefPanel {
       }
     );
 
-    const configUpdated = config => {
+    const configUpdated = async config => {
       const lastActiveDialect = this.activeDialect();
-      if (config.app_config && config.app_config.editor && config.app_config.editor.interpreters) {
-        const dialectIndex = {};
-        config.app_config.editor.interpreters.forEach(interpreter => {
-          if (interpreter.dialect === 'hive' || interpreter.dialect === 'impala') {
-            dialectIndex[interpreter.dialect] = true;
-          }
-        });
-        this.availableDialects(Object.keys(dialectIndex).sort());
-
-        if (lastActiveDialect && dialectIndex[lastActiveDialect]) {
-          this.activeDialect(lastActiveDialect);
-        } else {
-          this.activeDialect(
-            this.availableDialects().length ? this.availableDialects()[0] : undefined
-          );
-        }
+
+      const configuredDialects = (await filterConnectors(
+        connector => connector.dialect === 'hive' || connector.dialect === 'impala'
+      )).map(connector => connector.dialect);
+      configuredDialects.sort();
+      this.availableDialects(configuredDialects);
+
+      if (
+        lastActiveDialect &&
+        this.availableDialects().find(dialect => dialect === lastActiveDialect)
+      ) {
+        this.activeDialect(lastActiveDialect);
       } else {
-        this.availableDialects([]);
+        this.activeDialect(
+          this.availableDialects().length ? this.availableDialects()[0] : undefined
+        );
       }
     };
 

+ 31 - 28
desktop/core/src/desktop/js/ko/components/assist/ko.assistLangRefPanel.test.js

@@ -14,9 +14,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import huePubSub from 'utils/huePubSub';
 import AssistLangRefPanel from './ko.assistLangRefPanel';
-import { CONFIG_REFRESHED_EVENT, GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
+import apiHelper from 'api/apiHelper';
+import $ from 'jquery';
+import { refreshConfig } from 'utils/hueConfig';
+import { sleep } from 'utils/hueUtils';
 
 describe('ko.assistLangRefPanel.js', () => {
   beforeAll(() => {
@@ -24,48 +26,49 @@ describe('ko.assistLangRefPanel.js', () => {
     window.HIVE_DOC_TOP_LEVEL = [];
   });
 
-  it('should handle cluster config updates', () => {
-    const spy = jest.spyOn(huePubSub, 'publish').mockImplementation((topic, cb) => {
-      if (topic === GET_KNOWN_CONFIG_EVENT && cb) {
-        cb({
+  it('should handle cluster config updates', async () => {
+    const spy = jest.spyOn(apiHelper, 'getClusterConfig').mockImplementation(() =>
+      $.Deferred()
+        .resolve({
+          status: 0,
           app_config: {
             editor: {
               interpreters: [{ dialect: 'hive' }, { dialect: 'impala' }, { dialect: 'banana' }]
             }
           }
-        });
-      }
-    });
-
+        })
+        .promise()
+    );
+    await refreshConfig();
     const subject = new AssistLangRefPanel();
+    await sleep(0);
 
     expect(spy).toHaveBeenCalled();
     expect(subject.availableDialects()).toEqual(['hive', 'impala']);
 
     spy.mockRestore();
 
-    huePubSub.publish(CONFIG_REFRESHED_EVENT, {
-      app_config: {
-        editor: {
-          interpreters: [{ dialect: 'impala' }]
-        }
-      }
-    });
+    const changeSpy = jest.spyOn(apiHelper, 'getClusterConfig').mockImplementation(() =>
+      $.Deferred()
+        .resolve({
+          status: 0,
+          app_config: {
+            editor: {
+              interpreters: [{ dialect: 'impala' }]
+            }
+          }
+        })
+        .promise()
+    );
+    await refreshConfig();
+    expect(changeSpy).toHaveBeenCalled();
+    changeSpy.mockRestore();
+
+    await sleep(0);
 
     expect(subject.availableDialects()).toEqual(['impala']);
     expect(subject.activeDialect()).toEqual('impala');
 
-    huePubSub.publish(CONFIG_REFRESHED_EVENT, {
-      app_config: {
-        editor: {
-          interpreters: [{ dialect: 'banana' }]
-        }
-      }
-    });
-
-    expect(subject.availableDialects()).toEqual([]);
-    expect(subject.activeDialect()).toBeFalsy();
-
     subject.dispose();
   });
 });

+ 4 - 14
desktop/core/src/desktop/js/ko/components/contextPopover/ko.quickQueryContext.js

@@ -27,7 +27,7 @@ import DisposableComponent from 'ko/components/DisposableComponent';
 import Executor from 'apps/notebook2/execution/executor';
 import SqlExecutable from 'apps/notebook2/execution/sqlExecutable';
 import sqlStatementsParser from 'parse/sqlStatementsParser';
-import { CONFIG_REFRESHED_EVENT, GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
+import { CONFIG_REFRESHED_EVENT, filterConnectors, GET_KNOWN_CONFIG_EVENT } from 'utils/hueConfig';
 import huePubSub from 'utils/huePubSub';
 
 export const NAME = 'quick-query-context';
@@ -166,19 +166,9 @@ class QuickQueryContext extends DisposableComponent {
     this.subscribe(this.database, refreshExecutable);
   }
 
-  updateFromConfig(config) {
-    if (
-      config &&
-      config.app_config &&
-      config.app_config.editor &&
-      config.app_config.editor.interpreters
-    ) {
-      this.availableInterpreters(
-        config.app_config.editor.interpreters.filter(interpreter => interpreter.is_sql)
-      );
-    } else {
-      this.availableInterpreters([]);
-    }
+  async updateFromConfig() {
+    const configuredSqlConnectors = await filterConnectors(connector => connector.is_sql);
+    this.availableInterpreters(configuredSqlConnectors);
 
     const found =
       this.interpreter() &&

+ 8 - 7
desktop/core/src/desktop/js/topNavViewModel.js

@@ -20,6 +20,7 @@ import apiHelper from 'api/apiHelper';
 import huePubSub from 'utils/huePubSub';
 import {
   CONFIG_REFRESHED_EVENT,
+  findConnector,
   GET_KNOWN_CONFIG_EVENT,
   REFRESH_CONFIG_EVENT
 } from 'utils/hueConfig';
@@ -37,19 +38,19 @@ class TopNavViewModel {
     self.hasJobBrowser = ko.observable(window.HAS_JOB_BROWSER);
     self.clusters = ko.observableArray();
 
-    const configUpdated = config => {
+    const configUpdated = async config => {
       if (config && config.clusters) {
         self.clusters(config.clusters);
       }
 
       self.hasJobBrowser(
         window.HAS_JOB_BROWSER &&
-          config &&
-          config.app_config &&
-          config.app_config.browser &&
-          (config.app_config.browser.interpreter_names.indexOf('yarn') !== -1 ||
-            config.app_config.editor.interpreter_names.indexOf('impala') !== -1 ||
-            config.app_config.browser.interpreter_names.indexOf('dataeng') !== -1)
+          (await findConnector(
+            connector =>
+              connector.dialect === 'yarn' ||
+              connector.dialect === 'impala' ||
+              connector.dialect === 'dataeng'
+          ))
       );
     };
 

+ 28 - 1
desktop/core/src/desktop/js/utils/hueConfig.js

@@ -24,7 +24,7 @@ export const GET_KNOWN_CONFIG_EVENT = 'cluster.config.get.config';
 
 let lastConfigPromise = undefined;
 
-const refreshConfig = () => {
+export const refreshConfig = async () => {
   lastConfigPromise = new Promise((resolve, reject) => {
     apiHelper
       .getClusterConfig()
@@ -46,6 +46,33 @@ const refreshConfig = () => {
     .catch(() => {
       huePubSub.publish(CONFIG_REFRESHED_EVENT);
     });
+
+  return lastConfigPromise;
+};
+
+const validateConfigForConnectors = config => {
+  if (
+    !config ||
+    !config.app_config ||
+    !config.app_config.editor ||
+    !config.app_config.editor.interpreters
+  ) {
+    throw new Error('No "interpreters" attribute present in the config.');
+  }
+};
+
+export const findConnector = async connectorTest => {
+  const config = await lastConfigPromise;
+  validateConfigForConnectors(config);
+  const connectors = config.app_config.editor.interpreters;
+  return connectors.find(connectorTest);
+};
+
+export const filterConnectors = async connectorTest => {
+  const config = await lastConfigPromise;
+  validateConfigForConnectors(config);
+  const connectors = config.app_config.editor.interpreters;
+  return connectors.filter(connectorTest);
 };
 
 huePubSub.subscribe(REFRESH_CONFIG_EVENT, refreshConfig);