Parcourir la source

[frontend] Improve worker registration to support worker usage in the web components

Johan Åhlén il y a 4 ans
Parent
commit
bbd2758d22
24 fichiers modifiés avec 499 ajouts et 355 suppressions
  1. 2 2
      desktop/core/src/desktop/js/apps/editor/app.js
  2. 1 1
      desktop/core/src/desktop/js/apps/editor/components/aceEditor/AceLocationHandler.ts
  3. 1 1
      desktop/core/src/desktop/js/apps/editor/components/presentationMode/PresentationModeKoBridge.vue
  4. 1 1
      desktop/core/src/desktop/js/apps/editor/components/variableSubstitution/VariableSubstitutionKoBridge.vue
  5. 7 7
      desktop/core/src/desktop/js/apps/notebook/app.js
  6. 1 1
      desktop/core/src/desktop/js/apps/notebook/snippet.js
  7. 0 2
      desktop/core/src/desktop/js/hue.js
  8. 1 1
      desktop/core/src/desktop/js/ko/bindings/ace/aceLocationHandler.js
  9. 7 7
      desktop/core/src/desktop/js/ko/components/simpleAceEditor/ko.simpleAceEditor.js
  10. 2 1
      desktop/core/src/desktop/js/parse/types.ts
  11. 0 119
      desktop/core/src/desktop/js/sql/sqlLocationWebWorker.js
  12. 0 70
      desktop/core/src/desktop/js/sql/sqlSyntaxWebWorker.js
  13. 0 127
      desktop/core/src/desktop/js/sql/sqlWorkerHandler.js
  14. 20 0
      desktop/core/src/desktop/js/sql/workers/events.ts
  15. 53 0
      desktop/core/src/desktop/js/sql/workers/hueWorkerHandler.ts
  16. 137 0
      desktop/core/src/desktop/js/sql/workers/registrationUtils.ts
  17. 34 0
      desktop/core/src/desktop/js/sql/workers/sqlLocationWebWorker.ts
  18. 34 0
      desktop/core/src/desktop/js/sql/workers/sqlSyntaxWebWorker.ts
  19. 186 0
      desktop/core/src/desktop/js/sql/workers/workerUtils.ts
  20. 5 4
      desktop/core/src/desktop/js/types/types.ts
  21. 0 4
      desktop/core/src/desktop/templates/ace_sql_location_worker.mako
  22. 0 4
      desktop/core/src/desktop/templates/ace_sql_syntax_worker.mako
  23. 5 1
      tsconfig.json
  24. 2 2
      webpack.config.workers.js

+ 2 - 2
desktop/core/src/desktop/js/apps/editor/app.js

@@ -22,7 +22,7 @@ import 'ext/bootstrap-datepicker.min';
 import 'ext/jquery.hotkeys';
 import 'jquery/plugins/jquery.hdfstree';
 
-import sqlWorkerHandler from 'sql/sqlWorkerHandler';
+import { registerHueWorkers } from 'sql/workers/hueWorkerHandler';
 import huePubSub from 'utils/huePubSub';
 import I18n from 'utils/i18n';
 import scrollbarWidth from 'utils/screen/scrollbarWidth';
@@ -319,7 +319,7 @@ huePubSub.subscribe('app.dom.loaded', app => {
     ko.applyBindings(viewModel, $(window.EDITOR_BINDABLE_ELEMENT)[0]);
     viewModel.init();
 
-    sqlWorkerHandler.registerWorkers();
+    registerHueWorkers();
 
     viewModel.selectedNotebook.subscribe(newVal => {
       huePubSub.publish('selected.notebook.changed', newVal);

+ 1 - 1
desktop/core/src/desktop/js/apps/editor/components/aceEditor/AceLocationHandler.ts

@@ -43,7 +43,7 @@ import {
   POST_FROM_SYNTAX_WORKER_EVENT,
   POST_TO_LOCATION_WORKER_EVENT,
   POST_TO_SYNTAX_WORKER_EVENT
-} from 'sql/sqlWorkerHandler';
+} from 'sql/workers/events';
 import { getFromLocalStorage } from 'utils/storageUtils';
 import { SqlReferenceProvider } from 'sql/reference/types';
 

+ 1 - 1
desktop/core/src/desktop/js/apps/editor/components/presentationMode/PresentationModeKoBridge.vue

@@ -34,7 +34,7 @@
 
   import { Variable } from 'apps/editor/components/variableSubstitution/types';
   import { IdentifierLocation } from 'parse/types';
-  import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/sqlWorkerHandler';
+  import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/workers/events';
   import { wrap } from 'vue/webComponentWrap';
 
   import PresentationMode from './PresentationMode.vue';

+ 1 - 1
desktop/core/src/desktop/js/apps/editor/components/variableSubstitution/VariableSubstitutionKoBridge.vue

@@ -32,7 +32,7 @@
   import VariableSubstitution from './VariableSubstitution.vue';
   import SubscriptionTracker from 'components/utils/SubscriptionTracker';
   import { IdentifierLocation } from 'parse/types';
-  import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/sqlWorkerHandler';
+  import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/workers/events';
   import { wrap } from 'vue/webComponentWrap';
 
   const VariableSubstitutionKoBridge = defineComponent({

+ 7 - 7
desktop/core/src/desktop/js/apps/notebook/app.js

@@ -22,17 +22,17 @@ import 'ext/bootstrap-datepicker.min';
 import 'ext/jquery.hotkeys';
 import 'jquery/plugins/jquery.hdfstree';
 
-import huePubSub from 'utils/huePubSub';
-import I18n from 'utils/i18n';
-import sqlWorkerHandler from 'sql/sqlWorkerHandler';
-import bootstrapRatios from 'utils/html/bootstrapRatios';
-import scrollbarWidth from 'utils/screen/scrollbarWidth';
-import waitForRendered from 'utils/timing/waitForRendered';
 import NotebookViewModel from './NotebookViewModel';
 import {
   ACTIVE_SNIPPET_CONNECTOR_CHANGED_EVENT,
   IGNORE_NEXT_UNLOAD_EVENT
 } from 'apps/editor/events';
+import { registerHueWorkers } from 'sql/workers/hueWorkerHandler';
+import huePubSub from 'utils/huePubSub';
+import I18n from 'utils/i18n';
+import bootstrapRatios from 'utils/html/bootstrapRatios';
+import scrollbarWidth from 'utils/screen/scrollbarWidth';
+import waitForRendered from 'utils/timing/waitForRendered';
 import { SHOW_LEFT_ASSIST_EVENT } from 'ko/components/assist/events';
 
 window.Clipboard = Clipboard;
@@ -558,7 +558,7 @@ huePubSub.subscribe('app.dom.loaded', app => {
     ko.applyBindings(viewModel, $(window.EDITOR_BINDABLE_ELEMENT)[0]);
     viewModel.init();
 
-    sqlWorkerHandler.registerWorkers();
+    registerHueWorkers();
 
     viewModel.selectedNotebook.subscribe(newVal => {
       huePubSub.publish('selected.notebook.changed', newVal);

+ 1 - 1
desktop/core/src/desktop/js/apps/notebook/snippet.js

@@ -37,7 +37,7 @@ import {
   ASSIST_GET_SOURCE_EVENT,
   ASSIST_SET_SOURCE_EVENT
 } from 'ko/components/assist/events';
-import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/sqlWorkerHandler';
+import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/workers/events';
 import {
   ACTIVE_STATEMENT_CHANGED_EVENT,
   CURSOR_POSITION_CHANGED_EVENT,

+ 0 - 2
desktop/core/src/desktop/js/hue.js

@@ -55,7 +55,6 @@ import I18n from 'utils/i18n';
 import MultiLineEllipsisHandler from 'utils/multiLineEllipsisHandler';
 
 import sqlUtils from 'sql/sqlUtils';
-import sqlWorkerHandler from 'sql/sqlWorkerHandler';
 
 import 'webComponents/HueIcons';
 import 'components/sidebar/HueSidebarWebComponent';
@@ -113,7 +112,6 @@ window.sprintf = sprintf;
 window.SqlAutocompleter = SqlAutocompleter;
 window.sqlStatementsParser = sqlStatementsParser;
 window.sqlUtils = sqlUtils;
-window.sqlWorkerHandler = sqlWorkerHandler;
 
 $(document).ready(async () => {
   await refreshConfig(); // Make sure we have config up front

+ 1 - 1
desktop/core/src/desktop/js/ko/bindings/ace/aceLocationHandler.js

@@ -27,7 +27,7 @@ import {
   POST_FROM_SYNTAX_WORKER_EVENT,
   POST_TO_LOCATION_WORKER_EVENT,
   POST_TO_SYNTAX_WORKER_EVENT
-} from 'sql/sqlWorkerHandler';
+} from 'sql/workers/events';
 import stringDistance from 'sql/stringDistance';
 import hueDebug from 'utils/hueDebug';
 import huePubSub from 'utils/huePubSub';

+ 7 - 7
desktop/core/src/desktop/js/ko/components/simpleAceEditor/ko.simpleAceEditor.js

@@ -18,16 +18,16 @@ import $ from 'jquery';
 import * as ko from 'knockout';
 import ace from 'ext/aceHelper';
 
-import AceLocationHandler from 'ko/bindings/ace/aceLocationHandler';
-import componentUtils from 'ko/components/componentUtils';
-import UUID from 'utils/string/UUID';
-import { hueLocalStorage } from 'utils/storageUtils';
 import SolrFormulaAutocompleter from './solrFormulaAutocompleter';
 import SolrQueryAutocompleter from './solrQueryAutocompleter';
+import { findEditorConnector } from 'config/hueConfig';
+import AceLocationHandler from 'ko/bindings/ace/aceLocationHandler';
+import componentUtils from 'ko/components/componentUtils';
+import { registerHueWorkers } from 'sql/workers/hueWorkerHandler';
 import SqlAutocompleter from 'sql/sqlAutocompleter';
-import sqlWorkerHandler from 'sql/sqlWorkerHandler';
 import AceGutterHandler from 'ko/bindings/ace/aceGutterHandler';
-import { findEditorConnector } from 'config/hueConfig';
+import UUID from 'utils/string/UUID';
+import { hueLocalStorage } from 'utils/storageUtils';
 
 export const NAME = 'hue-simple-ace-editor';
 export const MULTI_NAME = 'hue-simple-ace-editor-multi';
@@ -154,7 +154,7 @@ class SimpleAceEditor {
       };
 
       if (connector.is_sql && !params.disableWorkers) {
-        sqlWorkerHandler.registerWorkers();
+        registerHueWorkers();
         const aceLocationHandler = new AceLocationHandler({
           editor: editor,
           editorId: $element.attr('id'),

+ 2 - 1
desktop/core/src/desktop/js/parse/types.ts

@@ -61,7 +61,7 @@ export interface IdentifierLocation {
   value?: string;
   active?: boolean;
   tables?: ParsedTable[];
-  colRef: boolean;
+  colRef: boolean | { identifierChain: IdentifierChainEntry[]; tables: ParsedTable[] };
   argumentPosition?: number;
   identifierChain?: IdentifierChainEntry[];
   expression?: { types: string[]; text: string };
@@ -69,6 +69,7 @@ export interface IdentifierLocation {
   path?: string;
   qualified?: boolean;
   resolveCatalogEntry: (options?: {
+    cachedOnly?: boolean;
     cancellable?: boolean;
     temporaryOnly?: boolean;
   }) => CancellablePromise<DataCatalogEntry>;

+ 0 - 119
desktop/core/src/desktop/js/sql/sqlLocationWebWorker.js

@@ -1,119 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import 'core-js/stable';
-import 'regenerator-runtime/runtime';
-import sqlParserRepository from 'parse/sql/sqlParserRepository';
-
-const handleStatement = (statement, locations, autocompleteParser, active) => {
-  // Statement locations come in the message to the worker and are generally more accurate
-  locations.push(statement);
-  try {
-    const sqlParseResult = autocompleteParser.parseSql(statement.statement + ' ', '');
-    if (sqlParseResult.locations) {
-      sqlParseResult.locations.forEach(location => {
-        location.active = active;
-        // Skip statement locations from the sql parser
-        if (location.type !== 'statement') {
-          if (location.location.first_line === 1) {
-            location.location.first_column += statement.location.first_column;
-            location.location.last_column += statement.location.first_column;
-          }
-          location.location.first_line += statement.location.first_line - 1;
-          location.location.last_line += statement.location.first_line - 1;
-          locations.push(location);
-        }
-      });
-    }
-  } catch (error) {}
-};
-
-let throttle = -1;
-let baseUrlSet = false;
-
-const onMessage = msg => {
-  if (!baseUrlSet) {
-    __webpack_public_path__ = (msg.data.hueBaseUrl || '') + '/dynamic_bundle/workers/';
-    baseUrlSet = true;
-  }
-  if (msg.data.ping) {
-    postMessage({ ping: true });
-    return;
-  }
-  clearTimeout(throttle);
-  throttle = setTimeout(() => {
-    if (msg.data.statementDetails) {
-      sqlParserRepository.getAutocompleteParser(msg.data.connector.dialect).then(parser => {
-        let locations = [];
-        const activeStatementLocations = [];
-        msg.data.statementDetails.precedingStatements.forEach(statement => {
-          handleStatement(statement, locations, parser, false);
-        });
-        if (msg.data.statementDetails.activeStatement) {
-          handleStatement(
-            msg.data.statementDetails.activeStatement,
-            activeStatementLocations,
-            parser,
-            true
-          );
-          locations = locations.concat(activeStatementLocations);
-        }
-        msg.data.statementDetails.followingStatements.forEach(statement => {
-          handleStatement(statement, locations, parser, false);
-        });
-
-        // Add databases where missing in the table identifier chains
-        if (msg.data.defaultDatabase) {
-          locations.forEach(location => {
-            if (
-              location.identifierChain &&
-              location.identifierChain.length &&
-              location.identifierChain[0].name
-            ) {
-              if (location.tables) {
-                location.tables.forEach(table => {
-                  if (
-                    table.identifierChain &&
-                    table.identifierChain.length === 1 &&
-                    table.identifierChain[0].name
-                  ) {
-                    table.identifierChain.unshift({ name: msg.data.defaultDatabase });
-                  }
-                });
-              } else if (location.type === 'table' && location.identifierChain.length === 1) {
-                location.identifierChain.unshift({ name: msg.data.defaultDatabase });
-              }
-            }
-          });
-        }
-
-        postMessage({
-          id: msg.data.id,
-          connector: msg.data.connector,
-          namespace: msg.data.namespace,
-          compute: msg.data.compute,
-          editorChangeTime: msg.data.statementDetails.editorChangeTime,
-          locations: locations,
-          activeStatementLocations: activeStatementLocations,
-          totalStatementCount: msg.data.statementDetails.totalStatementCount,
-          activeStatementIndex: msg.data.statementDetails.activeStatementIndex
-        });
-      });
-    }
-  }, 400);
-};
-
-WorkerGlobalScope.onLocationMessage = onMessage;

+ 0 - 70
desktop/core/src/desktop/js/sql/sqlSyntaxWebWorker.js

@@ -1,70 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import 'core-js/stable';
-import 'regenerator-runtime/runtime';
-import sqlParserRepository from 'parse/sql/sqlParserRepository';
-
-/**
- * This function turns the relative nested location into an absolute location given the statement location.
- *
- * @param statementLocation
- * @param nestedLocation
- */
-const toAbsoluteLocation = (statementLocation, nestedLocation) => {
-  if (nestedLocation.first_line === 1) {
-    nestedLocation.first_column += statementLocation.first_column;
-  }
-  if (nestedLocation.last_line === 1) {
-    nestedLocation.last_column += statementLocation.first_column;
-  }
-  const lineAdjust = statementLocation.first_line - 1;
-  nestedLocation.first_line += lineAdjust;
-  nestedLocation.last_line += lineAdjust;
-};
-
-let throttle = -1;
-let baseUrlSet = false;
-
-const onMessage = msg => {
-  if (!baseUrlSet) {
-    __webpack_public_path__ = (msg.data.hueBaseUrl || '') + '/dynamic_bundle/workers/';
-    baseUrlSet = true;
-  }
-  if (msg.data.ping) {
-    postMessage({ ping: true });
-    return;
-  }
-  clearTimeout(throttle);
-  throttle = setTimeout(() => {
-    sqlParserRepository.getSyntaxParser(msg.data.connector.dialect).then(parser => {
-      const syntaxError = parser.parseSyntax(msg.data.beforeCursor, msg.data.afterCursor);
-
-      if (syntaxError) {
-        toAbsoluteLocation(msg.data.statementLocation, syntaxError.loc);
-      }
-      postMessage({
-        id: msg.data.id,
-        connector: msg.data.connector,
-        editorChangeTime: msg.data.editorChangeTime,
-        syntaxError: syntaxError,
-        statementLocation: msg.data.statementLocation
-      });
-    });
-  }, 400);
-};
-
-WorkerGlobalScope.onSyntaxMessage = onMessage;

+ 0 - 127
desktop/core/src/desktop/js/sql/sqlWorkerHandler.js

@@ -1,127 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import huePubSub from 'utils/huePubSub';
-import { resolveCatalogEntry } from 'sql/sqlUtils';
-import { applyCancellable } from '../catalog/catalogUtils';
-
-export const POST_TO_LOCATION_WORKER_EVENT = 'ace.sql.location.worker.post';
-export const POST_FROM_LOCATION_WORKER_EVENT = 'ace.sql.location.worker.message';
-export const POST_TO_SYNTAX_WORKER_EVENT = 'ace.sql.syntax.worker.post';
-export const POST_FROM_SYNTAX_WORKER_EVENT = 'ace.sql.syntax.worker.message';
-
-const attachEntryResolver = function (location, connector, namespace, compute) {
-  location.resolveCatalogEntry = function (options) {
-    if (!options) {
-      options = {};
-    }
-    if (location.resolvePathPromise && !location.resolvePathPromise.cancelled) {
-      applyCancellable(location.resolvePathPromise, options);
-      return location.resolvePathPromise;
-    }
-
-    if (!location.identifierChain && !location.colRef && !location.colRef.identifierChain) {
-      if (!location.resolvePathPromise) {
-        location.resolvePathPromise = Promise.reject();
-      }
-      return location.resolvePathPromise;
-    }
-
-    const promise = resolveCatalogEntry({
-      connector: connector,
-      namespace: namespace,
-      compute: compute,
-      temporaryOnly: options.temporaryOnly,
-      cancellable: options.cancellable,
-      cachedOnly: options.cachedOnly,
-      identifierChain: location.identifierChain || location.colRef.identifierChain,
-      tables: location.tables || (location.colRef && location.colRef.tables)
-    });
-
-    if (!options.cachedOnly) {
-      location.resolvePathPromise = promise;
-    }
-    return promise;
-  };
-};
-
-let registered = false;
-
-export default {
-  registerWorkers: function () {
-    if (!registered && window.Worker) {
-      // It can take a while before the worker is active
-      const whenWorkerIsReady = function (worker, message) {
-        message.hueBaseUrl = window.HUE_BASE_URL;
-        if (!worker.isReady) {
-          window.clearTimeout(worker.pingTimeout);
-          worker.postMessage({ ping: true, hueBaseUrl: message.hueBaseUrl });
-          worker.pingTimeout = window.setTimeout(() => {
-            whenWorkerIsReady(worker, message);
-          }, 500);
-        } else {
-          // To JSON and back as Vue creates proxy objects with methods which are not serializable
-          worker.postMessage(JSON.parse(JSON.stringify(message)));
-        }
-      };
-
-      // For syntax checking
-      const aceSqlSyntaxWorker = new Worker(
-        window.HUE_BASE_URL +
-          '/desktop/workers/aceSqlSyntaxWorker.js?v=' +
-          window.HUE_VERSION +
-          '.1'
-      );
-      aceSqlSyntaxWorker.onmessage = function (e) {
-        if (e.data.ping) {
-          aceSqlSyntaxWorker.isReady = true;
-        } else {
-          huePubSub.publish(POST_FROM_SYNTAX_WORKER_EVENT, e);
-        }
-      };
-
-      huePubSub.subscribe(POST_TO_SYNTAX_WORKER_EVENT, message => {
-        whenWorkerIsReady(aceSqlSyntaxWorker, message);
-      });
-
-      // For location marking
-      const aceSqlLocationWorker = new Worker(
-        window.HUE_BASE_URL +
-          '/desktop/workers/aceSqlLocationWorker.js?v=' +
-          window.HUE_VERSION +
-          '.1'
-      );
-      aceSqlLocationWorker.onmessage = function (e) {
-        if (e.data.ping) {
-          aceSqlLocationWorker.isReady = true;
-        } else {
-          if (e.data.locations) {
-            e.data.locations.forEach(location => {
-              attachEntryResolver(location, e.data.connector, e.data.namespace, e.data.compute);
-            });
-          }
-          huePubSub.publish(POST_FROM_LOCATION_WORKER_EVENT, e);
-        }
-      };
-
-      huePubSub.subscribe(POST_TO_LOCATION_WORKER_EVENT, message => {
-        whenWorkerIsReady(aceSqlLocationWorker, message);
-      });
-
-      registered = true;
-    }
-  }
-};

+ 20 - 0
desktop/core/src/desktop/js/sql/workers/events.ts

@@ -0,0 +1,20 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export const POST_TO_LOCATION_WORKER_EVENT = 'ace.sql.location.worker.post';
+export const POST_FROM_LOCATION_WORKER_EVENT = 'ace.sql.location.worker.message';
+export const POST_TO_SYNTAX_WORKER_EVENT = 'ace.sql.syntax.worker.post';
+export const POST_FROM_SYNTAX_WORKER_EVENT = 'ace.sql.syntax.worker.message';

+ 53 - 0
desktop/core/src/desktop/js/sql/workers/hueWorkerHandler.ts

@@ -0,0 +1,53 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  attachLocationWorkerEvents,
+  attachSyntaxWorkerEvents
+} from 'sql/workers/registrationUtils';
+import { hueWindow } from 'types/types';
+
+const registerSyntaxWorker = (): Worker | undefined =>
+  new Worker(
+    `${(<hueWindow>window).HUE_BASE_URL}/desktop/workers/aceSqlSyntaxWorker.js?v=${
+      (<hueWindow>window).HUE_VERSION
+    }.1`
+  );
+
+const registerLocationWorker = (): Worker | undefined =>
+  new Worker(
+    `${(<hueWindow>window).HUE_BASE_URL}/desktop/workers/aceSqlLocationWorker.js?v=${
+      (<hueWindow>window).HUE_VERSION
+    }.1`
+  );
+
+let registered = false;
+
+export const registerHueWorkers = (): void => {
+  if (!window.Worker || registered) {
+    return;
+  }
+  // It can take a while before the worker is active
+
+  // For syntax checking
+  const aceSqlSyntaxWorker = registerSyntaxWorker();
+  attachSyntaxWorkerEvents(aceSqlSyntaxWorker);
+
+  // For location marking
+  const aceSqlLocationWorker = registerLocationWorker();
+  attachLocationWorkerEvents(aceSqlLocationWorker);
+  registered = true;
+};

+ 137 - 0
desktop/core/src/desktop/js/sql/workers/registrationUtils.ts

@@ -0,0 +1,137 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  POST_FROM_LOCATION_WORKER_EVENT,
+  POST_FROM_SYNTAX_WORKER_EVENT,
+  POST_TO_LOCATION_WORKER_EVENT,
+  POST_TO_SYNTAX_WORKER_EVENT
+} from './events';
+import { resolveCatalogEntry } from '../sqlUtils';
+import { CancellablePromise } from 'api/cancellablePromise';
+import { applyCancellable } from 'catalog/catalogUtils';
+import DataCatalogEntry from 'catalog/DataCatalogEntry';
+import { Compute, Connector, Namespace } from 'config/types';
+import { IdentifierChainEntry, IdentifierLocation, ParsedTable } from 'parse/types';
+import { hueWindow } from 'types/types';
+import huePubSub from 'utils/huePubSub';
+
+const whenWorkerIsReady = (
+  worker: Worker & { isReady?: boolean; pingTimeout?: number },
+  message: unknown & { hueBaseUrl?: string }
+): void => {
+  if (window) {
+    message.hueBaseUrl = (<hueWindow>window).HUE_BASE_URL;
+    if (!worker.isReady) {
+      window.clearTimeout(worker.pingTimeout);
+      worker.postMessage({ ping: true, hueBaseUrl: message.hueBaseUrl });
+      worker.pingTimeout = window.setTimeout(() => {
+        whenWorkerIsReady(worker, message);
+      }, 500);
+    } else {
+      // To JSON and back as Vue creates proxy objects with methods which are not serializable
+      worker.postMessage(JSON.parse(JSON.stringify(message)));
+    }
+  }
+};
+
+const attachEntryResolver = (
+  location: IdentifierLocation & {
+    resolvePathPromise?: CancellablePromise<DataCatalogEntry>;
+  },
+  connector: Connector,
+  namespace: Namespace,
+  compute: Compute
+): void => {
+  location.resolveCatalogEntry = (options): CancellablePromise<DataCatalogEntry> => {
+    if (!options) {
+      options = {};
+    }
+    if (location.resolvePathPromise && !location.resolvePathPromise.cancelled) {
+      applyCancellable(location.resolvePathPromise, options);
+      return location.resolvePathPromise;
+    }
+
+    if (!location.identifierChain && !location.colRef) {
+      if (!location.resolvePathPromise) {
+        location.resolvePathPromise = CancellablePromise.reject();
+      }
+      return location.resolvePathPromise;
+    }
+
+    const promise = resolveCatalogEntry({
+      connector: connector,
+      namespace: namespace,
+      compute: compute,
+      temporaryOnly: options.temporaryOnly,
+      cancellable: options.cancellable,
+      cachedOnly: options.cachedOnly,
+      identifierChain:
+        location.identifierChain ||
+        (<{ identifierChain: IdentifierChainEntry[] }>location.colRef).identifierChain,
+      tables:
+        location.tables ||
+        (location.colRef && (<{ tables: ParsedTable[] }>location.colRef).tables) ||
+        undefined
+    });
+
+    if (!options.cachedOnly) {
+      location.resolvePathPromise = promise;
+    }
+    return promise;
+  };
+};
+
+export const attachSyntaxWorkerEvents = (syntaxWorker?: Worker & { isReady?: boolean }): void => {
+  if (!syntaxWorker) {
+    return;
+  }
+  syntaxWorker.onmessage = function (e) {
+    if (e.data.ping) {
+      syntaxWorker.isReady = true;
+    } else {
+      huePubSub.publish(POST_FROM_SYNTAX_WORKER_EVENT, e);
+    }
+  };
+
+  huePubSub.subscribe(POST_TO_SYNTAX_WORKER_EVENT, message => {
+    whenWorkerIsReady(syntaxWorker, message);
+  });
+};
+
+export const attachLocationWorkerEvents = (
+  locationWorker?: Worker & { isReady?: boolean }
+): void => {
+  if (!locationWorker) {
+    return;
+  }
+  locationWorker.onmessage = function (e) {
+    if (e.data.ping) {
+      locationWorker.isReady = true;
+    } else {
+      if (e.data.locations) {
+        (e.data.locations as IdentifierLocation[]).forEach(location => {
+          attachEntryResolver(location, e.data.connector, e.data.namespace, e.data.compute);
+        });
+      }
+      huePubSub.publish(POST_FROM_LOCATION_WORKER_EVENT, e);
+    }
+  };
+
+  huePubSub.subscribe(POST_TO_LOCATION_WORKER_EVENT, message => {
+    whenWorkerIsReady(locationWorker, message);
+  });
+};

+ 34 - 0
desktop/core/src/desktop/js/sql/workers/sqlLocationWebWorker.ts

@@ -0,0 +1,34 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'core-js/stable';
+import 'regenerator-runtime/runtime';
+
+import { attachLocationListeners } from './workerUtils';
+import sqlParserRepository from 'parse/sql/sqlParserRepository';
+
+declare let __webpack_public_path__: string;
+
+const ctx = self as DedicatedWorkerGlobalScope;
+let baseUrlSet = false;
+
+attachLocationListeners(ctx, sqlParserRepository, msg => {
+  if (!baseUrlSet) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    __webpack_public_path__ = (msg.data.hueBaseUrl || '') + '/dynamic_bundle/workers/';
+    baseUrlSet = true;
+  }
+});

+ 34 - 0
desktop/core/src/desktop/js/sql/workers/sqlSyntaxWebWorker.ts

@@ -0,0 +1,34 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'core-js/stable';
+import 'regenerator-runtime/runtime';
+
+import { attachSyntaxListener } from './workerUtils';
+import sqlParserRepository from 'parse/sql/sqlParserRepository';
+
+declare let __webpack_public_path__: string;
+
+const ctx = self as DedicatedWorkerGlobalScope;
+let baseUrlSet = false;
+
+attachSyntaxListener(ctx, sqlParserRepository, msg => {
+  if (!baseUrlSet) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    __webpack_public_path__ = (msg.data.hueBaseUrl || '') + '/dynamic_bundle/workers/';
+    baseUrlSet = true;
+  }
+});

+ 186 - 0
desktop/core/src/desktop/js/sql/workers/workerUtils.ts

@@ -0,0 +1,186 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { ParsedSqlStatement } from 'parse/sqlStatementsParser';
+import {
+  AutocompleteParser,
+  IdentifierLocation,
+  ParsedLocation,
+  SqlParserProvider
+} from 'parse/types';
+
+const handleStatement = (
+  statement: ParsedSqlStatement,
+  locations: IdentifierLocation[],
+  autocompleteParser: AutocompleteParser,
+  active: boolean
+) => {
+  // Statement locations come in the message to the worker and are generally more accurate
+  locations.push(statement as unknown as IdentifierLocation);
+  try {
+    const sqlParseResult = autocompleteParser.parseSql(statement.statement + ' ', '');
+    if (sqlParseResult.locations) {
+      sqlParseResult.locations.forEach(location => {
+        location.active = active;
+        // Skip statement locations from the sql parser
+        if (location.type !== 'statement') {
+          if (location.location.first_line === 1) {
+            location.location.first_column += statement.location.first_column;
+            location.location.last_column += statement.location.first_column;
+          }
+          location.location.first_line += statement.location.first_line - 1;
+          location.location.last_line += statement.location.first_line - 1;
+          locations.push(location);
+        }
+      });
+    }
+  } catch (error) {}
+};
+
+/**
+ * This function turns the relative nested location into an absolute location given the statement location.
+ */
+const toAbsoluteLocation = (statementLocation: ParsedLocation, nestedLocation: ParsedLocation) => {
+  if (nestedLocation.first_line === 1) {
+    nestedLocation.first_column += statementLocation.first_column;
+  }
+  if (nestedLocation.last_line === 1) {
+    nestedLocation.last_column += statementLocation.first_column;
+  }
+  const lineAdjust = statementLocation.first_line - 1;
+  nestedLocation.first_line += lineAdjust;
+  nestedLocation.last_line += lineAdjust;
+};
+
+export const attachSyntaxListener = (
+  ctx: DedicatedWorkerGlobalScope,
+  parserProvider: SqlParserProvider,
+  beforeMessage?: (message: MessageEvent) => void
+): void => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let throttle: any = -1;
+
+  ctx.addEventListener('message', msg => {
+    if (beforeMessage) {
+      beforeMessage(msg);
+    }
+    if (msg.data.ping) {
+      ctx.postMessage({ ping: true });
+      return;
+    }
+    clearTimeout(throttle);
+    throttle = setTimeout(() => {
+      parserProvider.getSyntaxParser(msg.data.connector.dialect).then(parser => {
+        const syntaxError = parser.parseSyntax(
+          msg.data.beforeCursor,
+          msg.data.afterCursor
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        ) as any;
+
+        if (syntaxError) {
+          toAbsoluteLocation(msg.data.statementLocation, syntaxError.loc);
+        }
+        ctx.postMessage({
+          id: msg.data.id,
+          connector: msg.data.connector,
+          editorChangeTime: msg.data.editorChangeTime,
+          syntaxError: syntaxError,
+          statementLocation: msg.data.statementLocation
+        });
+      });
+    }, 400);
+  });
+};
+
+export const attachLocationListeners = (
+  ctx: DedicatedWorkerGlobalScope,
+  parserProvider: SqlParserProvider,
+  beforeMessage?: (message: MessageEvent) => void
+): void => {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let throttle: any = -1;
+
+  ctx.addEventListener('message', msg => {
+    if (beforeMessage) {
+      beforeMessage(msg);
+    }
+    if (msg.data.ping) {
+      ctx.postMessage({ ping: true });
+      return;
+    }
+    clearTimeout(throttle);
+    throttle = setTimeout(() => {
+      if (msg.data.statementDetails) {
+        parserProvider.getAutocompleteParser(msg.data.connector.dialect).then(parser => {
+          let locations: IdentifierLocation[] = [];
+          const activeStatementLocations: IdentifierLocation[] = [];
+          msg.data.statementDetails.precedingStatements.forEach((statement: ParsedSqlStatement) => {
+            handleStatement(statement, locations, parser, false);
+          });
+          if (msg.data.statementDetails.activeStatement) {
+            handleStatement(
+              msg.data.statementDetails.activeStatement,
+              activeStatementLocations,
+              parser,
+              true
+            );
+            locations = locations.concat(activeStatementLocations);
+          }
+          msg.data.statementDetails.followingStatements.forEach((statement: ParsedSqlStatement) => {
+            handleStatement(statement, locations, parser, false);
+          });
+
+          // Add databases where missing in the table identifier chains
+          if (msg.data.defaultDatabase) {
+            locations.forEach(location => {
+              if (
+                location.identifierChain &&
+                location.identifierChain.length &&
+                location.identifierChain[0].name
+              ) {
+                if (location.tables) {
+                  location.tables.forEach(table => {
+                    if (
+                      table.identifierChain &&
+                      table.identifierChain.length === 1 &&
+                      table.identifierChain[0].name
+                    ) {
+                      table.identifierChain.unshift({ name: msg.data.defaultDatabase });
+                    }
+                  });
+                } else if (location.type === 'table' && location.identifierChain.length === 1) {
+                  location.identifierChain.unshift({ name: msg.data.defaultDatabase });
+                }
+              }
+            });
+          }
+
+          ctx.postMessage({
+            id: msg.data.id,
+            connector: msg.data.connector,
+            namespace: msg.data.namespace,
+            compute: msg.data.compute,
+            editorChangeTime: msg.data.statementDetails.editorChangeTime,
+            locations: locations,
+            activeStatementLocations: activeStatementLocations,
+            totalStatementCount: msg.data.statementDetails.totalStatementCount,
+            activeStatementIndex: msg.data.statementDetails.activeStatementIndex
+          });
+        });
+      }
+    }, 400);
+  });
+};

+ 5 - 4
desktop/core/src/desktop/js/types/types.ts

@@ -44,25 +44,26 @@ declare global {
 }
 
 export interface hueWindow {
+  AUTOCOMPLETE_TIMEOUT?: number;
   CACHEABLE_TTL?: { default?: number; sqlAnalyzer?: number };
   CLOSE_SESSIONS?: { [dialect: string]: boolean };
   CUSTOM_DASHBOARD_URL?: string;
+  DISABLE_LOCAL_STORAGE?: boolean;
   ENABLE_PREDICT?: boolean;
+  ENABLE_SQL_SYNTAX_CHECK?: boolean;
   HAS_CATALOG?: boolean;
   HAS_CONNECTORS?: boolean;
   HAS_SQL_ANALYZER?: boolean;
-  AUTOCOMPLETE_TIMEOUT?: number;
-  ENABLE_SQL_SYNTAX_CHECK?: boolean;
   HUE_BASE_URL?: string;
+  HUE_VERSION?: string;
   LOGGED_USERNAME?: string;
-  SQL_ANALYZER_MODE?: string;
   SHOW_ADD_MORE_EDITORS?: boolean;
+  SQL_ANALYZER_MODE?: string;
   USER_IS_ADMIN?: boolean;
   USER_IS_HUE_ADMIN?: boolean;
   USER_VIEW_EDIT_USER_ENABLED?: boolean;
   WEB_SOCKETS_ENABLED?: boolean;
   WS_CHANNEL?: string;
   hueDebug?: HueDebug;
-  DISABLE_LOCAL_STORAGE?: boolean;
   trackOnGA?(track: string): void;
 }

+ 0 - 4
desktop/core/src/desktop/templates/ace_sql_location_worker.mako

@@ -21,7 +21,3 @@
 % for js_file in utils.get_files('sqlLocationWebWorker', config='WORKERS'):
   importScripts('${ js_file.get('url') }');
 % endfor
-
-(function () {
-  this.onmessage = WorkerGlobalScope.onLocationMessage
-})();

+ 0 - 4
desktop/core/src/desktop/templates/ace_sql_syntax_worker.mako

@@ -21,7 +21,3 @@
 % for js_file in utils.get_files('sqlSyntaxWebWorker', config='WORKERS'):
   importScripts('${ js_file.get('url') }');
 % endfor
-
-(function () {
-  this.onmessage = WorkerGlobalScope.onSyntaxMessage
-})();

+ 5 - 1
tsconfig.json

@@ -14,7 +14,11 @@
     "esModuleInterop": true,
     "baseUrl": "desktop/core/src/desktop/js",
     "typeRoots": [ "desktop/core/src/desktop/js/types", "./node_modules/@types"],
-    "lib": ["es2019", "dom"]
+    "lib": [
+      "es2019",
+      "dom",
+      "webworker"
+    ]
   },
   "include": [
     "."

+ 2 - 2
webpack.config.workers.js

@@ -29,8 +29,8 @@ module.exports = {
   performance: shared.performance,
   resolve: shared.resolve,
   entry: {
-    sqlLocationWebWorker: ['./desktop/core/src/desktop/js/sql/sqlLocationWebWorker.js'],
-    sqlSyntaxWebWorker: ['./desktop/core/src/desktop/js/sql/sqlSyntaxWebWorker.js']
+    sqlLocationWebWorker: ['./desktop/core/src/desktop/js/sql/workers/sqlLocationWebWorker.ts'],
+    sqlSyntaxWebWorker: ['./desktop/core/src/desktop/js/sql/workers/sqlSyntaxWebWorker.ts']
   },
   optimization: {
     minimize: false,