Browse Source

HUE-8856 [autocomplete] Use dynamic import in location web worker

Johan Ahlen 6 years ago
parent
commit
15a5906c6b

+ 1 - 1
.eslintrc.js

@@ -10,7 +10,7 @@ const hueGlobals = [
   'USER_HOME_DIR', 'WorkerGlobalScope',
 
   // other misc
-  'ace', 'CodeMirror', 'impalaDagre', 'less', 'MediumEditor', 'moment', 'Role', 'trackOnGA',
+  'ace', 'CodeMirror', 'impalaDagre', 'less', 'MediumEditor', 'moment', 'Role', 'trackOnGA', '__webpack_public_path__',
 
   // jasmine
   'afterAll', 'afterEach', 'beforeAll', 'beforeEach', 'describe', 'expect', 'fail', 'fdescribe', 'fit', 'it', 'jasmine',

+ 1 - 0
desktop/core/src/desktop/js/hue.js

@@ -14,6 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import 'utils/publicPath';
 import '@babel/polyfill';
 import _ from 'lodash';
 import $ from 'jquery/jquery.common';

+ 17 - 7
desktop/core/src/desktop/js/parse/sql/sqlParserRepository.js

@@ -35,18 +35,28 @@ class SqlParserRepository {
     this.modulePromises = {};
   }
 
-  async getAutocompleter(sourceType) {
-    if (!this.modulePromises[sourceType + 'Autocomplete']) {
-      this.modulePromises[sourceType + 'Autocomplete'] = AUTOCOMPLETE_MODULES[sourceType]();
+  async getParser(sourceType, parserType) {
+    if (!this.modulePromises[sourceType + parserType]) {
+      const modules = parserType === 'Autocomplete' ? AUTOCOMPLETE_MODULES : SYNTAX_MODULES;
+      this.modulePromises[sourceType + parserType] = new Promise((resolve, reject) => {
+        if (modules[sourceType]) {
+          modules[sourceType]()
+            .then(module => resolve(module.default))
+            .catch(reject);
+        } else {
+          reject('No ' + parserType.toLowerCase() + ' parser found for "' + sourceType + '"');
+        }
+      });
     }
     return this.modulePromises[sourceType + 'Autocomplete'];
   }
 
+  async getAutocompleter(sourceType) {
+    return this.getParser(sourceType, 'Autocomplete');
+  }
+
   async getSyntaxParser(sourceType) {
-    if (!this.modulePromises[sourceType + 'Syntax']) {
-      this.modulePromises[sourceType + 'Syntax'] = AUTOCOMPLETE_MODULES[sourceType]();
-    }
-    return this.modulePromises[sourceType + 'Syntax'];
+    return this.getParser(sourceType, 'Syntax');
   }
 }
 

+ 97 - 2
desktop/core/src/desktop/js/sql/sqlLocationWebWorker.js

@@ -14,6 +14,101 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import sqlAutocompleteParser from 'parse/sqlAutocompleteParser';
+import 'utils/workerPublicPath';
+import '@babel/polyfill';
+import sqlParserRepository from 'parse/sql/sqlParserRepository';
 
-WorkerGlobalScope.sqlAutocompleteParser = sqlAutocompleteParser;
+const handleStatement = (statement, locations, autocompleteParser, active) => {
+  // Statement locations come in the message to the worker and are generally more accurate
+  locations.push(statement);
+  try {
+    const sqlParseResult = autocompleteParser.parse(statement.statement + ' ', '');
+    if (sqlParseResult.locations) {
+      sqlParseResult.locations.forEach(location => {
+        location.active = active;
+        // Skip statement locations from the sql parser
+        if (location.type !== 'statement') {
+          if (location.location.first_line === 1) {
+            location.location.first_column += statement.location.first_column;
+            location.location.last_column += statement.location.first_column;
+          }
+          location.location.first_line += statement.location.first_line - 1;
+          location.location.last_line += statement.location.first_line - 1;
+          locations.push(location);
+        }
+      });
+    }
+  } catch (error) {}
+};
+
+let throttle = -1;
+
+const onMessage = msg => {
+  if (msg.data.ping) {
+    postMessage({ ping: true });
+    return;
+  }
+  clearTimeout(throttle);
+  throttle = setTimeout(() => {
+    if (msg.data.statementDetails) {
+      sqlParserRepository.getAutocompleter(msg.data.type).then(parser => {
+        let locations = [];
+        const activeStatementLocations = [];
+        msg.data.statementDetails.precedingStatements.forEach(statement => {
+          handleStatement(statement, locations, msg.data.type, false);
+        });
+        if (msg.data.statementDetails.activeStatement) {
+          handleStatement(
+            msg.data.statementDetails.activeStatement,
+            activeStatementLocations,
+            parser,
+            true
+          );
+          locations = locations.concat(activeStatementLocations);
+        }
+        msg.data.statementDetails.followingStatements.forEach(statement => {
+          handleStatement(statement, locations, msg.data.type, false);
+        });
+
+        // Add databases where missing in the table identifier chains
+        if (msg.data.defaultDatabase) {
+          locations.forEach(location => {
+            if (
+              location.identifierChain &&
+              location.identifierChain.length &&
+              location.identifierChain[0].name
+            ) {
+              if (location.tables) {
+                location.tables.forEach(table => {
+                  if (
+                    table.identifierChain &&
+                    table.identifierChain.length === 1 &&
+                    table.identifierChain[0].name
+                  ) {
+                    table.identifierChain.unshift({ name: msg.data.defaultDatabase });
+                  }
+                });
+              } else if (location.type === 'table' && location.identifierChain.length === 1) {
+                location.identifierChain.unshift({ name: msg.data.defaultDatabase });
+              }
+            }
+          });
+        }
+
+        postMessage({
+          id: msg.data.id,
+          sourceType: msg.data.type,
+          namespace: msg.data.namespace,
+          compute: msg.data.compute,
+          editorChangeTime: msg.data.statementDetails.editorChangeTime,
+          locations: locations,
+          activeStatementLocations: activeStatementLocations,
+          totalStatementCount: msg.data.statementDetails.totalStatementCount,
+          activeStatementIndex: msg.data.statementDetails.activeStatementIndex
+        });
+      });
+    }
+  }, 400);
+};
+
+WorkerGlobalScope.onLocationMessage = onMessage;

+ 0 - 0
desktop/core/src/desktop/js/publicPath.js → desktop/core/src/desktop/js/utils/publicPath.js


+ 3 - 0
desktop/core/src/desktop/js/utils/workerPublicPath.js

@@ -0,0 +1,3 @@
+__webpack_public_path__ = (WorkerGlobalScope.HUE_BASE_URL || '') + '/dynamic_bundle/';
+
+console.log(__webpack_public_path__);

+ 5 - 79
desktop/core/src/desktop/templates/ace_sql_location_worker.mako

@@ -18,87 +18,13 @@
   from webpack_loader import utils
 %>
 
+WorkerGlobalScope.KNOX_BASE_PATH_HUE = '/KNOX_BASE_PATH_HUE';
+WorkerGlobalScope.HUE_BASE_URL = WorkerGlobalScope.KNOX_BASE_PATH_HUE.indexOf('KNOX_BASE_PATH_HUE') < 0 ? WorkerGlobalScope.KNOX_BASE_PATH_HUE : '';
+
 % for js_file in utils.get_files('sqlLocationWebWorker', config='WORKERS'):
   importScripts('${ js_file.get('url') }');
 % endfor
 
 (function () {
-
-  this.throttle = -1;
-
-  this.handleStatement = function (statement, locations, type, active) {
-    // Statement locations come in the message to the worker and are generally more accurate
-    locations.push(statement);
-    try {
-      var sqlParseResult =  WorkerGlobalScope.sqlAutocompleteParser.parseSql(statement.statement + ' ', '', type, false);
-      if (sqlParseResult.locations) {
-        sqlParseResult.locations.forEach(function (location) {
-          location.active = active;
-          // Skip statement locations from the sql parser
-          if (location.type !== 'statement') {
-            if (location.location.first_line === 1) {
-              location.location.first_column += statement.location.first_column;
-              location.location.last_column += statement.location.first_column;
-            }
-            location.location.first_line += statement.location.first_line - 1;
-            location.location.last_line += statement.location.first_line - 1;
-            locations.push(location);
-          }
-        })
-      }
-    } catch (error) {}
-  };
-
-  this.onmessage = function (msg) {
-    if (msg.data.ping) {
-      postMessage({ ping: true });
-      return;
-    }
-    clearTimeout(this.throttle);
-    this.throttle = setTimeout(function () {
-      if (msg.data.statementDetails) {
-        var locations = [];
-        var activeStatementLocations = [];
-        msg.data.statementDetails.precedingStatements.forEach(function (statement) {
-          this.handleStatement(statement, locations, msg.data.type, false);
-        });
-        if (msg.data.statementDetails.activeStatement) {
-          this.handleStatement(msg.data.statementDetails.activeStatement, activeStatementLocations, msg.data.type, true);
-          locations = locations.concat(activeStatementLocations);
-        }
-        msg.data.statementDetails.followingStatements.forEach(function (statement) {
-          this.handleStatement(statement, locations, msg.data.type, false);
-        });
-
-        // Add databases where missing in the table identifier chains
-        if (msg.data.defaultDatabase) {
-          locations.forEach(function (location) {
-            if (location.identifierChain && location.identifierChain.length && location.identifierChain[0].name) {
-              if (location.tables) {
-                location.tables.forEach(function (table) {
-                  if (table.identifierChain && table.identifierChain.length === 1 && table.identifierChain[0].name) {
-                    table.identifierChain.unshift({ name: msg.data.defaultDatabase });
-                  }
-                });
-              } else if (location.type === 'table' && location.identifierChain.length === 1) {
-                location.identifierChain.unshift({ name: msg.data.defaultDatabase });
-              }
-            }
-          });
-        }
-
-        postMessage({
-          id: msg.data.id,
-          sourceType: msg.data.type,
-          namespace: msg.data.namespace,
-          compute: msg.data.compute,
-          editorChangeTime: msg.data.statementDetails.editorChangeTime,
-          locations: locations,
-          activeStatementLocations: activeStatementLocations,
-          totalStatementCount: msg.data.statementDetails.totalStatementCount,
-          activeStatementIndex: msg.data.statementDetails.activeStatementIndex
-        });
-      }
-    }, 400);
-  }
-})();
+  this.onmessage = WorkerGlobalScope.onLocationMessage
+})();

File diff suppressed because it is too large
+ 406 - 226
package-lock.json


+ 26 - 22
webpack.config.workers.js

@@ -3,54 +3,54 @@ const BundleTracker = require('webpack-bundle-tracker');
 const CleanWebpackPlugin = require('clean-webpack-plugin');
 const CleanObsoleteChunks = require('webpack-clean-obsolete-chunks');
 
-
-const path = require('path')
-const each = require('lodash/fp/each')
+const path = require('path');
+const each = require('lodash/fp/each');
 
 // https://github.com/ezhome/webpack-bundle-tracker/issues/25
 class RelativeBundleTracker extends BundleTracker {
-  convertPathChunks(chunks){
-    each(each(chunk => {
-      chunk.path = path.relative(this.options.path, chunk.path)
-    }))(chunks)
+  convertPathChunks(chunks) {
+    each(
+      each(chunk => {
+        chunk.path = path.relative(this.options.path, chunk.path);
+      })
+    )(chunks);
   }
   writeOutput(compiler, contents) {
-    if (contents.status === 'done')  {
-      this.convertPathChunks(contents.chunks)
+    if (contents.status === 'done') {
+      this.convertPathChunks(contents.chunks);
     }
 
-    super.writeOutput(compiler, contents)
+    super.writeOutput(compiler, contents);
   }
 }
-    
 
 module.exports = {
   devtool: 'source-map',
   mode: 'development',
+  target: 'webworker',
   performance: {
     maxEntrypointSize: 400 * 1024, // 400kb
     maxAssetSize: 400 * 1024 // 400kb
   },
   resolve: {
     extensions: ['.json', '.jsx', '.js'],
-    modules: [
-      'node_modules',
-      'js'
-    ],
+    modules: ['node_modules', 'js'],
     alias: {
-      'bootstrap': __dirname + '/node_modules/bootstrap-2.3.2/js'
+      bootstrap: __dirname + '/node_modules/bootstrap-2.3.2/js'
     }
   },
   entry: {
     sqlLocationWebWorker: ['./desktop/core/src/desktop/js/sql/sqlLocationWebWorker.js'],
-    sqlSyntaxWebWorker: ['./desktop/core/src/desktop/js/sql/sqlSyntaxWebWorker.js'],
+    sqlSyntaxWebWorker: ['./desktop/core/src/desktop/js/sql/sqlSyntaxWebWorker.js']
   },
   optimization: {
     minimize: true
   },
   output: {
-    path:  __dirname + '/desktop/core/src/desktop/static/desktop/js/bundles/workers',
-    filename: '[name]-bundle-[hash].js'
+    path: __dirname + '/desktop/core/src/desktop/static/desktop/js/bundles/workers',
+    filename: '[name]-bundle-[hash].js',
+    chunkFilename: '[name]-chunk-[hash].js',
+    globalObject: 'this'
   },
   module: {
     rules: [
@@ -69,8 +69,12 @@ module.exports = {
 
   plugins: [
     new CleanObsoleteChunks(),
-    new CleanWebpackPlugin([__dirname + '/desktop/core/src/desktop/static/desktop/js/bundles/workers/']),
-    new RelativeBundleTracker({path: '.', filename: "webpack-stats-workers.json"}),
-    new webpack.BannerPlugin('\nLicensed to Cloudera, Inc. under one\nor more contributor license agreements.  See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership.  Cloudera, Inc. licenses this file\nto you under the Apache License, Version 2.0 (the\n"License"); you may not use this file except in compliance\nwith the License.  You may obtain a copy of the License at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n')
+    new CleanWebpackPlugin([
+      __dirname + '/desktop/core/src/desktop/static/desktop/js/bundles/workers/'
+    ]),
+    new RelativeBundleTracker({ path: '.', filename: 'webpack-stats-workers.json' }),
+    new webpack.BannerPlugin(
+      '\nLicensed to Cloudera, Inc. under one\nor more contributor license agreements.  See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership.  Cloudera, Inc. licenses this file\nto you under the Apache License, Version 2.0 (the\n"License"); you may not use this file except in compliance\nwith the License.  You may obtain a copy of the License at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n'
+    )
   ]
 };

Some files were not shown because too many files changed in this diff