Browse Source

HUE-9456 [editor] Implement variable substitution in editor V2

This moves all the variable logic from the snippet to a separate class and the substitution is now triggered on execute as opposed to on any change as before.
Johan Ahlen 5 years ago
parent
commit
259136abba

+ 2 - 1
desktop/core/src/desktop/js/api/cancellableJqPromise.ts

@@ -14,10 +14,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import { Cancellable } from 'api/cancellablePromise';
 import $ from 'jquery';
 import { cancelActiveRequest } from './apiUtils';
 
-export default class CancellableJqPromise<T> {
+export default class CancellableJqPromise<T> implements Cancellable {
   cancelCallbacks: (() => void)[] = [];
   deferred: JQuery.Deferred<T>;
   request?: JQuery.jqXHR;

+ 4 - 0
desktop/core/src/desktop/js/apps/notebook2/execution/executor.ts

@@ -16,6 +16,7 @@
 
 import Executable, { ExecutableRaw } from 'apps/notebook2/execution/executable';
 import { syncSqlExecutables } from 'apps/notebook2/execution/utils';
+import { VariableSubstitutionHandler } from 'apps/notebook2/variableSubstitution';
 import { StatementDetails } from 'parse/types';
 import { Compute, Connector, Namespace } from 'types/config';
 
@@ -32,6 +33,7 @@ export default class Executor {
   isSqlEngine?: boolean;
   isOptimizerEnabled?: boolean;
   executables: Executable[] = [];
+  variableSubstitionHandler?: VariableSubstitutionHandler;
 
   constructor(options: {
     connector: () => Connector;
@@ -42,6 +44,7 @@ export default class Executor {
     isSqlEngine?: boolean;
     isOptimizerEnabled?: boolean;
     executables: Executable[];
+    variableSubstitionHandler?: VariableSubstitutionHandler;
   }) {
     this.connector = options.connector;
     this.compute = options.compute;
@@ -51,6 +54,7 @@ export default class Executor {
     this.isOptimizerEnabled = options.isOptimizerEnabled;
     this.executables = [];
     this.defaultLimit = options.defaultLimit;
+    this.variableSubstitionHandler = options.variableSubstitionHandler;
   }
 
   toJs(): ExecutorRaw {

+ 4 - 0
desktop/core/src/desktop/js/apps/notebook2/execution/sqlExecutable.ts

@@ -62,6 +62,10 @@ export default class SqlExecutable extends Executable {
       }
     }
 
+    if (this.executor.variableSubstitionHandler) {
+      statement = this.executor.variableSubstitionHandler.substitute(statement);
+    }
+
     return statement;
   }
 

+ 14 - 350
desktop/core/src/desktop/js/apps/notebook2/snippet.js

@@ -55,6 +55,7 @@ import {
   ASSIST_SET_SOURCE_EVENT
 } from 'ko/components/assist/events';
 import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/sqlWorkerHandler';
+import { VariableSubstitutionHandler } from './variableSubstitution';
 
 // TODO: Remove for ENABLE_NOTEBOOK_2. Temporary here for debug
 window.SqlExecutable = SqlExecutable;
@@ -439,249 +440,20 @@ export default class Snippet {
         }
       }, 100);
     });
-    if (snippetRaw.variables) {
-      snippetRaw.variables.forEach(variable => {
-        variable.meta = (typeof variable.defaultValue === 'object' && variable.defaultValue) || {
-          type: 'text',
-          placeholder: ''
-        };
-        variable.value = variable.value || '';
-        variable.type = variable.type || 'text';
-        variable.sample = [];
-        variable.sampleUser = variable.sampleUser || [];
-        variable.path = variable.path || '';
-        variable.step = '';
-        delete variable.defaultValue;
-      });
-    }
-    this.variables = komapping.fromJS(snippetRaw.variables || []);
-    this.variables.subscribe(() => {
-      $(document).trigger('updateResultHeaders', this);
-    });
-    this.hasCurlyBracketParameters = ko.pureComputed(() => this.dialect() !== DIALECT.pig);
-
-    this.variableNames = ko.pureComputed(() => {
-      let match,
-        matches = {},
-        matchList;
-      if (this.dialect() === DIALECT.pig) {
-        matches = this.getPigParameters();
-      } else {
-        const re = /(?:^|\W)\${(\w*)=?([^{}]*)}/g;
-        const reComment = /(^\s*--.*)|(\/\*[\s\S]*?\*\/)/gm;
-        const reList = /(?!\s*$)\s*(?:(?:([^,|()\\]*)\(\s*([^,|()\\]*)\)(?:\\[\S\s][^,|()\\]*)?)|([^,|\\]*(?:\\[\S\s][^,|\\]*)*))\s*(?:,|\||$)/g;
-        const statement = this.statement_raw();
-        let matchComment = reComment.exec(statement);
-        // if re is n & reComment is m
-        // finding variables is O(n+m)
-        while ((match = re.exec(statement))) {
-          while (matchComment && match.index > matchComment.index + matchComment[0].length) {
-            // Comments before our match
-            matchComment = reComment.exec(statement);
-          }
-          const isWithinComment = matchComment && match.index >= matchComment.index;
-          if (isWithinComment) {
-            continue;
-          }
 
-          // If 1 match, text value
-          // If multiple matches, list value
-          const value = { type: 'text', placeholder: '' };
-          while ((matchList = reList.exec(match[2]))) {
-            const option = {
-              text: matchList[2] || matchList[3],
-              value: matchList[3] || matchList[1]
-            };
-            option.text = option.text && option.text.trim();
-            option.value =
-              option.value &&
-              option.value.trim().replace(',', ',').replace('(', '(').replace(')', ')');
-
-            if (value.placeholder || matchList[2]) {
-              if (!value.options) {
-                value.options = [];
-                value.type = 'select';
-              }
-              value.options.push(option);
-            }
-            if (!value.placeholder) {
-              value.placeholder = option.value;
-            }
-          }
-          const isPlaceholderInOptions =
-            !value.options || value.options.some(current => current.value === value.placeholder);
-          if (!isPlaceholderInOptions) {
-            value.options.unshift({ text: value.placeholder, value: value.placeholder });
-          }
-          matches[match[1]] = matches[match[1]] || value;
-        }
-      }
-      return $.map(matches, (match, key) => {
-        const isMatchObject = typeof matches[key] === 'object';
-        const meta = isMatchObject ? matches[key] : { type: 'text', placeholder: matches[key] };
-        return { name: key, meta: meta };
-      });
-    });
-    this.variableValues = {};
-    this.variableNames.extend({ rateLimit: 150 });
-    this.variableNames.subscribe(newVal => {
-      const variablesLength = this.variables().length;
-      const diffLengthVariables = variablesLength - newVal.length;
-      const needsMore = diffLengthVariables < 0;
-      const needsLess = diffLengthVariables > 0;
-      this.variableValues = this.variables().reduce((variableValues, variable) => {
-        if (!variableValues[variable.name()]) {
-          variableValues[variable.name()] = { sampleUser: [] };
-        }
-        variableValues[variable.name()].value = variable.value();
-        variableValues[variable.name()].sampleUser = variable.sampleUser();
-        variableValues[variable.name()].catalogEntry = variable.catalogEntry;
-        variableValues[variable.name()].path = variable.path();
-        variableValues[variable.name()].type = variable.type();
-        return variableValues;
-      }, this.variableValues);
-      if (needsMore) {
-        for (let i = 0, length = Math.abs(diffLengthVariables); i < length; i++) {
-          this.variables.push(
-            komapping.fromJS({
-              name: '',
-              value: '',
-              meta: { type: 'text', placeholder: '', options: [] },
-              sample: [],
-              sampleUser: [],
-              type: 'text',
-              step: '',
-              path: ''
-            })
-          );
-        }
-      } else if (needsLess) {
-        this.variables.splice(this.variables().length - diffLengthVariables, diffLengthVariables);
-      }
-      newVal.forEach((item, index) => {
-        const variable = this.variables()[index];
-        variable.name(item.name);
-        window.setTimeout(() => {
-          variable.value(
-            this.variableValues[item.name]
-              ? this.variableValues[item.name].value
-              : (!needsMore && variable.value()) || ''
-          );
-        }, 0);
-        variable.meta = komapping.fromJS(item.meta, {}, variable.meta);
-        variable.sample(
-          variable.meta.options
-            ? variable.meta.options().concat(variable.sampleUser())
-            : variable.sampleUser()
-        );
-        variable.sampleUser(
-          this.variableValues[item.name] ? this.variableValues[item.name].sampleUser : []
-        );
-        variable.type(
-          this.variableValues[item.name] ? this.variableValues[item.name].type || 'text' : 'text'
-        );
-        variable.path(
-          this.variableValues[item.name] ? this.variableValues[item.name].path || '' : ''
-        );
-        variable.catalogEntry =
-          this.variableValues[item.name] && this.variableValues[item.name].catalogEntry;
-      });
+    this.variableSubstitutionHandler = new VariableSubstitutionHandler(
+      this.connector,
+      this.statement_raw,
+      snippetRaw.variables
+    );
+
+    this.variableSubstitutionHandler.variables.subscribe(() => {
+      $(document).trigger('updateResultHeaders', this);
     });
 
-    const activeSourcePromises = [];
     huePubSub.subscribe(POST_FROM_LOCATION_WORKER_EVENT, e => {
-      while (activeSourcePromises.length) {
-        const promise = activeSourcePromises.pop();
-        if (promise.cancel) {
-          promise.cancel();
-        }
-      }
-      const oLocations = e.data.locations
-        .filter(location => {
-          return location.type === 'variable' && location.colRef;
-        })
-        .reduce((variables, location) => {
-          const re = /\${(\w*)=?([^{}]*)}/g;
-          const name = re.exec(location.value)[1];
-          variables[name] = location;
-          return variables;
-        }, {});
-      const updateVariableType = (variable, sourceMeta) => {
-        let type;
-        if (sourceMeta && sourceMeta.type) {
-          type = sourceMeta.type.toLowerCase();
-        } else {
-          type = 'string';
-        }
-        const variablesValues = {};
-        const value = variable.value();
-        switch (type) {
-          case 'timestamp':
-            variablesValues.type = 'datetime-local';
-            variablesValues.step = '1';
-            variablesValues.value =
-              (value && moment.utc(value).format('YYYY-MM-DD HH:mm:ss.S')) ||
-              moment(Date.now()).format('YYYY-MM-DD 00:00:00.0');
-            break;
-          case 'decimal':
-          case 'double':
-          case 'float':
-            variablesValues.type = 'number';
-            variablesValues.step = 'any';
-            break;
-          case 'int':
-          case 'smallint':
-          case 'tinyint':
-          case 'bigint':
-            variablesValues.type = 'number';
-            variablesValues.step = '1';
-            break;
-          case 'date':
-            variablesValues.type = 'date';
-            variablesValues.step = '';
-            variablesValues.value =
-              (value && moment.utc(value).format('YYYY-MM-DD')) ||
-              moment(Date.now()).format('YYYY-MM-DD');
-            break;
-          case 'boolean':
-            variablesValues.type = 'checkbox';
-            variablesValues.step = '';
-            break;
-          default:
-            variablesValues.type = 'text';
-            variablesValues.step = '';
-        }
-        if (variablesValues.value) {
-          setTimeout(() => {
-            variable.value(variablesValues.value);
-          }, 0);
-        }
-        variable.type(variablesValues.type);
-        variable.step(variablesValues.step);
-      };
-      this.variables().forEach(variable => {
-        if (oLocations[variable.name()]) {
-          activeSourcePromises.push(
-            oLocations[variable.name()].resolveCatalogEntry({ cancellable: true }).done(entry => {
-              variable.path(entry.path.join('.'));
-              variable.catalogEntry = entry;
-
-              activeSourcePromises.push(
-                entry
-                  .getSourceMeta({
-                    silenceErrors: true,
-                    cancellable: true
-                  })
-                  .then(updateVariableType.bind(this, variable))
-              );
-            })
-          );
-        } else {
-          updateVariableType(variable, {
-            type: 'text'
-          });
-        }
-      });
+      this.variableSubstitutionHandler.cancelRunningRequests();
+      this.variableSubstitutionHandler.updateFromLocations(e.data.locations);
     });
 
     this.statement = ko.pureComputed(() => {
@@ -696,41 +468,6 @@ export default class Snippet {
         }
       }
 
-      const variables = this.variables().reduce((variables, variable) => {
-        variables[variable.name()] = variable;
-        return variables;
-      }, {});
-      if (this.variables().length) {
-        const variablesString = this.variables()
-          .map(variable => {
-            return variable.name();
-          })
-          .join('|');
-        statement = statement.replace(
-          RegExp(
-            '([^\\\\])?\\$' +
-              (this.hasCurlyBracketParameters() ? '{(' : '(') +
-              variablesString +
-              ')(=[^}]*)?' +
-              (this.hasCurlyBracketParameters() ? '}' : ''),
-            'g'
-          ),
-          (match, p1, p2) => {
-            const variable = variables[p2];
-            const pad =
-              variable.type() === 'datetime-local' && variable.value().length === 16 ? ':00' : ''; // Chrome drops the seconds from the timestamp when it's at 0 second.
-            const value = variable.value();
-            const isValuePresent = //If value is string there is a need to check whether it is empty
-              typeof value === 'string' ? value : value !== undefined && value !== null;
-            return (
-              p1 +
-              (isValuePresent
-                ? value + pad
-                : variable.meta.placeholder && variable.meta.placeholder())
-            );
-          }
-        );
-      }
       return statement;
     });
 
@@ -871,7 +608,8 @@ export default class Snippet {
       defaultLimit: this.defaultLimit,
       isOptimizerEnabled: this.parentVm.isOptimizerEnabled(),
       snippet: this,
-      isSqlEngine: this.isSqlDialect
+      isSqlEngine: this.isSqlDialect,
+      variableSubstitionHandler: this.variableSubstitutionHandler
     });
 
     if (snippetRaw.executor) {
@@ -1151,67 +889,6 @@ export default class Snippet {
       });
   }
 
-  getPigParameters() {
-    const params = {};
-    const variables = this.statement_raw().match(/([^\\]|^)\$[^\d'"](\w*)/g);
-    const declares = this.statement_raw().match(/%declare +([^ ])+/gi);
-    const defaults = this.statement_raw().match(/%default +([^;])+/gi);
-    const macro_defines = this.statement_raw().match(/define [^ ]+ *\(([^)]*)\)/gi); // no multiline
-    const macro_returns = this.statement_raw().match(/returns +([^{]*)/gi); // no multiline
-
-    if (variables) {
-      variables.forEach(param => {
-        const p = param.substring(param.indexOf('$') + 1);
-        params[p] = '';
-      });
-    }
-    if (declares) {
-      declares.forEach(param => {
-        param = param.match(/(\w+)/g);
-        if (param && param.length >= 2) {
-          delete params[param[1]];
-        }
-      });
-    }
-    if (defaults) {
-      defaults.forEach(param => {
-        const line = param.match(/(\w+)/g);
-        if (line && line.length >= 2) {
-          const name = line[1];
-          params[name] = param.substring(param.indexOf(name) + name.length + 1);
-        }
-      });
-    }
-    if (macro_defines) {
-      macro_defines.forEach(params_line => {
-        const param_line = params_line.match(/(\w+)/g);
-        if (param_line && param_line.length > 2) {
-          param_line.forEach((param, index) => {
-            if (index >= 2) {
-              // Skips define NAME
-              delete params[param];
-            }
-          });
-        }
-      });
-    }
-    if (macro_returns) {
-      macro_returns.forEach(params_line => {
-        const param_line = params_line.match(/(\w+)/g);
-        if (param_line) {
-          param_line.forEach((param, index) => {
-            if (index >= 1) {
-              // Skip returns
-              delete params[param];
-            }
-          });
-        }
-      });
-    }
-
-    return params;
-  }
-
   getPlaceHolder() {
     return this.parentVm.getSnippetViewSettings(this.dialect()).placeHolder;
   }
@@ -1500,20 +1177,7 @@ export default class Snippet {
       statementType: this.statementType(),
       status: this.status(),
       type: this.dialect(), // TODO: Drop once connectors are stable
-      variables: this.variables().map(variable => ({
-        meta: variable.meta && {
-          options: variable.meta.options && variable.meta.options(), // TODO: Map?
-          placeHolder: variable.meta.placeHolder && variable.meta.placeHolder(),
-          type: variable.meta.type && variable.meta.type()
-        },
-        name: variable.name(),
-        path: variable.path(),
-        sample: variable.sample(),
-        sampleUser: variable.sampleUser(),
-        step: variable.step(),
-        type: variable.type(),
-        value: variable.value()
-      })),
+      variables: this.variableSubstitutionHandler.variables().map(variable => variable.toJs()),
       wasBatchExecuted: this.wasBatchExecuted()
     };
   }

+ 484 - 0
desktop/core/src/desktop/js/apps/notebook2/variableSubstitution.ts

@@ -0,0 +1,484 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Cancellable } from 'api/cancellablePromise';
+import { DIALECT } from 'apps/notebook2/snippet';
+import DataCatalogEntry from 'catalog/dataCatalogEntry';
+import ko, { PureComputed } from 'knockout';
+import { Observable, ObservableArray } from 'knockout';
+import { IdentifierLocation } from 'parse/types';
+import { Connector } from 'types/config';
+import { defer } from 'utils/hueUtils';
+
+interface VariableMetaRaw {
+  type: string;
+  placeholder: string;
+  options?: VariableOption[];
+}
+
+interface VariableMeta {
+  type: Observable<string>;
+  placeholder: Observable<string>;
+  options: ObservableArray<VariableOption>;
+}
+
+const variableMetaFromJs = (variableMetaRaw: VariableMetaRaw): VariableMeta => ({
+  type: ko.observable(variableMetaRaw.type),
+  placeholder: ko.observable(variableMetaRaw.placeholder),
+  options: ko.observableArray(variableMetaRaw.options || [])
+});
+
+const variableMetaToJs = (variableMeta: VariableMeta): VariableMetaRaw => ({
+  type: variableMeta.type(),
+  placeholder: variableMeta.placeholder(),
+  options: variableMeta.options()
+});
+
+interface VariableRaw {
+  name: string;
+  defaultValue?: VariableMetaRaw | string;
+  meta?: VariableMetaRaw;
+  value: string;
+  type: string;
+  catalogEntry?: DataCatalogEntry;
+  sample: VariableOption[];
+  sampleUser: VariableOption[];
+  path: string;
+  step: string;
+}
+
+interface ExtractedVariable {
+  name: string;
+  meta: VariableMeta;
+}
+
+interface VariableOption {
+  text: string;
+  value: string;
+}
+
+class Variable {
+  name: Observable<string>;
+  meta: VariableMeta;
+  value: Observable<string>;
+  type: Observable<string>;
+  sample: ObservableArray<VariableOption>;
+  sampleUser: ObservableArray<VariableOption>;
+  path: Observable<string>;
+  step: Observable<string> = ko.observable('');
+  catalogEntry?: DataCatalogEntry;
+
+  constructor(variableRaw?: VariableRaw) {
+    this.meta = variableMetaFromJs(
+      (typeof variableRaw?.defaultValue === 'object' && variableRaw?.defaultValue) ||
+        variableRaw?.meta || {
+          type: 'text',
+          placeholder: ''
+        }
+    );
+    this.name = ko.observable(variableRaw?.name || '');
+    this.value = ko.observable(variableRaw?.value || '');
+    this.type = ko.observable(variableRaw?.type || '');
+    this.sampleUser = ko.observableArray(variableRaw?.sampleUser || []);
+    this.path = ko.observable(variableRaw?.path || '');
+
+    this.sample = ko.observableArray(); // TODO: in raw?
+    this.sampleUser = ko.observableArray(); // TODO: in raw?
+  }
+
+  toJs(): VariableRaw {
+    return {
+      name: this.name(),
+      meta: variableMetaToJs(this.meta),
+      value: this.value(),
+      type: this.type(),
+      sample: this.sample(),
+      sampleUser: this.sampleUser(),
+      path: this.path(),
+      step: this.step(),
+      catalogEntry: this.catalogEntry
+    };
+  }
+}
+
+const LOCATION_VALUE_REGEX = /\${(\w*)=?([^{}]*)}/g;
+
+export class VariableSubstitutionHandler {
+  variables: ObservableArray<Variable> = ko.observableArray();
+  connector: Observable<Connector>;
+  variableNames: PureComputed<ExtractedVariable[]>;
+  statementRaw: Observable<string>;
+  variableValues: { [key: string]: VariableRaw } = {};
+  activeCancellables: Cancellable[] = [];
+
+  constructor(
+    connector: Observable<Connector>,
+    statementRaw: Observable<string>,
+    rawVariables?: VariableRaw[]
+  ) {
+    this.connector = connector;
+    this.statementRaw = statementRaw;
+    if (rawVariables) {
+      this.variables(rawVariables.map(rawVariable => new Variable(rawVariable)));
+    }
+
+    this.variableNames = ko.pureComputed(() => {
+      return this.extractVariables(this.statementRaw());
+    });
+
+    this.variableNames.extend({ rateLimit: 150 });
+
+    this.variableNames.subscribe(newVal => {
+      const variablesLength = this.variables().length;
+      const diffLengthVariables = variablesLength - newVal.length;
+      const needsMore = diffLengthVariables < 0;
+      const needsLess = diffLengthVariables > 0;
+
+      this.variableValues = {};
+      this.variables().forEach(variable => {
+        const name = variable.name();
+        if (!this.variableValues[name]) {
+          this.variableValues[name] = variable.toJs();
+        }
+      });
+
+      if (needsMore) {
+        for (let i = 0, length = Math.abs(diffLengthVariables); i < length; i++) {
+          this.variables.push(
+            new Variable({
+              name: '',
+              value: '',
+              meta: { type: 'text', placeholder: '', options: [] },
+              sample: [],
+              sampleUser: [],
+              type: 'text',
+              step: '',
+              path: ''
+            })
+          );
+        }
+      } else if (needsLess) {
+        this.variables.splice(this.variables().length - diffLengthVariables, diffLengthVariables);
+      }
+
+      newVal.forEach((item, index) => {
+        const variable = this.variables()[index];
+        const variableValue = this.variableValues[item.name];
+        variable.name(item.name);
+        window.setTimeout(() => {
+          variable.value(
+            variableValue ? variableValue.value : (!needsMore && variable.value()) || ''
+          );
+        }, 0);
+        variable.meta.placeholder(item.meta.placeholder());
+        variable.meta.options(item.meta.options());
+        variable.meta.type(item.meta.type());
+        variable.sample(
+          variable.meta.options()
+            ? variable.meta.options().concat(variable.sampleUser())
+            : variable.sampleUser()
+        );
+        variable.sampleUser(variableValue?.sampleUser || []);
+        variable.type(variableValue?.type || 'text');
+        variable.path(variableValue?.path || '');
+        variable.catalogEntry = variableValue?.catalogEntry;
+      });
+    });
+  }
+
+  extractVariables(statement: string): ExtractedVariable[] {
+    if (this.connector().dialect === DIALECT.pig) {
+      return this.getPigParameters(statement);
+    }
+
+    const foundParameters: { [key: string]: VariableMeta } = {};
+
+    let match: RegExpExecArray | null;
+    let matchList: RegExpExecArray | null;
+
+    const re = /(?:^|\W)\${(\w*)=?([^{}]*)}/g;
+    const reComment = /(^\s*--.*)|(\/\*[\s\S]*?\*\/)/gm;
+    const reList = /(?!\s*$)\s*(?:(?:([^,|()\\]*)\(\s*([^,|()\\]*)\)(?:\\[\S\s][^,|()\\]*)?)|([^,|\\]*(?:\\[\S\s][^,|\\]*)*))\s*(?:,|\||$)/g;
+
+    let matchComment = reComment.exec(statement);
+
+    while ((match = re.exec(statement))) {
+      while (matchComment && match.index > matchComment.index + matchComment[0].length) {
+        // Comments before our match
+        matchComment = reComment.exec(statement);
+      }
+      const isWithinComment = matchComment && match.index >= matchComment.index;
+      if (isWithinComment) {
+        continue;
+      }
+
+      const name = match[1];
+      if (foundParameters[name]) {
+        continue; // Return the first if multiple present
+      }
+
+      // If 1 match, text value
+      // If multiple matches, list value
+      const value: { type: string; placeholder: string; options: VariableOption[] } = {
+        type: 'text',
+        placeholder: '',
+        options: []
+      };
+
+      while ((matchList = reList.exec(match[2]))) {
+        const option: VariableOption = {
+          text: matchList[2] || matchList[3],
+          value: matchList[3] || matchList[1]
+        };
+        option.text = option.text && option.text.trim();
+        option.value =
+          option.value && option.value.trim().replace(',', ',').replace('(', '(').replace(')', ')');
+
+        if (value.placeholder || matchList[2]) {
+          value.type = 'select';
+          value.options.push(option);
+        }
+        if (!value.placeholder) {
+          value.placeholder = option.value;
+        }
+      }
+      const isPlaceholderInOptions =
+        !value.options || value.options.some(current => current.value === value.placeholder);
+      if (!isPlaceholderInOptions) {
+        value.options.unshift({ text: value.placeholder, value: value.placeholder });
+      }
+
+      foundParameters[name] = variableMetaFromJs(value);
+    }
+
+    return Object.keys(foundParameters).map(key => ({ name: key, meta: foundParameters[key] }));
+  }
+
+  getPigParameters(statement: string): ExtractedVariable[] {
+    const foundParameters: { [key: string]: string } = {};
+
+    const variables = statement.match(/([^\\]|^)\$[^\d'"](\w*)/g);
+    if (variables) {
+      variables.forEach(param => {
+        const p = param.substring(param.indexOf('$') + 1);
+        foundParameters[p] = '';
+      });
+    }
+
+    const declares = statement.match(/%declare +([^ ])+/gi);
+    if (declares) {
+      declares.forEach(param => {
+        const match = param.match(/(\w+)/g);
+        if (match && match.length >= 2) {
+          delete foundParameters[match[1]];
+        }
+      });
+    }
+
+    const defaults = statement.match(/%default +([^;])+/gi);
+    if (defaults) {
+      defaults.forEach(param => {
+        const line = param.match(/(\w+)/g);
+        if (line && line.length >= 2) {
+          const name = line[1];
+          foundParameters[name] = param.substring(param.indexOf(name) + name.length + 1);
+        }
+      });
+    }
+
+    const macroDefines = statement.match(/define [^ ]+ *\(([^)]*)\)/gi); // no multiline
+    if (macroDefines) {
+      macroDefines.forEach(param => {
+        const line = param.match(/(\w+)/g);
+        if (line && line.length > 2) {
+          line.forEach((param, index) => {
+            if (index >= 2) {
+              // Skips define NAME
+              delete foundParameters[param];
+            }
+          });
+        }
+      });
+    }
+
+    const macroReturns = statement.match(/returns +([^{]*)/gi); // no multiline
+    if (macroReturns) {
+      macroReturns.forEach(param => {
+        const line = param.match(/(\w+)/g);
+        if (line) {
+          line.forEach((param, index) => {
+            if (index >= 1) {
+              // Skip returns
+              delete foundParameters[param];
+            }
+          });
+        }
+      });
+    }
+    return Object.keys(foundParameters).map(key => ({
+      name: key,
+      meta: variableMetaFromJs({ type: 'text', placeholder: foundParameters[key] })
+    }));
+  }
+
+  cancelRunningRequests(): void {
+    while (this.activeCancellables.length) {
+      const cancellable = this.activeCancellables.pop();
+      if (cancellable) {
+        cancellable.cancel();
+      }
+    }
+  }
+
+  updateFromLocations(locations: IdentifierLocation[]): void {
+    const oLocations: { [key: string]: IdentifierLocation } = {};
+
+    for (const location of locations) {
+      if (location.type !== 'variable' || !location.colRef || !location.value) {
+        continue;
+      }
+      const match = LOCATION_VALUE_REGEX.exec(location.value);
+      if (match) {
+        const name = match[1];
+        oLocations[name] = location;
+      }
+    }
+
+    const updateVariableType = (variable: Variable, sourceMeta?: { type?: string }) => {
+      let type;
+      if (sourceMeta && sourceMeta.type) {
+        type = sourceMeta.type.toLowerCase();
+      } else {
+        type = 'string';
+      }
+      const variablesValues: { type: string; step: string; value?: string } = {
+        type: '',
+        step: ''
+      };
+      const value = variable.value();
+      switch (type) {
+        case 'timestamp':
+          variablesValues.type = 'datetime-local';
+          variablesValues.step = '1';
+          variablesValues.value =
+            (value && moment.utc(value).format('YYYY-MM-DD HH:mm:ss.S')) ||
+            moment(Date.now()).format('YYYY-MM-DD 00:00:00.0');
+          break;
+        case 'decimal':
+        case 'double':
+        case 'float':
+          variablesValues.type = 'number';
+          variablesValues.step = 'any';
+          break;
+        case 'int':
+        case 'smallint':
+        case 'tinyint':
+        case 'bigint':
+          variablesValues.type = 'number';
+          variablesValues.step = '1';
+          break;
+        case 'date':
+          variablesValues.type = 'date';
+          variablesValues.step = '';
+          variablesValues.value =
+            (value && moment.utc(value).format('YYYY-MM-DD')) ||
+            moment(Date.now()).format('YYYY-MM-DD');
+          break;
+        case 'boolean':
+          variablesValues.type = 'checkbox';
+          variablesValues.step = '';
+          break;
+        default:
+          variablesValues.type = 'text';
+          variablesValues.step = '';
+      }
+      if (variablesValues.value) {
+        defer(() => {
+          if (variablesValues.value) {
+            variable.value(variablesValues.value);
+          }
+        });
+      }
+      variable.type(variablesValues.type);
+      variable.step(variablesValues.step);
+    };
+    this.variables().forEach(variable => {
+      const location = oLocations[variable.name()];
+      if (location && location.resolveCatalogEntry) {
+        const catalogEntryPromise = location.resolveCatalogEntry({ cancellable: true });
+        this.activeCancellables.push(catalogEntryPromise);
+        catalogEntryPromise.then(entry => {
+          variable.path(entry.getQualifiedPath());
+          variable.catalogEntry = entry;
+
+          const sourceMetaPromise = entry.getSourceMeta({
+            silenceErrors: true,
+            cancellable: true
+          });
+
+          this.activeCancellables.push(sourceMetaPromise);
+
+          sourceMetaPromise.then(sourceMeta => {
+            updateVariableType(variable, sourceMeta);
+          });
+        });
+      } else {
+        updateVariableType(variable, {
+          type: 'text'
+        });
+      }
+    });
+  }
+
+  substitute(statement: string): string {
+    if (!this.variables().length) {
+      return statement;
+    }
+    const variables: { [key: string]: Variable } = {};
+    this.variables().forEach(variable => {
+      variables[variable.name()] = variable;
+    });
+
+    const variablesString = this.variables()
+      .map(variable => variable.name())
+      .join('|');
+
+    statement = statement.replace(
+      RegExp(
+        '([^\\\\])?\\$' +
+          (this.connector().dialect !== DIALECT.pig ? '{(' : '(') +
+          variablesString +
+          ')(=[^}]*)?' +
+          (this.connector().dialect !== DIALECT.pig ? '}' : ''),
+        'g'
+      ),
+      (match, p1, p2) => {
+        const variable = variables[p2];
+        const pad =
+          variable.type() === 'datetime-local' && variable.value().length === 16 ? ':00' : ''; // Chrome drops the seconds from the timestamp when it's at 0 second.
+        const value = variable.value();
+        const isValuePresent = //If value is string there is a need to check whether it is empty
+          typeof value === 'string' ? value : value !== undefined && value !== null;
+        return (
+          p1 +
+          (isValuePresent ? value + pad : variable.meta.placeholder && variable.meta.placeholder())
+        );
+      }
+    );
+
+    return statement;
+  }
+}

+ 20 - 0
desktop/core/src/desktop/js/parse/types.ts

@@ -14,6 +14,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import CancellableJqPromise from 'api/cancellableJqPromise';
+import DataCatalogEntry from 'catalog/dataCatalogEntry';
 import { ParsedSqlStatement } from './sqlStatementsParser';
 
 export interface IdentifierChainEntry {
@@ -32,6 +34,24 @@ export interface ParsedLocation {
   last_column: number;
 }
 
+export interface IdentifierLocation {
+  type: string;
+  alias?: string;
+  source?: string;
+  location: ParsedLocation;
+  function?: string;
+  missing?: boolean;
+  value?: string;
+  colRef: boolean;
+  argumentPosition?: number;
+  identifierChain?: IdentifierChainEntry[];
+  expression?: { types: string[]; text: string };
+  parentLocation?: ParsedLocation;
+  resolveCatalogEntry?: (options?: {
+    cancellable?: boolean;
+  }) => CancellableJqPromise<DataCatalogEntry>;
+}
+
 export interface StatementDetails {
   selectedStatements: ParsedSqlStatement[];
   precedingStatements: ParsedSqlStatement[];

+ 9 - 0
desktop/core/src/desktop/js/types/types.ts

@@ -18,3 +18,12 @@ export interface GenericApiResponse {
   status: number;
   message?: string;
 }
+
+declare global {
+  export interface Moment {
+    utc: (val: unknown) => Moment;
+    format: (format: string) => string;
+  }
+
+  const moment: Moment & ((val: unknown) => Moment);
+}

+ 1 - 1
desktop/libs/notebook/src/notebook/templates/editor_components2.mako

@@ -987,7 +987,7 @@
 
 
   <script type="text/html" id="snippet-variables">
-    <div class="variables">
+    <div class="variables" data-bind="with: variableSubstitutionHandler">
       <ul data-bind="foreach: variables" class="unstyled inline">
         <li>
           <div class="input-prepend margin-top-10">