Kaynağa Gözat

[editor] Add variable substitution in editor v2

Johan Ahlen 4 yıl önce
ebeveyn
işleme
a6ce56e0b9

+ 46 - 0
desktop/core/src/desktop/js/apps/editor/components/variableSubstitution/VariableSubstitution.scss

@@ -0,0 +1,46 @@
+@import '../../../../components/styles/colors.scss';
+@import '../../../../components/styles/mixins.scss';
+
+.variable-substitution {
+  margin: 10px 5px 0 5px;
+
+  >:not(:first-child) {
+    margin-left: 10px;
+  }
+
+  .variable-value {
+    display: inline-block;
+    font-size: 0; // Removes space between label and input
+
+    .variable-label,
+    .variable-input {
+      display: inline-block;
+      padding: 3px 6px;
+      font-size: 14px;
+      vertical-align: top;
+      border: 1px solid $fluid-gray-300;
+    }
+
+    .variable-label {
+      min-width: 16px;
+      background-color: $fluid-gray-050;
+      color: $fluid-gray-800;
+      border-radius: 2px 0 0 2px;
+      border-right: none;
+      text-align: center;
+    }
+
+    .variable-input {
+      min-width: 75px;
+      width: auto;
+      border-radius: 0 2px 2px 0;
+      box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.08);
+
+      &:active,
+      &:focus {
+        outline: none;
+        border-color: $hue-primary-color-dark;
+      }
+    }
+  }
+}

+ 238 - 0
desktop/core/src/desktop/js/apps/editor/components/variableSubstitution/VariableSubstitution.vue

@@ -0,0 +1,238 @@
+<!--
+  Licensed to Cloudera, Inc. under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  Cloudera, Inc. licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<template>
+  <div class="variable-substitution">
+    <span v-for="variable of activeVariables" :key="variable.name">
+      <div class="variable-value">
+        <div class="variable-label">{{ variable.name }}</div>
+        <input
+          v-model="variable.value"
+          class="variable-input"
+          :placeholder="variable.meta.placeholder"
+        />
+      </div>
+    </span>
+  </div>
+</template>
+
+<script lang="ts">
+  import { cloneDeep } from 'lodash';
+  import Vue from 'vue';
+  import Component from 'vue-class-component';
+  import { Prop, Watch } from 'vue-property-decorator';
+
+  import { Variable, VariableIndex, VariableOption } from './types';
+  import './VariableSubstitution.scss';
+  import { IdentifierLocation } from 'parse/types';
+  import DataCatalogEntry, { FieldSourceMeta } from 'catalog/DataCatalogEntry';
+  import { noop } from 'utils/hueUtils';
+
+  const NAMED_OPTION_REGEX = /^(.+)\(([^)]+)\)$/;
+  const LOCATION_VALUE_REGEX = /\${(\w+)=?([^{}]*)}/;
+
+  interface KnownVariable extends Variable {
+    active: boolean;
+    index: number;
+  }
+
+  const parseOption = (val: string): VariableOption => {
+    const namedMatch = val.match(NAMED_OPTION_REGEX);
+    if (namedMatch) {
+      return {
+        value: namedMatch[1].trim(),
+        text: namedMatch[2].trim()
+      };
+    }
+    const trimmed = val.trim();
+    return {
+      value: trimmed,
+      text: trimmed
+    };
+  };
+
+  const updateFromDataCatalog = async (
+    variable: Variable,
+    entry: DataCatalogEntry
+  ): Promise<void> => {
+    try {
+      variable.path = entry.getQualifiedPath();
+      variable.catalogEntry = entry;
+
+      const sourceMeta = <FieldSourceMeta>await entry.getSourceMeta({ silenceErrors: true });
+
+      const colType = (sourceMeta && sourceMeta.type) || 'string';
+
+      switch (colType) {
+        case 'timestamp':
+          variable.type = 'datetime-local';
+          variable.step = '1';
+          if (!variable.value) {
+            variable.value = String(Date.now());
+          }
+          break;
+        case 'date':
+          variable.type = 'date';
+          variable.step = '';
+          if (!variable.value) {
+            variable.value = String(Date.now());
+          }
+          break;
+        case 'decimal':
+        case 'double':
+        case 'float':
+          variable.type = 'number';
+          variable.step = 'any';
+          break;
+        case 'int':
+        case 'smallint':
+        case 'tinyint':
+        case 'bigint':
+          variable.type = 'number';
+          variable.step = '1';
+          break;
+        case 'boolean':
+          variable.type = 'checkbox';
+          variable.step = '';
+          break;
+        default:
+          variable.type = 'text';
+          variable.step = '';
+      }
+    } catch (err) {}
+  };
+
+  @Component
+  export default class VariableSubstitution extends Vue {
+    @Prop({ required: false, default: [] })
+    initialVariables?: Variable[];
+    @Prop()
+    locations!: IdentifierLocation[];
+
+    knownVariables: { [name: string]: KnownVariable } = {};
+
+    get activeVariables(): Variable[] {
+      const active = Object.values(this.knownVariables).filter(variable => variable.active);
+      active.sort((a, b) => a.index - b.index);
+      return active;
+    }
+
+    mounted(): void {
+      if (this.initialVariables) {
+        this.initialVariables.forEach((variable, index) => {
+          const cloned = <KnownVariable>cloneDeep(variable);
+          cloned.active = true;
+          cloned.index = index;
+          this.$set(this.knownVariables, cloned.name, cloned);
+        });
+      }
+    }
+
+    @Watch('activeVariables')
+    notifyVariableChange(): void {
+      this.$emit(
+        'variables-changed',
+        this.activeVariables.reduce((result, variable) => {
+          result[variable.name] = variable;
+          return result;
+        }, <VariableIndex>{})
+      );
+    }
+
+    @Watch('locations', { immediate: true })
+    updateFromLocations(locations: IdentifierLocation[]): void {
+      const toDeactivate = new Set<string>(Object.keys(this.knownVariables));
+
+      locations
+        .filter(location => location.type === 'variable' && location.value)
+        .forEach(location => {
+          const match = location.value && location.value.match(LOCATION_VALUE_REGEX);
+          if (!match) {
+            return;
+          }
+
+          const name = match[1];
+          let variable = this.knownVariables[name];
+          if (variable) {
+            toDeactivate.delete(variable.name);
+            if (!variable.active) {
+              this.$set(variable, 'active', true);
+            }
+          } else {
+            variable = {
+              meta: { type: 'text', placeholder: '', options: [] },
+              sample: [],
+              sampleUser: [],
+              step: '',
+              type: 'text',
+              value: '',
+              name: name,
+              active: true,
+              index: Object.keys(this.knownVariables).length + 1
+            };
+            this.$set(this.knownVariables, name, variable);
+          }
+
+          // Case for ${name=1} or ${name=a,b,c}
+          if (match[2]) {
+            const optionStrings = match[2].split(',');
+
+            // When it's just one option it's a placeholder only
+            if (optionStrings.length === 1) {
+              const option = parseOption(optionStrings[0]);
+              if (variable.meta.placeholder !== option.value) {
+                this.$set(variable.meta, 'placeholder', option.value);
+              }
+              variable.meta.options = [];
+            } else {
+              variable.type = 'select';
+              variable.meta.placeholder = '';
+              variable.meta.options = optionStrings.map(parseOption);
+            }
+          } else {
+            if (variable.type === 'select') {
+              // Revert to last known type if options are removed
+              if (variable.catalogEntry) {
+                updateFromDataCatalog(variable, variable.catalogEntry);
+              } else {
+                variable.type = 'text';
+              }
+            }
+            variable.meta.placeholder = '';
+            variable.meta.options = [];
+          }
+
+          if (location.colRef && location.resolveCatalogEntry) {
+            location
+              .resolveCatalogEntry()
+              .then(async entry => {
+                await updateFromDataCatalog(variable, entry);
+              })
+              .catch(noop);
+          }
+        });
+
+      toDeactivate.forEach(name => {
+        const variable = this.knownVariables[name];
+        if (variable) {
+          this.$set(variable, 'active', false);
+        }
+      });
+    }
+  }
+</script>

+ 71 - 0
desktop/core/src/desktop/js/apps/editor/components/variableSubstitution/VariableSubstitutionKoBridge.vue

@@ -0,0 +1,71 @@
+<!--
+  Licensed to Cloudera, Inc. under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  Cloudera, Inc. licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<template>
+  <VariableSubstitution
+    v-if="initialVariables"
+    :locations="locations"
+    :initial-variables="initialVariables"
+    @variables-changed="onVariablesChanged"
+  />
+</template>
+
+<script lang="ts">
+  import Vue from 'vue';
+  import Component from 'vue-class-component';
+  import { Prop } from 'vue-property-decorator';
+  import { wrap } from 'vue/webComponentWrapper';
+
+  import { Variable } from './types';
+  import VariableSubstitution from './VariableSubstitution.vue';
+  import SubscriptionTracker from 'components/utils/SubscriptionTracker';
+  import { IdentifierLocation } from 'parse/types';
+  import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/sqlWorkerHandler';
+
+  @Component({
+    components: { VariableSubstitution }
+  })
+  export default class VariableSubstitutionKoBridge extends Vue {
+    @Prop()
+    initialVariables?: Variable[];
+
+    locations: IdentifierLocation[] = [];
+
+    subTracker = new SubscriptionTracker();
+
+    mounted(): void {
+      this.subTracker.subscribe(
+        POST_FROM_LOCATION_WORKER_EVENT,
+        (e: { data?: { locations?: IdentifierLocation[] } }) => {
+          if (e.data && e.data.locations) {
+            this.locations = e.data.locations;
+          }
+        }
+      );
+    }
+
+    onVariablesChanged(variables: Variable[]): void {
+      this.$el.dispatchEvent(
+        new CustomEvent<Variable[]>('variables-changed', { bubbles: true, detail: variables })
+      );
+    }
+  }
+
+  export const COMPONENT_NAME = 'variable-substitution-ko-bridge';
+  wrap(COMPONENT_NAME, VariableSubstitutionKoBridge);
+</script>

+ 45 - 0
desktop/core/src/desktop/js/apps/editor/components/variableSubstitution/types.ts

@@ -0,0 +1,45 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import DataCatalogEntry from 'catalog/DataCatalogEntry';
+
+export interface VariableOption {
+  text: string;
+  value: string;
+}
+
+export interface VariableMeta {
+  type: string;
+  placeholder: string;
+  options: VariableOption[];
+}
+
+export type VariableType = 'checkbox' | 'date' | 'datetime-local' | 'number' | 'select' | 'text';
+
+export interface Variable {
+  defaultValue?: VariableMeta | string;
+  name: string;
+  meta: VariableMeta;
+  value: string;
+  type: VariableType;
+  sample: VariableOption[];
+  sampleUser: VariableOption[];
+  path?: string;
+  step: string;
+  catalogEntry?: DataCatalogEntry;
+}
+
+export type VariableIndex = { [name: string]: Variable };

+ 1 - 0
desktop/core/src/desktop/js/apps/editor/execution/executable.ts

@@ -429,6 +429,7 @@ export default abstract class Executable {
   async toContext(id?: string): Promise<ExecutableContext> {
     const session = await sessionManager.getSession({ type: this.executor.connector().id });
     if (this.executor.snippet) {
+      this.executor.snippet.statement_raw(this.getStatement());
       return {
         operationId: this.operationId,
         snippet: this.executor.snippet.toContextJson(),

+ 2 - 4
desktop/core/src/desktop/js/apps/editor/execution/executor.ts

@@ -17,9 +17,9 @@
 import Snippet from 'apps/editor/snippet';
 import Executable, { ExecutableRaw } from 'apps/editor/execution/executable';
 import { syncSqlExecutables } from 'apps/editor/execution/utils';
-import { VariableSubstitutionHandler } from 'apps/editor/variableSubstitution';
 import { StatementDetails } from 'parse/types';
 import { Compute, Connector, Namespace } from 'types/config';
+import { VariableIndex } from 'apps/editor/components/variableSubstitution/types';
 
 export interface ExecutorRaw {
   executables: ExecutableRaw[];
@@ -34,9 +34,9 @@ export default class Executor {
   isSqlEngine?: boolean;
   isOptimizerEnabled?: boolean;
   executables: Executable[] = [];
-  variableSubstitionHandler?: VariableSubstitutionHandler;
   snippet?: Snippet;
   activeExecutable?: Executable;
+  variables: VariableIndex = {};
 
   constructor(options: {
     connector: KnockoutObservable<Connector>;
@@ -48,7 +48,6 @@ export default class Executor {
     snippet?: Snippet;
     isOptimizerEnabled?: boolean;
     executables: Executable[];
-    variableSubstitionHandler?: VariableSubstitutionHandler;
   }) {
     this.connector = options.connector;
     this.compute = options.compute;
@@ -59,7 +58,6 @@ export default class Executor {
     this.executables = [];
     this.defaultLimit = options.defaultLimit;
     this.snippet = options.snippet;
-    this.variableSubstitionHandler = options.variableSubstitionHandler;
   }
 
   toJs(): ExecutorRaw {

+ 24 - 2
desktop/core/src/desktop/js/apps/editor/execution/sqlExecutable.ts

@@ -19,6 +19,7 @@ import Executable, { ExecutableRaw } from 'apps/editor/execution/executable';
 import { ExecutionError } from 'apps/editor/execution/executionLogs';
 import Executor from 'apps/editor/execution/executor';
 import { ParsedSqlStatement } from 'parse/sqlStatementsParser';
+import { VariableIndex } from '../components/variableSubstitution/types';
 
 const BATCHABLE_STATEMENT_TYPES = /ALTER|WITH|REFRESH|CREATE|DELETE|DROP|GRANT|INSERT|INVALIDATE|LOAD|SET|TRUNCATE|UPDATE|UPSERT|USE/i;
 
@@ -30,6 +31,27 @@ export interface SqlExecutableRaw extends ExecutableRaw {
   parsedStatement: ParsedSqlStatement;
 }
 
+const substituteVariables = (statement: string, variables: VariableIndex): string => {
+  if (!Object.keys(variables).length) {
+    return statement;
+  }
+
+  const variablesString = Object.values(variables)
+    .map(variable => variable.name)
+    .join('|');
+
+  return statement.replace(
+    RegExp('([^\\\\])?\\${(' + variablesString + ')(=[^}]*)?}', 'g'),
+    (match, p1, p2) => {
+      const { value, type, meta } = variables[p2];
+      const pad = type === 'datetime-local' && value.length === 16 ? ':00' : ''; // Chrome drops the seconds from the timestamp when it's at 0 second.
+      const isValuePresent = //If value is string there is a need to check whether it is empty
+        typeof value === 'string' ? value : value !== undefined && value !== null;
+      return p1 + (isValuePresent ? value + pad : meta.placeholder);
+    }
+  );
+};
+
 export default class SqlExecutable extends Executable {
   database: string;
   parsedStatement: ParsedSqlStatement;
@@ -69,8 +91,8 @@ export default class SqlExecutable extends Executable {
       }
     }
 
-    if (this.executor.variableSubstitionHandler) {
-      statement = this.executor.variableSubstitionHandler.substitute(statement);
+    if (this.executor.variables) {
+      statement = substituteVariables(statement, this.executor.variables);
     }
 
     return statement;

+ 8 - 20
desktop/core/src/desktop/js/apps/editor/snippet.js

@@ -31,6 +31,7 @@ import './components/aceEditor/AceEditorKoBridge.vue';
 import './components/executionAnalysis/ExecutionAnalysisPanelKoBridge.vue';
 import './components/presentationMode/PresentationModeKoBridge.vue';
 import './components/result/ResultTableKoBridge.vue';
+import './components/variableSubstitution/VariableSubstitutionKoBridge.vue';
 
 import AceAutocompleteWrapper from 'apps/notebook/aceAutocompleteWrapper';
 import apiHelper from 'api/apiHelper';
@@ -49,7 +50,6 @@ import {
 } from 'apps/editor/execution/executable';
 import {
   ACTIVE_STATEMENT_CHANGED_EVENT,
-  CURSOR_POSITION_CHANGED_EVENT,
   REFRESH_STATEMENT_LOCATIONS_EVENT
 } from 'ko/bindings/ace/aceLocationHandler';
 import { EXECUTE_ACTIVE_EXECUTABLE_EVENT } from './components/ExecutableActions.vue';
@@ -62,8 +62,6 @@ import {
   ASSIST_GET_SOURCE_EVENT,
   ASSIST_SET_SOURCE_EVENT
 } from 'ko/components/assist/events';
-import { POST_FROM_LOCATION_WORKER_EVENT } from 'sql/sqlWorkerHandler';
-import { VariableSubstitutionHandler } from './variableSubstitution';
 
 // TODO: Remove for ENABLE_NOTEBOOK_2. Temporary here for debug
 window.SqlExecutable = SqlExecutable;
@@ -448,20 +446,7 @@ export default class Snippet {
       }, 100);
     });
 
-    this.variableSubstitutionHandler = new VariableSubstitutionHandler(
-      this.connector,
-      this.statement_raw,
-      snippetRaw.variables
-    );
-
-    this.variableSubstitutionHandler.variables.subscribe(() => {
-      $(document).trigger('updateResultHeaders', this);
-    });
-
-    huePubSub.subscribe(POST_FROM_LOCATION_WORKER_EVENT, e => {
-      this.variableSubstitutionHandler.cancelRunningRequests();
-      this.variableSubstitutionHandler.updateFromLocations(e.data.locations);
-    });
+    this.initialVariables = snippetRaw.variables || [];
 
     this.statement = ko.pureComputed(() => {
       let statement = this.statement_raw();
@@ -613,8 +598,7 @@ export default class Snippet {
       defaultLimit: this.defaultLimit,
       isOptimizerEnabled: this.parentVm.isOptimizerEnabled(),
       snippet: this,
-      isSqlEngine: this.isSqlDialect,
-      variableSubstitionHandler: this.variableSubstitutionHandler
+      isSqlEngine: this.isSqlDialect
     });
 
     if (snippetRaw.executor) {
@@ -801,6 +785,10 @@ export default class Snippet {
     }
   }
 
+  setVariables(variables) {
+    this.executor.variables = variables;
+  }
+
   changeDialect(dialect) {
     const connector = findEditorConnector(connector => connector.dialect === dialect);
     if (!connector) {
@@ -1151,7 +1139,7 @@ export default class Snippet {
       statementType: this.statementType(),
       status: this.status(),
       type: this.dialect(), // TODO: Drop once connectors are stable
-      variables: this.variableSubstitutionHandler.variables().map(variable => variable.toJs()),
+      variables: this.variables,
       wasBatchExecuted: this.wasBatchExecuted()
     };
   }

+ 0 - 482
desktop/core/src/desktop/js/apps/editor/variableSubstitution.ts

@@ -1,482 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { Cancellable } from 'api/cancellablePromise';
-import { DIALECT } from 'apps/editor/snippet';
-import DataCatalogEntry, { FieldSourceMeta, SourceMeta } from 'catalog/DataCatalogEntry';
-import ko, { PureComputed } from 'knockout';
-import { Observable, ObservableArray } from 'knockout';
-import { IdentifierLocation } from 'parse/types';
-import { Connector } from 'types/config';
-import { defer } from 'utils/hueUtils';
-
-interface VariableMetaRaw {
-  type: string;
-  placeholder: string;
-  options?: VariableOption[];
-}
-
-interface VariableMeta {
-  type: Observable<string>;
-  placeholder: Observable<string>;
-  options: ObservableArray<VariableOption>;
-}
-
-const variableMetaFromJs = (variableMetaRaw: VariableMetaRaw): VariableMeta => ({
-  type: ko.observable(variableMetaRaw.type),
-  placeholder: ko.observable(variableMetaRaw.placeholder),
-  options: ko.observableArray(variableMetaRaw.options || [])
-});
-
-const variableMetaToJs = (variableMeta: VariableMeta): VariableMetaRaw => ({
-  type: variableMeta.type(),
-  placeholder: variableMeta.placeholder(),
-  options: variableMeta.options()
-});
-
-interface VariableRaw {
-  name: string;
-  defaultValue?: VariableMetaRaw | string;
-  meta?: VariableMetaRaw;
-  value: string;
-  type: string;
-  catalogEntry?: DataCatalogEntry;
-  sample: VariableOption[];
-  sampleUser: VariableOption[];
-  path: string;
-  step: string;
-}
-
-interface ExtractedVariable {
-  name: string;
-  meta: VariableMeta;
-}
-
-interface VariableOption {
-  text: string;
-  value: string;
-}
-
-class Variable {
-  name: Observable<string>;
-  meta: VariableMeta;
-  value: Observable<string>;
-  type: Observable<string>;
-  sample: ObservableArray<VariableOption>;
-  sampleUser: ObservableArray<VariableOption>;
-  path: Observable<string>;
-  step: Observable<string> = ko.observable('');
-  catalogEntry?: DataCatalogEntry;
-
-  constructor(variableRaw?: VariableRaw) {
-    this.meta = variableMetaFromJs(
-      (typeof variableRaw?.defaultValue === 'object' && variableRaw?.defaultValue) ||
-        variableRaw?.meta || {
-          type: 'text',
-          placeholder: ''
-        }
-    );
-    this.name = ko.observable(variableRaw?.name || '');
-    this.value = ko.observable(variableRaw?.value || '');
-    this.type = ko.observable(variableRaw?.type || '');
-    this.sampleUser = ko.observableArray(variableRaw?.sampleUser || []);
-    this.path = ko.observable(variableRaw?.path || '');
-
-    this.sample = ko.observableArray(); // TODO: in raw?
-    this.sampleUser = ko.observableArray(); // TODO: in raw?
-  }
-
-  toJs(): VariableRaw {
-    return {
-      name: this.name(),
-      meta: variableMetaToJs(this.meta),
-      value: this.value(),
-      type: this.type(),
-      sample: this.sample(),
-      sampleUser: this.sampleUser(),
-      path: this.path(),
-      step: this.step(),
-      catalogEntry: this.catalogEntry
-    };
-  }
-}
-
-const LOCATION_VALUE_REGEX = /\${(\w*)=?([^{}]*)}/g;
-
-export class VariableSubstitutionHandler {
-  variables: ObservableArray<Variable> = ko.observableArray();
-  connector: Observable<Connector>;
-  variableNames: PureComputed<ExtractedVariable[]>;
-  statementRaw: Observable<string>;
-  variableValues: { [key: string]: VariableRaw } = {};
-  activeCancellables: Cancellable[] = [];
-
-  constructor(
-    connector: Observable<Connector>,
-    statementRaw: Observable<string>,
-    rawVariables?: VariableRaw[]
-  ) {
-    this.connector = connector;
-    this.statementRaw = statementRaw;
-    if (rawVariables) {
-      this.variables(rawVariables.map(rawVariable => new Variable(rawVariable)));
-    }
-
-    this.variableNames = ko.pureComputed(() => this.extractVariables(this.statementRaw()));
-
-    this.variableNames.extend({ rateLimit: 150 });
-
-    this.variableNames.subscribe(newVal => {
-      const variablesLength = this.variables().length;
-      const diffLengthVariables = variablesLength - newVal.length;
-      const needsMore = diffLengthVariables < 0;
-      const needsLess = diffLengthVariables > 0;
-
-      this.variableValues = {};
-      this.variables().forEach(variable => {
-        const name = variable.name();
-        if (!this.variableValues[name]) {
-          this.variableValues[name] = variable.toJs();
-        }
-      });
-
-      if (needsMore) {
-        for (let i = 0, length = Math.abs(diffLengthVariables); i < length; i++) {
-          this.variables.push(
-            new Variable({
-              name: '',
-              value: '',
-              meta: { type: 'text', placeholder: '', options: [] },
-              sample: [],
-              sampleUser: [],
-              type: 'text',
-              step: '',
-              path: ''
-            })
-          );
-        }
-      } else if (needsLess) {
-        this.variables.splice(this.variables().length - diffLengthVariables, diffLengthVariables);
-      }
-
-      newVal.forEach((item, index) => {
-        const variable = this.variables()[index];
-        const variableValue = this.variableValues[item.name];
-        variable.name(item.name);
-        window.setTimeout(() => {
-          variable.value(
-            variableValue ? variableValue.value : (!needsMore && variable.value()) || ''
-          );
-        }, 0);
-        variable.meta.placeholder(item.meta.placeholder());
-        variable.meta.options(item.meta.options());
-        variable.meta.type(item.meta.type());
-        variable.sample(
-          variable.meta.options()
-            ? variable.meta.options().concat(variable.sampleUser())
-            : variable.sampleUser()
-        );
-        variable.sampleUser(variableValue?.sampleUser || []);
-        variable.type(variableValue?.type || 'text');
-        variable.path(variableValue?.path || '');
-        variable.catalogEntry = variableValue?.catalogEntry;
-      });
-    });
-  }
-
-  extractVariables(statement: string): ExtractedVariable[] {
-    if (this.connector().dialect === DIALECT.pig) {
-      return this.getPigParameters(statement);
-    }
-
-    const foundParameters: { [key: string]: VariableMeta } = {};
-
-    let match: RegExpExecArray | null;
-    let matchList: RegExpExecArray | null;
-
-    const re = /(?:^|\W)\${(\w*)=?([^{}]*)}/g;
-    const reComment = /(^\s*--.*)|(\/\*[\s\S]*?\*\/)/gm;
-    const reList = /(?!\s*$)\s*(?:(?:([^,|()\\]*)\(\s*([^,|()\\]*)\)(?:\\[\S\s][^,|()\\]*)?)|([^,|\\]*(?:\\[\S\s][^,|\\]*)*))\s*(?:,|\||$)/g;
-
-    let matchComment = reComment.exec(statement);
-
-    while ((match = re.exec(statement))) {
-      while (matchComment && match.index > matchComment.index + matchComment[0].length) {
-        // Comments before our match
-        matchComment = reComment.exec(statement);
-      }
-      const isWithinComment = matchComment && match.index >= matchComment.index;
-      if (isWithinComment) {
-        continue;
-      }
-
-      const name = match[1];
-      if (foundParameters[name]) {
-        continue; // Return the first if multiple present
-      }
-
-      // If 1 match, text value
-      // If multiple matches, list value
-      const value: { type: string; placeholder: string; options: VariableOption[] } = {
-        type: 'text',
-        placeholder: '',
-        options: []
-      };
-
-      while ((matchList = reList.exec(match[2]))) {
-        const option: VariableOption = {
-          text: matchList[2] || matchList[3],
-          value: matchList[3] || matchList[1]
-        };
-        option.text = option.text && option.text.trim();
-        option.value =
-          option.value && option.value.trim().replace(',', ',').replace('(', '(').replace(')', ')');
-
-        if (value.placeholder || matchList[2]) {
-          value.type = 'select';
-          value.options.push(option);
-        }
-        if (!value.placeholder) {
-          value.placeholder = option.value;
-        }
-      }
-      const isPlaceholderInOptions =
-        !value.options || value.options.some(current => current.value === value.placeholder);
-      if (!isPlaceholderInOptions) {
-        value.options.unshift({ text: value.placeholder, value: value.placeholder });
-      }
-
-      foundParameters[name] = variableMetaFromJs(value);
-    }
-
-    return Object.keys(foundParameters).map(key => ({ name: key, meta: foundParameters[key] }));
-  }
-
-  getPigParameters(statement: string): ExtractedVariable[] {
-    const foundParameters: { [key: string]: string } = {};
-
-    const variables = statement.match(/([^\\]|^)\$[^\d'"](\w*)/g);
-    if (variables) {
-      variables.forEach(param => {
-        const p = param.substring(param.indexOf('$') + 1);
-        foundParameters[p] = '';
-      });
-    }
-
-    const declares = statement.match(/%declare +([^ ])+/gi);
-    if (declares) {
-      declares.forEach(param => {
-        const match = param.match(/(\w+)/g);
-        if (match && match.length >= 2) {
-          delete foundParameters[match[1]];
-        }
-      });
-    }
-
-    const defaults = statement.match(/%default +([^;])+/gi);
-    if (defaults) {
-      defaults.forEach(param => {
-        const line = param.match(/(\w+)/g);
-        if (line && line.length >= 2) {
-          const name = line[1];
-          foundParameters[name] = param.substring(param.indexOf(name) + name.length + 1);
-        }
-      });
-    }
-
-    const macroDefines = statement.match(/define [^ ]+ *\(([^)]*)\)/gi); // no multiline
-    if (macroDefines) {
-      macroDefines.forEach(param => {
-        const line = param.match(/(\w+)/g);
-        if (line && line.length > 2) {
-          line.forEach((param, index) => {
-            if (index >= 2) {
-              // Skips define NAME
-              delete foundParameters[param];
-            }
-          });
-        }
-      });
-    }
-
-    const macroReturns = statement.match(/returns +([^{]*)/gi); // no multiline
-    if (macroReturns) {
-      macroReturns.forEach(param => {
-        const line = param.match(/(\w+)/g);
-        if (line) {
-          line.forEach((param, index) => {
-            if (index >= 1) {
-              // Skip returns
-              delete foundParameters[param];
-            }
-          });
-        }
-      });
-    }
-    return Object.keys(foundParameters).map(key => ({
-      name: key,
-      meta: variableMetaFromJs({ type: 'text', placeholder: foundParameters[key] })
-    }));
-  }
-
-  cancelRunningRequests(): void {
-    while (this.activeCancellables.length) {
-      const cancellable = this.activeCancellables.pop();
-      if (cancellable) {
-        cancellable.cancel();
-      }
-    }
-  }
-
-  updateFromLocations(locations: IdentifierLocation[]): void {
-    const oLocations: { [key: string]: IdentifierLocation } = {};
-
-    for (const location of locations) {
-      if (location.type !== 'variable' || !location.colRef || !location.value) {
-        continue;
-      }
-      const match = LOCATION_VALUE_REGEX.exec(location.value);
-      if (match) {
-        const name = match[1];
-        oLocations[name] = location;
-      }
-    }
-
-    const updateVariableType = (variable: Variable, sourceMeta?: SourceMeta) => {
-      let type;
-      if (sourceMeta && (<FieldSourceMeta>sourceMeta).type) {
-        type = (<FieldSourceMeta>sourceMeta).type.toLowerCase();
-      } else {
-        type = 'string';
-      }
-      const variablesValues: { type: string; step: string; value?: string } = {
-        type: '',
-        step: ''
-      };
-      const value = variable.value();
-      switch (type) {
-        case 'timestamp':
-          variablesValues.type = 'datetime-local';
-          variablesValues.step = '1';
-          variablesValues.value =
-            (value && moment.utc(value).format('YYYY-MM-DD HH:mm:ss.S')) ||
-            moment(Date.now()).format('YYYY-MM-DD 00:00:00.0');
-          break;
-        case 'decimal':
-        case 'double':
-        case 'float':
-          variablesValues.type = 'number';
-          variablesValues.step = 'any';
-          break;
-        case 'int':
-        case 'smallint':
-        case 'tinyint':
-        case 'bigint':
-          variablesValues.type = 'number';
-          variablesValues.step = '1';
-          break;
-        case 'date':
-          variablesValues.type = 'date';
-          variablesValues.step = '';
-          variablesValues.value =
-            (value && moment.utc(value).format('YYYY-MM-DD')) ||
-            moment(Date.now()).format('YYYY-MM-DD');
-          break;
-        case 'boolean':
-          variablesValues.type = 'checkbox';
-          variablesValues.step = '';
-          break;
-        default:
-          variablesValues.type = 'text';
-          variablesValues.step = '';
-      }
-      if (variablesValues.value) {
-        defer(() => {
-          if (variablesValues.value) {
-            variable.value(variablesValues.value);
-          }
-        });
-      }
-      variable.type(variablesValues.type);
-      variable.step(variablesValues.step);
-    };
-    this.variables().forEach(variable => {
-      const location = oLocations[variable.name()];
-      if (location && location.resolveCatalogEntry) {
-        const catalogEntryPromise = location.resolveCatalogEntry({ cancellable: true });
-        this.activeCancellables.push(catalogEntryPromise);
-        catalogEntryPromise.then(entry => {
-          variable.path(entry.getQualifiedPath());
-          variable.catalogEntry = entry;
-
-          const sourceMetaPromise = entry.getSourceMeta({
-            silenceErrors: true,
-            cancellable: true
-          });
-
-          this.activeCancellables.push(sourceMetaPromise);
-
-          sourceMetaPromise.then(sourceMeta => {
-            updateVariableType(variable, sourceMeta);
-          });
-        });
-      } else {
-        updateVariableType(variable, <FieldSourceMeta>{
-          type: 'text'
-        });
-      }
-    });
-  }
-
-  substitute(statement: string): string {
-    if (!this.variables().length) {
-      return statement;
-    }
-    const variables: { [key: string]: Variable } = {};
-    this.variables().forEach(variable => {
-      variables[variable.name()] = variable;
-    });
-
-    const variablesString = this.variables()
-      .map(variable => variable.name())
-      .join('|');
-
-    statement = statement.replace(
-      RegExp(
-        '([^\\\\])?\\$' +
-          (this.connector().dialect !== DIALECT.pig ? '{(' : '(') +
-          variablesString +
-          ')(=[^}]*)?' +
-          (this.connector().dialect !== DIALECT.pig ? '}' : ''),
-        'g'
-      ),
-      (match, p1, p2) => {
-        const variable = variables[p2];
-        const pad =
-          variable.type() === 'datetime-local' && variable.value().length === 16 ? ':00' : ''; // Chrome drops the seconds from the timestamp when it's at 0 second.
-        const value = variable.value();
-        const isValuePresent = //If value is string there is a need to check whether it is empty
-          typeof value === 'string' ? value : value !== undefined && value !== null;
-        return (
-          p1 +
-          (isValuePresent ? value + pad : variable.meta.placeholder && variable.meta.placeholder())
-        );
-      }
-    );
-
-    return statement;
-  }
-}

+ 1 - 1
desktop/core/src/desktop/js/parse/types.ts

@@ -68,7 +68,7 @@ export interface IdentifierLocation {
   parentLocation?: ParsedLocation;
   path?: string;
   qualified?: boolean;
-  resolveCatalogEntry: (options: {
+  resolveCatalogEntry: (options?: {
     cancellable?: boolean;
     temporaryOnly?: boolean;
   }) => CancellablePromise<DataCatalogEntry>;

+ 62 - 50
desktop/libs/notebook/src/notebook/templates/editor2.mako

@@ -382,56 +382,58 @@
   </div>
 </script>
 
-<script type="text/html" id="snippet-variables">
-  <div class="variables" data-bind="with: variableSubstitutionHandler">
-    <ul data-bind="foreach: variables" class="unstyled inline">
-      <li>
-        <div class="input-prepend margin-top-10">
-          <!-- ko ifnot: path() -->
-          <span class="muted add-on" data-bind="text: name"></span>
-          <!-- /ko -->
-          <!-- ko if: path() -->
-          <a href="javascript:void(0);" data-bind="click: $root.showContextPopover" style="float: left"> <span class="muted add-on" data-bind="text: name"></span></a>
-          <!-- /ko -->
-          <!-- ko if: meta.type() === 'text' -->
-          <!-- ko if: meta.placeholder() -->
-          <input class="input-medium" type="text" data-bind="value: value, attr: { value: value, type: type, placeholder: meta.placeholder() || '${ _ko('Variable value') }' }, valueUpdate: 'afterkeydown', event: { 'keydown': $parent.onKeydownInVariable }, autogrowInput: { minWidth: 150, maxWidth: 270, comfortZone: 15 }">
-          <!-- /ko -->
-          <!-- ko ifnot: meta.placeholder() -->
-          <!-- ko if: type() == 'datetime-local' -->
-          <input class="input-medium" type="text" data-bind="attr: { value: value }, value: value, datepicker: { momentFormat: 'YYYY-MM-DD HH:mm:ss.S' }">
-          <!-- /ko -->
-          <!-- ko if: type() == 'date' -->
-          <input class="input-medium" type="text" data-bind="attr: { value: value }, value: value, datepicker: { momentFormat: 'YYYY-MM-DD' }">
-          <!-- /ko -->
-          <!-- ko if: type() == 'checkbox' -->
-          <input class="input-medium" type="checkbox" data-bind="checked: value">
-          <!-- /ko -->
-          <!-- ko ifnot: (type() == 'datetime-local' || type() == 'date' || type() == 'checkbox') -->
-          <input class="input-medium" type="text" value="true" data-bind="value: value, attr: { value: value,  type: type() || 'text', step: step }, valueUpdate: 'afterkeydown', event: { 'keydown': $parent.onKeydownInVariable }, autogrowInput: { minWidth: 150, maxWidth: 270, comfortZone: 15 }">
-          <!-- /ko -->
-          <!-- /ko -->
-          <!-- /ko -->
-          <!-- ko if: meta.type() === 'select' -->
-          <select data-bind="
-                selectize: sample,
-                optionsText: 'text',
-                optionsValue: 'value',
-                selectizeOptions: {
-                  create: function (input) {
-                    sampleUser().push({ text: ko.observable(input), value: ko.observable(input) });
-                    return { text: input, value: input };
-                  }
-                },
-                value: value,
-                event: { 'keydown': $parent.onKeydownInVariable }
-              "></select>
-          <!-- /ko -->
-        </div>
-      </li>
-    </ul>
-  </div>
-</script>
+
+## TODO: Move additional types to VariableSubstitution.vue
+## <script type="text/html" id="snippet-variables">
+##   <div class="variables" data-bind="with: variableSubstitutionHandler">
+##     <ul data-bind="foreach: variables" class="unstyled inline">
+##       <li>
+##         <div class="input-prepend margin-top-10">
+##           <!-- ko ifnot: path() -->
+##           <span class="muted add-on" data-bind="text: name"></span>
+##           <!-- /ko -->
+##           <!-- ko if: path() -->
+##           <a href="javascript:void(0);" data-bind="click: $root.showContextPopover" style="float: left"> <span class="muted add-on" data-bind="text: name"></span></a>
+##           <!-- /ko -->
+##           <!-- ko if: meta.type() === 'text' -->
+##           <!-- ko if: meta.placeholder() -->
+##           <input class="input-medium" type="text" data-bind="value: value, attr: { value: value, type: type, placeholder: meta.placeholder() || '${ _ko('Variable value') }' }, valueUpdate: 'afterkeydown', event: { 'keydown': $parent.onKeydownInVariable }, autogrowInput: { minWidth: 150, maxWidth: 270, comfortZone: 15 }">
+##           <!-- /ko -->
+##           <!-- ko ifnot: meta.placeholder() -->
+##           <!-- ko if: type() == 'datetime-local' -->
+##           <input class="input-medium" type="text" data-bind="attr: { value: value }, value: value, datepicker: { momentFormat: 'YYYY-MM-DD HH:mm:ss.S' }">
+##           <!-- /ko -->
+##           <!-- ko if: type() == 'date' -->
+##           <input class="input-medium" type="text" data-bind="attr: { value: value }, value: value, datepicker: { momentFormat: 'YYYY-MM-DD' }">
+##           <!-- /ko -->
+##           <!-- ko if: type() == 'checkbox' -->
+##           <input class="input-medium" type="checkbox" data-bind="checked: value">
+##           <!-- /ko -->
+##           <!-- ko ifnot: (type() == 'datetime-local' || type() == 'date' || type() == 'checkbox') -->
+##           <input class="input-medium" type="text" value="true" data-bind="value: value, attr: { value: value,  type: type() || 'text', step: step }, valueUpdate: 'afterkeydown', event: { 'keydown': $parent.onKeydownInVariable }, autogrowInput: { minWidth: 150, maxWidth: 270, comfortZone: 15 }">
+##           <!-- /ko -->
+##           <!-- /ko -->
+##           <!-- /ko -->
+##           <!-- ko if: meta.type() === 'select' -->
+##           <select data-bind="
+##                 selectize: sample,
+##                 optionsText: 'text',
+##                 optionsValue: 'value',
+##                 selectizeOptions: {
+##                   create: function (input) {
+##                     sampleUser().push({ text: ko.observable(input), value: ko.observable(input) });
+##                     return { text: input, value: input };
+##                   }
+##                 },
+##                 value: value,
+##                 event: { 'keydown': $parent.onKeydownInVariable }
+##               "></select>
+##           <!-- /ko -->
+##         </div>
+##       </li>
+##     </ul>
+##   </div>
+## </script>
 
 <script type="text/html" id="editor-executable-snippet-body">
   <div style="padding:10px;">
@@ -895,6 +897,16 @@
           <!-- ko template: { name: 'editor-code-editor' } --><!-- /ko -->
         </div>
   ##      <!-- ko template: { name: 'snippet-variables' }--><!-- /ko -->
+          <variable-substitution-ko-bridge data-bind="
+            vueEvents: {
+              'variables-changed': function (event) {
+                setVariables(event.detail);
+              }
+            },
+            vueKoProps: {
+              initialVariables: []
+            }
+          "></variable-substitution-ko-bridge>
   ##      <!-- ko template: { name: 'editor-executable-snippet-body' } --><!-- /ko -->
         <div class="editor-execute-status">
           <!-- ko template: { name: 'editor-snippet-execution-status' } --><!-- /ko -->