// Licensed to Cloudera, Inc. under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. Cloudera, Inc. licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. var Resource = function (resource) { var self = this; self.type = ko.observable(resource.type); self.value = ko.observable(resource.value); }; var HadoopProperty = function (property) { var self = this; self.name = ko.observable(property.name); self.value = ko.observable(property.value); }; var PigParameter = HadoopProperty; var SparkScript = function (sparkScript) { var self = this; self.id = ko.observable(sparkScript.id); self.isDesign = ko.observable(sparkScript.isDesign); self.name = ko.observable(sparkScript.name); self.language = ko.observable(sparkScript.language); self.script = ko.observable(sparkScript.script); self.scriptSumup = ko.observable(sparkScript.script.replace(/\W+/g, ' ').substring(0, 100)); self.isRunning = ko.observable(false); self.selected = ko.observable(false); self.watchUrl = ko.observable(""); self.actions = ko.observableArray([]); self.handleSelect = function (row, e) { this.selected(!this.selected()); }; self.hovered = ko.observable(false); self.toggleHover = function (row, e) { this.hovered(!this.hovered()); }; self.parameters = ko.observableArray([]); ko.utils.arrayForEach(sparkScript.parameters, function (parameter) { self.parameters.push(new PigParameter({name: parameter.name, value: parameter.value})); }); self.addParameter = function () { self.parameters.push(new PigParameter({name: '', value: ''})); self.updateParentModel(); }; self.removeParameter = function () { self.parameters.remove(this); self.updateParentModel(); }; self.getParameters = function () { var params = {}; return params; }; self.hadoopProperties = ko.observableArray([]); ko.utils.arrayForEach(sparkScript.hadoopProperties, function (property) { self.hadoopProperties.push(new HadoopProperty({name: property.name, value: property.value})); }); self.addHadoopProperties = function () { self.hadoopProperties.push(new HadoopProperty({name: '', value: ''})); self.updateParentModel(); }; self.removeHadoopProperties = function () { self.hadoopProperties.remove(this); self.updateParentModel(); }; self.resources = ko.observableArray([]); ko.utils.arrayForEach(sparkScript.resources, function (resource) { self.resources.push(new Resource({type: resource.type, value: resource.value})); }); self.addResource = function () { self.resources.push(new Resource({type: 'file', value: ''})); self.updateParentModel(); }; self.removeResource = function () { self.resources.remove(this); self.updateParentModel(); }; self.parentModel = sparkScript.parentModel; self.updateParentModel = function () { if (typeof self.parentModel != "undefined" && self.parentModel != null) { self.parentModel.isDirty(true); } } self.name.subscribe(function (name) { self.updateParentModel(); }); } var Workflow = function (wf) { return { id: wf.id, scriptId: wf.scriptId, scriptContent: wf.scriptContent, lastModTime: wf.lastModTime, endTime: wf.endTime, status: wf.status, statusClass: "label " + getStatusClass(wf.status), isRunning: wf.isRunning, duration: wf.duration, appName: wf.appName, progress: wf.progress, progressPercent: wf.progressPercent, progressClass: "bar " + getStatusClass(wf.status, "bar-"), user: wf.user, absoluteUrl: wf.absoluteUrl, watchUrl: wf.watchUrl, canEdit: wf.canEdit, killUrl: wf.killUrl, created: wf.created, run: wf.run } } var SparkViewModel = function (props) { var self = this; self.LABELS = props.labels; self.LIST_SCRIPTS = props.listScripts; self.SAVE_URL = props.saveUrl; self.RUN_URL = props.runUrl; self.STOP_URL = props.stopUrl; self.COPY_URL = props.copyUrl; self.DELETE_URL = props.deleteUrl; self.isLoading = ko.observable(false); self.allSelected = ko.observable(false); self.submissionVariables = ko.observableArray([]); self.scripts = ko.observableArray([]); self.filteredScripts = ko.observableArray(self.scripts()); self.runningScripts = ko.observableArray([]); self.completedScripts = ko.observableArray([]); self.isDashboardLoaded = false; self.isDirty = ko.observable(false); var _defaultScript = { id: -1, name: self.LABELS.NEW_SCRIPT_NAME, language: 'python', script: self.LABELS.NEW_SCRIPT_CONTENT['python'], parameters: self.LABELS.NEW_SCRIPT_PARAMETERS, resources: self.LABELS.NEW_SCRIPT_RESOURCES, hadoopProperties: self.LABELS.NEW_SCRIPT_HADOOP_PROPERTIES, parentModel: self }; self.currentScript = ko.observable(new SparkScript(_defaultScript)); self.loadingScript = null; self.currentDeleteType = ko.observable(""); self.selectedScripts = ko.computed(function () { return ko.utils.arrayFilter(self.scripts(), function (script) { return script.selected(); }); }, self); self.selectedScript = ko.computed(function () { return self.selectedScripts()[0]; }, self); self.selectAll = function () { self.allSelected(! self.allSelected()); ko.utils.arrayForEach(self.scripts(), function (script) { script.selected(self.allSelected()); }); return true; }; self.getScriptById = function (id) { var _s = null; ko.utils.arrayForEach(self.scripts(), function (script) { if (script.id() == id) { _s = script; } }); return _s; } self.filterScripts = function (filter) { self.filteredScripts(ko.utils.arrayFilter(self.scripts(), function (script) { return script.isDesign() && script.name().toLowerCase().indexOf(filter.toLowerCase()) > -1 })); }; self.loadScript = function (id) { var _s = self.getScriptById(id); if (_s != null) { self.currentScript(_s); } else { self.currentScript(new SparkScript(_defaultScript)); } } self.confirmNewScript = function () { if (self.isDirty()) { showConfirmModal(); } else { self.newScript(); } }; self.confirmScript = function () { if (self.loadingScript != null){ self.viewScript(self.loadingScript); } else { self.newScript(); } }; self.newScript = function () { self.loadingScript = null; self.currentScript(new SparkScript(_defaultScript)); self.isDirty(false); $("#confirmModal").modal("hide"); $("#newScriptModal").modal("show"); $(document).trigger("loadEditor"); $(document).trigger("showEditor"); $(document).trigger("clearLogs"); }; self.createAndEditScript = function () { self.currentScript().script(LABELS.NEW_SCRIPT_CONTENT[self.currentScript().language()]); $(document).trigger("loadEditor"); $("#newScriptModal").modal("hide"); $(document).trigger("showEditor"); }; self.editScript = function (script) { $(document).trigger("showEditor"); }; self.editScriptProperties = function (script) { $(document).trigger("showProperties"); }; self.showScriptLogs = function (script) { $(document).trigger("showLogs"); }; self.confirmViewScript = function (script) { if (self.isDirty()) { self.loadingScript = script; showConfirmModal(); } else { self.viewScript(script); } }; self.viewScript = function (script) { self.loadingScript = null; self.currentScript(script); self.isDirty(false); $("#confirmModal").modal("hide"); $(document).trigger("loadEditor"); $(document).trigger("showEditor"); }; self.saveScript = function () { if (self.LABELS.NEW_SCRIPT_NAME == self.currentScript().name()){ showNameModal(); } else { $("#nameModal").modal("hide"); callSave(self.currentScript()); self.isDirty(false); } }; self.runScript = function () { callRun(self.currentScript()); }; self.copyScript = function () { callCopy(self.currentScript()); viewModel.isDirty(true); }; self.confirmDeleteScript = function () { self.currentDeleteType("single"); showDeleteModal(); }; self.stopScript = function () { callStop(self.currentScript()); }; self.listRunScript = function () { self.currentScript(self.selectedScript()); self.runOrShowSubmissionModal(); }; self.listCopyScript = function () { callCopy(self.selectedScript()); }; self.listConfirmDeleteScripts = function () { self.currentDeleteType("multiple"); showDeleteModal(); }; self.deleteScripts = function () { var ids = []; if (self.currentDeleteType() == "single") { ids.push(self.currentScript().id()); } if (self.currentDeleteType() == "multiple") { $(self.selectedScripts()).each(function (index, script) { ids.push(script.id()); }); } callDelete(ids); }; self.updateScripts = function () { $.getJSON(self.LIST_SCRIPTS, function (data) { self.scripts(ko.utils.arrayMap(data, function (script) { script.parentModel = self; return new SparkScript(script); })); self.filteredScripts(self.scripts()); $(document).trigger("scriptsRefreshed"); }); }; self.updateDashboard = function (workflows) { self.isDashboardLoaded = true; var koWorkflows = ko.utils.arrayMap(workflows, function (wf) { return new Workflow(wf); }); self.runningScripts(ko.utils.arrayFilter(koWorkflows, function (wf) { return wf.isRunning })); self.completedScripts(ko.utils.arrayFilter(koWorkflows, function (wf) { return !wf.isRunning })); } self.runOrShowSubmissionModal = function runOrShowSubmissionModal() { var script = self.currentScript(); if (! $.isEmptyObject(script.getParameters())) { self.submissionVariables.removeAll(); $.each(script.getParameters(), function (key, value) { self.submissionVariables.push({'name': key, 'value': value}); }); $("#runScriptBtn").button("reset"); $("#runScriptBtn").attr("data-loading-text", $("#runScriptBtn").text() + " ..."); $("#submitModal").modal({ keyboard: true, show: true }); } else { self.runScript(); } }; self.showStopModal = function showStopModal() { $("#stopScriptBtn").button("reset"); $("#stopScriptBtn").attr("data-loading-text", $("#stopScriptBtn").text() + " ..."); $("#stopModal").modal({ keyboard: true, show: true }); } self.showFileChooser = function showFileChooser() { var inputPath = this; var path = inputPath.value().substr(0, inputPath.value().lastIndexOf("/")); $("#filechooser").jHueFileChooser({ initialPath: path, onFileChoose: function (filePath) { inputPath.value(filePath); $("#chooseFile").modal("hide"); }, createFolder: false }); $("#chooseFile").modal("show"); }; function showDeleteModal() { $(".deleteMsg").addClass("hide"); if (self.currentDeleteType() == "single") { $(".deleteMsg.single").removeClass("hide"); } if (self.currentDeleteType() == "multiple") { if (self.selectedScripts().length > 1) { $(".deleteMsg.multiple").removeClass("hide"); } else { $(".deleteMsg.single").removeClass("hide"); } } $("#deleteModal").modal({ keyboard: true, show: true }); } function showStopModal() { $(".stopMsg").addClass("hide"); if (self.currentStopType() == "single") { $(".stopMsg.single").removeClass("hide"); } if (self.currentStopType() == "multiple") { if (self.selectedScripts().length > 1) { $(".stopMsg.multiple").removeClass("hide"); } else { $(".stopMsg.single").removeClass("hide"); } } $("#stopModal").modal({ keyboard: true, show: true }); } function showConfirmModal() { $("#confirmModal").modal({ keyboard: true, show: true }); } function showNameModal() { $("#nameModal").modal({ keyboard: true, show: true }); } function callSave(script) { $(document).trigger("saving"); $.post(self.SAVE_URL, { id: script.id(), name: script.name(), script: script.script(), parameters: ko.toJSON(script.parameters()), resources: ko.toJSON(script.resources()), hadoopProperties: ko.toJSON(script.hadoopProperties()), language: ko.toJSON(script.language()) }, function (data) { self.currentScript().id(data.id); $(document).trigger("saved"); self.updateScripts(); }, "json"); } function callRun(script) { self.currentScript(script); $(document).trigger("showLogs"); $(document).trigger("running"); $("#submitModal").modal("hide"); $.post(self.RUN_URL, { id: script.id(), name: script.name(), script: script.script(), parameters: ko.toJSON(script.parameters()), submissionVariables: ko.utils.stringifyJson(self.submissionVariables()), resources: ko.toJSON(script.resources()), hadoopProperties: ko.toJSON(script.hadoopProperties()), language: ko.toJSON(script.language()) }, function (data) { if (data.id && self.currentScript().id() != data.id){ script.id(data.id); $(document).trigger("loadEditor"); } script.isRunning(true); script.watchUrl(data.watchUrl); $(document).trigger("startLogsRefresh"); $(document).trigger("refreshDashboard"); self.updateScripts(); }, "json").fail( function(xhr, textStatus, errorThrown) { $(document).trigger("error", xhr.responseText); }); } function callStop(script) { $(document).trigger("stopping"); $.post(self.STOP_URL, { id: script.id() }, function (data) { $(document).trigger("stopped"); $("#stopModal").modal("hide"); }, "json"); } function callCopy(script) { $.post(self.COPY_URL, { id: script.id() }, function (data) { data.parentModel = self; self.currentScript(new SparkScript(data)); $(document).trigger("loadEditor"); self.updateScripts(); }, "json"); } function callDelete(ids) { if (ids.indexOf(self.currentScript().id()) > -1) { self.currentScript(new SparkScript(_defaultScript)); $(document).trigger("loadEditor"); } $.post(self.DELETE_URL, { ids: ids.join(",") }, function (data) { self.updateScripts(); $("#deleteModal").modal("hide"); viewModel.isDirty(false); }, "json"); } self.viewSubmittedScript = function (workflow) { self.loadScript(workflow.scriptId); self.currentScript().script(workflow.scriptContent); self.currentScript().isRunning(true); self.currentScript().watchUrl(workflow.watchUrl); $(document).trigger("loadEditor"); $(document).trigger("clearLogs"); $(document).trigger("startLogsRefresh"); $(document).trigger("showLogs"); }; self.showLogsInterval = -1; self.showLogsAtEnd = true; self.showLogs = function (workflow) { window.clearInterval(self.showLogsInterval); $("#logsModal pre").scroll(function () { self.showLogsAtEnd = $(this).scrollTop() + $(this).height() + 20 >= $(this)[0].scrollHeight; }); if (workflow.isRunning) { $("#logsModal img").removeClass("hide"); $("#logsModal pre").addClass("hide"); $("#logsModal").modal({ keyboard: true, show: true }); $("#logsModal").on("hide", function () { window.clearInterval(self.showLogsInterval); }); self.showLogsInterval = window.setInterval(function () { $.getJSON(workflow.watchUrl, function (data) { if (data.workflow && !data.workflow.isRunning) { window.clearInterval(self.showLogsInterval); } if (data.logs.spark) { $("#logsModal img").addClass("hide"); $("#logsModal pre").removeClass("hide"); var _logsEl = $("#logsModal pre"); var _newLinesCount = _logsEl.html() == "" ? 0 : _logsEl.html().split("
").length; var newLines = data.logs.spark.split("\n").slice(_newLinesCount); if (newLines.length > 0){ _logsEl.html(_logsEl.html() + newLines.join("
") + "
"); } if (self.showLogsAtEnd) { _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height()); } } }); }, 1000); } }; };