spark.ko.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. var Resource = function (resource) {
  17. var self = this;
  18. self.type = ko.observable(resource.type);
  19. self.value = ko.observable(resource.value);
  20. };
  21. var HadoopProperty = function (property) {
  22. var self = this;
  23. self.name = ko.observable(property.name);
  24. self.value = ko.observable(property.value);
  25. };
  26. var PigParameter = HadoopProperty;
  27. var SparkScript = function (sparkScript) {
  28. var self = this;
  29. self.id = ko.observable(sparkScript.id);
  30. self.isDesign = ko.observable(sparkScript.isDesign);
  31. self.name = ko.observable(sparkScript.name);
  32. self.language = ko.observable(sparkScript.language);
  33. self.script = ko.observable(sparkScript.script);
  34. self.scriptSumup = ko.observable(sparkScript.script.replace(/\W+/g, ' ').substring(0, 100));
  35. self.isRunning = ko.observable(false);
  36. self.selected = ko.observable(false);
  37. self.watchUrl = ko.observable("");
  38. self.actions = ko.observableArray([]);
  39. self.handleSelect = function (row, e) {
  40. this.selected(!this.selected());
  41. };
  42. self.hovered = ko.observable(false);
  43. self.toggleHover = function (row, e) {
  44. this.hovered(!this.hovered());
  45. };
  46. self.parameters = ko.observableArray([]);
  47. ko.utils.arrayForEach(sparkScript.parameters, function (parameter) {
  48. self.parameters.push(new PigParameter({name: parameter.name, value: parameter.value}));
  49. });
  50. self.addParameter = function () {
  51. self.parameters.push(new PigParameter({name: '', value: ''}));
  52. self.updateParentModel();
  53. };
  54. self.removeParameter = function () {
  55. self.parameters.remove(this);
  56. self.updateParentModel();
  57. };
  58. self.getParameters = function () {
  59. var params = {};
  60. return params;
  61. };
  62. self.hadoopProperties = ko.observableArray([]);
  63. ko.utils.arrayForEach(sparkScript.hadoopProperties, function (property) {
  64. self.hadoopProperties.push(new HadoopProperty({name: property.name, value: property.value}));
  65. });
  66. self.addHadoopProperties = function () {
  67. self.hadoopProperties.push(new HadoopProperty({name: '', value: ''}));
  68. self.updateParentModel();
  69. };
  70. self.removeHadoopProperties = function () {
  71. self.hadoopProperties.remove(this);
  72. self.updateParentModel();
  73. };
  74. self.resources = ko.observableArray([]);
  75. ko.utils.arrayForEach(sparkScript.resources, function (resource) {
  76. self.resources.push(new Resource({type: resource.type, value: resource.value}));
  77. });
  78. self.addResource = function () {
  79. self.resources.push(new Resource({type: 'file', value: ''}));
  80. self.updateParentModel();
  81. };
  82. self.removeResource = function () {
  83. self.resources.remove(this);
  84. self.updateParentModel();
  85. };
  86. self.parentModel = sparkScript.parentModel;
  87. self.updateParentModel = function () {
  88. if (typeof self.parentModel != "undefined" && self.parentModel != null) {
  89. self.parentModel.isDirty(true);
  90. }
  91. }
  92. self.name.subscribe(function (name) {
  93. self.updateParentModel();
  94. });
  95. self.language.subscribe(function (lang) {
  96. if (typeof codeMirror != "undefined"){
  97. codeMirror.setOption("mode", "text/x-" + lang);
  98. }
  99. });
  100. }
  101. var Workflow = function (wf) {
  102. return {
  103. id: wf.id,
  104. scriptId: wf.scriptId,
  105. scriptContent: wf.scriptContent,
  106. lastModTime: wf.lastModTime,
  107. endTime: wf.endTime,
  108. status: wf.status,
  109. statusClass: "label " + getStatusClass(wf.status),
  110. isRunning: wf.isRunning,
  111. duration: wf.duration,
  112. appName: wf.appName,
  113. progress: wf.progress,
  114. progressPercent: wf.progressPercent,
  115. progressClass: "bar " + getStatusClass(wf.status, "bar-"),
  116. user: wf.user,
  117. absoluteUrl: wf.absoluteUrl,
  118. watchUrl: wf.watchUrl,
  119. canEdit: wf.canEdit,
  120. killUrl: wf.killUrl,
  121. created: wf.created,
  122. run: wf.run
  123. }
  124. }
  125. var SparkViewModel = function (props) {
  126. var self = this;
  127. self.LABELS = props.labels;
  128. self.LIST_SCRIPTS = props.listScripts;
  129. self.SAVE_URL = props.saveUrl;
  130. self.RUN_URL = props.runUrl;
  131. self.STOP_URL = props.stopUrl;
  132. self.COPY_URL = props.copyUrl;
  133. self.DELETE_URL = props.deleteUrl;
  134. self.isLoading = ko.observable(false);
  135. self.allSelected = ko.observable(false);
  136. self.submissionVariables = ko.observableArray([]);
  137. self.scripts = ko.observableArray([]);
  138. self.filteredScripts = ko.observableArray(self.scripts());
  139. self.runningScripts = ko.observableArray([]);
  140. self.completedScripts = ko.observableArray([]);
  141. self.isDashboardLoaded = false;
  142. self.isDirty = ko.observable(false);
  143. var _defaultScript = {
  144. id: -1,
  145. name: self.LABELS.NEW_SCRIPT_NAME,
  146. language: 'python',
  147. script: self.LABELS.NEW_SCRIPT_CONTENT['python'],
  148. parameters: self.LABELS.NEW_SCRIPT_PARAMETERS,
  149. resources: self.LABELS.NEW_SCRIPT_RESOURCES,
  150. hadoopProperties: self.LABELS.NEW_SCRIPT_HADOOP_PROPERTIES,
  151. parentModel: self
  152. };
  153. self.currentScript = ko.observable(new SparkScript(_defaultScript));
  154. self.loadingScript = null;
  155. self.currentDeleteType = ko.observable("");
  156. self.selectedScripts = ko.computed(function () {
  157. return ko.utils.arrayFilter(self.scripts(), function (script) {
  158. return script.selected();
  159. });
  160. }, self);
  161. self.selectedScript = ko.computed(function () {
  162. return self.selectedScripts()[0];
  163. }, self);
  164. self.selectAll = function () {
  165. self.allSelected(! self.allSelected());
  166. ko.utils.arrayForEach(self.scripts(), function (script) {
  167. script.selected(self.allSelected());
  168. });
  169. return true;
  170. };
  171. self.getScriptById = function (id) {
  172. var _s = null;
  173. ko.utils.arrayForEach(self.scripts(), function (script) {
  174. if (script.id() == id) {
  175. _s = script;
  176. }
  177. });
  178. return _s;
  179. }
  180. self.filterScripts = function (filter) {
  181. self.filteredScripts(ko.utils.arrayFilter(self.scripts(), function (script) {
  182. return script.isDesign() && script.name().toLowerCase().indexOf(filter.toLowerCase()) > -1
  183. }));
  184. };
  185. self.loadScript = function (id) {
  186. var _s = self.getScriptById(id);
  187. if (_s != null) {
  188. self.currentScript(_s);
  189. }
  190. else {
  191. self.currentScript(new SparkScript(_defaultScript));
  192. }
  193. }
  194. self.confirmNewScript = function () {
  195. if (self.isDirty()) {
  196. showConfirmModal();
  197. }
  198. else {
  199. self.newScript();
  200. }
  201. };
  202. self.confirmScript = function () {
  203. if (self.loadingScript != null){
  204. self.viewScript(self.loadingScript);
  205. }
  206. else {
  207. self.newScript();
  208. }
  209. };
  210. self.newScript = function () {
  211. self.loadingScript = null;
  212. self.currentScript(new SparkScript(_defaultScript));
  213. self.isDirty(false);
  214. $("#confirmModal").modal("hide");
  215. $("#newScriptModal").modal("show");
  216. $(document).trigger("loadEditor");
  217. $(document).trigger("showEditor");
  218. $(document).trigger("clearLogs");
  219. };
  220. self.createAndEditScript = function () {
  221. self.currentScript().script(LABELS.NEW_SCRIPT_CONTENT[self.currentScript().language()]);
  222. $(document).trigger("loadEditor");
  223. $("#newScriptModal").modal("hide");
  224. $(document).trigger("showEditor");
  225. };
  226. self.editScript = function (script) {
  227. $(document).trigger("showEditor");
  228. };
  229. self.editScriptProperties = function (script) {
  230. $(document).trigger("showProperties");
  231. };
  232. self.showScriptLogs = function (script) {
  233. $(document).trigger("showLogs");
  234. };
  235. self.confirmViewScript = function (script) {
  236. if (self.isDirty()) {
  237. self.loadingScript = script;
  238. showConfirmModal();
  239. }
  240. else {
  241. self.viewScript(script);
  242. }
  243. };
  244. self.viewScript = function (script) {
  245. self.loadingScript = null;
  246. self.currentScript(script);
  247. self.isDirty(false);
  248. $("#confirmModal").modal("hide");
  249. $(document).trigger("loadEditor");
  250. $(document).trigger("showEditor");
  251. };
  252. self.saveScript = function () {
  253. if (self.LABELS.NEW_SCRIPT_NAME == self.currentScript().name()){
  254. showNameModal();
  255. }
  256. else {
  257. $("#nameModal").modal("hide");
  258. callSave(self.currentScript());
  259. self.isDirty(false);
  260. }
  261. };
  262. self.runScript = function () {
  263. callRun(self.currentScript());
  264. };
  265. self.copyScript = function () {
  266. callCopy(self.currentScript());
  267. viewModel.isDirty(true);
  268. };
  269. self.confirmDeleteScript = function () {
  270. self.currentDeleteType("single");
  271. showDeleteModal();
  272. };
  273. self.stopScript = function () {
  274. callStop(self.currentScript());
  275. };
  276. self.listRunScript = function () {
  277. self.currentScript(self.selectedScript());
  278. self.runOrShowSubmissionModal();
  279. };
  280. self.listCopyScript = function () {
  281. callCopy(self.selectedScript());
  282. };
  283. self.listConfirmDeleteScripts = function () {
  284. self.currentDeleteType("multiple");
  285. showDeleteModal();
  286. };
  287. self.deleteScripts = function () {
  288. var ids = [];
  289. if (self.currentDeleteType() == "single") {
  290. ids.push(self.currentScript().id());
  291. }
  292. if (self.currentDeleteType() == "multiple") {
  293. $(self.selectedScripts()).each(function (index, script) {
  294. ids.push(script.id());
  295. });
  296. }
  297. callDelete(ids);
  298. };
  299. self.updateScripts = function () {
  300. $.getJSON(self.LIST_SCRIPTS, function (data) {
  301. self.scripts(ko.utils.arrayMap(data, function (script) {
  302. script.parentModel = self;
  303. return new SparkScript(script);
  304. }));
  305. self.filteredScripts(self.scripts());
  306. $(document).trigger("scriptsRefreshed");
  307. });
  308. };
  309. self.updateDashboard = function (workflows) {
  310. self.isDashboardLoaded = true;
  311. var koWorkflows = ko.utils.arrayMap(workflows, function (wf) {
  312. return new Workflow(wf);
  313. });
  314. self.runningScripts(ko.utils.arrayFilter(koWorkflows, function (wf) {
  315. return wf.isRunning
  316. }));
  317. self.completedScripts(ko.utils.arrayFilter(koWorkflows, function (wf) {
  318. return !wf.isRunning
  319. }));
  320. }
  321. self.runOrShowSubmissionModal = function runOrShowSubmissionModal() {
  322. var script = self.currentScript();
  323. if (! $.isEmptyObject(script.getParameters())) {
  324. self.submissionVariables.removeAll();
  325. $.each(script.getParameters(), function (key, value) {
  326. self.submissionVariables.push({'name': key, 'value': value});
  327. });
  328. $("#runScriptBtn").button("reset");
  329. $("#runScriptBtn").attr("data-loading-text", $("#runScriptBtn").text() + " ...");
  330. $("#submitModal").modal({
  331. keyboard: true,
  332. show: true
  333. });
  334. } else {
  335. self.runScript();
  336. }
  337. };
  338. self.showStopModal = function showStopModal() {
  339. $("#stopScriptBtn").button("reset");
  340. $("#stopScriptBtn").attr("data-loading-text", $("#stopScriptBtn").text() + " ...");
  341. $("#stopModal").modal({
  342. keyboard: true,
  343. show: true
  344. });
  345. }
  346. self.showFileChooser = function showFileChooser() {
  347. var inputPath = this;
  348. var path = inputPath.value().substr(0, inputPath.value().lastIndexOf("/"));
  349. $("#filechooser").jHueFileChooser({
  350. initialPath: path,
  351. onFileChoose: function (filePath) {
  352. inputPath.value(filePath);
  353. $("#chooseFile").modal("hide");
  354. },
  355. createFolder: false
  356. });
  357. $("#chooseFile").modal("show");
  358. };
  359. function showDeleteModal() {
  360. $(".deleteMsg").addClass("hide");
  361. if (self.currentDeleteType() == "single") {
  362. $(".deleteMsg.single").removeClass("hide");
  363. }
  364. if (self.currentDeleteType() == "multiple") {
  365. if (self.selectedScripts().length > 1) {
  366. $(".deleteMsg.multiple").removeClass("hide");
  367. }
  368. else {
  369. $(".deleteMsg.single").removeClass("hide");
  370. }
  371. }
  372. $("#deleteModal").modal({
  373. keyboard: true,
  374. show: true
  375. });
  376. }
  377. function showStopModal() {
  378. $(".stopMsg").addClass("hide");
  379. if (self.currentStopType() == "single") {
  380. $(".stopMsg.single").removeClass("hide");
  381. }
  382. if (self.currentStopType() == "multiple") {
  383. if (self.selectedScripts().length > 1) {
  384. $(".stopMsg.multiple").removeClass("hide");
  385. } else {
  386. $(".stopMsg.single").removeClass("hide");
  387. }
  388. }
  389. $("#stopModal").modal({
  390. keyboard: true,
  391. show: true
  392. });
  393. }
  394. function showConfirmModal() {
  395. $("#confirmModal").modal({
  396. keyboard: true,
  397. show: true
  398. });
  399. }
  400. function showNameModal() {
  401. $("#nameModal").modal({
  402. keyboard: true,
  403. show: true
  404. });
  405. }
  406. function callSave(script) {
  407. $(document).trigger("saving");
  408. $.post(self.SAVE_URL,
  409. {
  410. id: script.id(),
  411. name: script.name(),
  412. script: script.script(),
  413. parameters: ko.toJSON(script.parameters()),
  414. resources: ko.toJSON(script.resources()),
  415. hadoopProperties: ko.toJSON(script.hadoopProperties()),
  416. language: ko.toJSON(script.language())
  417. },
  418. function (data) {
  419. self.currentScript().id(data.id);
  420. $(document).trigger("saved");
  421. self.updateScripts();
  422. }, "json");
  423. }
  424. function callRun(script) {
  425. self.currentScript(script);
  426. $(document).trigger("showLogs");
  427. $(document).trigger("running");
  428. $("#submitModal").modal("hide");
  429. $.post(self.RUN_URL,
  430. {
  431. id: script.id(),
  432. name: script.name(),
  433. script: script.script(),
  434. parameters: ko.toJSON(script.parameters()),
  435. submissionVariables: ko.utils.stringifyJson(self.submissionVariables()),
  436. resources: ko.toJSON(script.resources()),
  437. hadoopProperties: ko.toJSON(script.hadoopProperties()),
  438. language: ko.toJSON(script.language())
  439. },
  440. function (data) {
  441. if (data.id && self.currentScript().id() != data.id){
  442. script.id(data.id);
  443. $(document).trigger("loadEditor");
  444. }
  445. script.isRunning(true);
  446. script.watchUrl(data.watchUrl);
  447. $(document).trigger("startLogsRefresh");
  448. $(document).trigger("refreshDashboard");
  449. self.updateScripts();
  450. }, "json").fail( function(xhr, textStatus, errorThrown) {
  451. $(document).trigger("error", xhr.responseText);
  452. });
  453. }
  454. function callStop(script) {
  455. $(document).trigger("stopping");
  456. $.post(self.STOP_URL, {
  457. id: script.id()
  458. },
  459. function (data) {
  460. $(document).trigger("stopped");
  461. $("#stopModal").modal("hide");
  462. }, "json");
  463. }
  464. function callCopy(script) {
  465. $.post(self.COPY_URL,
  466. {
  467. id: script.id()
  468. },
  469. function (data) {
  470. data.parentModel = self;
  471. self.currentScript(new SparkScript(data));
  472. $(document).trigger("loadEditor");
  473. self.updateScripts();
  474. }, "json");
  475. }
  476. function callDelete(ids) {
  477. if (ids.indexOf(self.currentScript().id()) > -1) {
  478. self.currentScript(new SparkScript(_defaultScript));
  479. $(document).trigger("loadEditor");
  480. }
  481. $.post(self.DELETE_URL, {
  482. ids: ids.join(",")
  483. },
  484. function (data) {
  485. self.updateScripts();
  486. $("#deleteModal").modal("hide");
  487. viewModel.isDirty(false);
  488. }, "json");
  489. }
  490. self.viewSubmittedScript = function (workflow) {
  491. self.loadScript(workflow.scriptId);
  492. self.currentScript().script(workflow.scriptContent);
  493. self.currentScript().isRunning(true);
  494. self.currentScript().watchUrl(workflow.watchUrl);
  495. $(document).trigger("loadEditor");
  496. $(document).trigger("clearLogs");
  497. $(document).trigger("startLogsRefresh");
  498. $(document).trigger("showLogs");
  499. };
  500. self.showLogsInterval = -1;
  501. self.showLogsAtEnd = true;
  502. self.showLogs = function (workflow) {
  503. window.clearInterval(self.showLogsInterval);
  504. $("#logsModal pre").scroll(function () {
  505. self.showLogsAtEnd = $(this).scrollTop() + $(this).height() + 20 >= $(this)[0].scrollHeight;
  506. });
  507. if (workflow.isRunning) {
  508. $("#logsModal img").removeClass("hide");
  509. $("#logsModal pre").addClass("hide");
  510. $("#logsModal").modal({
  511. keyboard: true,
  512. show: true
  513. });
  514. $("#logsModal").on("hide", function () {
  515. window.clearInterval(self.showLogsInterval);
  516. });
  517. self.showLogsInterval = window.setInterval(function () {
  518. $.getJSON(workflow.watchUrl, function (data) {
  519. if (data.workflow && !data.workflow.isRunning) {
  520. window.clearInterval(self.showLogsInterval);
  521. }
  522. if (data.logs.spark) {
  523. $("#logsModal img").addClass("hide");
  524. $("#logsModal pre").removeClass("hide");
  525. var _logsEl = $("#logsModal pre");
  526. var _newLinesCount = _logsEl.html() == "" ? 0 : _logsEl.html().split("<br>").length;
  527. var newLines = data.logs.spark.split("\n").slice(_newLinesCount);
  528. if (newLines.length > 0){
  529. _logsEl.html(_logsEl.html() + newLines.join("<br>") + "<br>");
  530. }
  531. if (self.showLogsAtEnd) {
  532. _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height());
  533. }
  534. }
  535. });
  536. }, 1000);
  537. }
  538. };
  539. };