spark.ko.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600
  1. // Licensed to Cloudera, Inc. under one
  2. // or more contributor license agreements. See the NOTICE file
  3. // distributed with this work for additional information
  4. // regarding copyright ownership. Cloudera, Inc. licenses this file
  5. // to you under the Apache License, Version 2.0 (the
  6. // "License"); you may not use this file except in compliance
  7. // with the License. You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. var Resource = function (resource) {
  17. var self = this;
  18. self.type = ko.observable(resource.type);
  19. self.value = ko.observable(resource.value);
  20. };
  21. var HadoopProperty = function (property) {
  22. var self = this;
  23. self.name = ko.observable(property.name);
  24. self.value = ko.observable(property.value);
  25. };
  26. var PigParameter = HadoopProperty;
  27. var SparkScript = function (sparkScript) {
  28. var self = this;
  29. self.id = ko.observable(sparkScript.id);
  30. self.isDesign = ko.observable(sparkScript.isDesign);
  31. self.name = ko.observable(sparkScript.name);
  32. self.language = ko.observable(sparkScript.language);
  33. self.script = ko.observable(sparkScript.script);
  34. self.scriptSumup = ko.observable(sparkScript.script.replace(/\W+/g, ' ').substring(0, 100));
  35. self.isRunning = ko.observable(false);
  36. self.selected = ko.observable(false);
  37. self.watchUrl = ko.observable("");
  38. self.actions = ko.observableArray([]);
  39. self.handleSelect = function (row, e) {
  40. this.selected(!this.selected());
  41. };
  42. self.hovered = ko.observable(false);
  43. self.toggleHover = function (row, e) {
  44. this.hovered(!this.hovered());
  45. };
  46. self.parameters = ko.observableArray([]);
  47. ko.utils.arrayForEach(sparkScript.parameters, function (parameter) {
  48. self.parameters.push(new PigParameter({name: parameter.name, value: parameter.value}));
  49. });
  50. self.addParameter = function () {
  51. self.parameters.push(new PigParameter({name: '', value: ''}));
  52. self.updateParentModel();
  53. };
  54. self.removeParameter = function () {
  55. self.parameters.remove(this);
  56. self.updateParentModel();
  57. };
  58. self.getParameters = function () {
  59. var params = {};
  60. return params;
  61. };
  62. self.hadoopProperties = ko.observableArray([]);
  63. ko.utils.arrayForEach(sparkScript.hadoopProperties, function (property) {
  64. self.hadoopProperties.push(new HadoopProperty({name: property.name, value: property.value}));
  65. });
  66. self.addHadoopProperties = function () {
  67. self.hadoopProperties.push(new HadoopProperty({name: '', value: ''}));
  68. self.updateParentModel();
  69. };
  70. self.removeHadoopProperties = function () {
  71. self.hadoopProperties.remove(this);
  72. self.updateParentModel();
  73. };
  74. self.resources = ko.observableArray([]);
  75. ko.utils.arrayForEach(sparkScript.resources, function (resource) {
  76. self.resources.push(new Resource({type: resource.type, value: resource.value}));
  77. });
  78. self.addResource = function () {
  79. self.resources.push(new Resource({type: 'file', value: ''}));
  80. self.updateParentModel();
  81. };
  82. self.removeResource = function () {
  83. self.resources.remove(this);
  84. self.updateParentModel();
  85. };
  86. self.parentModel = sparkScript.parentModel;
  87. self.updateParentModel = function () {
  88. if (typeof self.parentModel != "undefined" && self.parentModel != null) {
  89. self.parentModel.isDirty(true);
  90. }
  91. }
  92. self.name.subscribe(function (name) {
  93. self.updateParentModel();
  94. });
  95. }
  96. var Workflow = function (wf) {
  97. return {
  98. id: wf.id,
  99. scriptId: wf.scriptId,
  100. scriptContent: wf.scriptContent,
  101. lastModTime: wf.lastModTime,
  102. endTime: wf.endTime,
  103. status: wf.status,
  104. statusClass: "label " + getStatusClass(wf.status),
  105. isRunning: wf.isRunning,
  106. duration: wf.duration,
  107. appName: wf.appName,
  108. progress: wf.progress,
  109. progressPercent: wf.progressPercent,
  110. progressClass: "bar " + getStatusClass(wf.status, "bar-"),
  111. user: wf.user,
  112. absoluteUrl: wf.absoluteUrl,
  113. watchUrl: wf.watchUrl,
  114. canEdit: wf.canEdit,
  115. killUrl: wf.killUrl,
  116. created: wf.created,
  117. run: wf.run
  118. }
  119. }
  120. var SparkViewModel = function (props) {
  121. var self = this;
  122. self.LABELS = props.labels;
  123. self.LIST_SCRIPTS = props.listScripts;
  124. self.SAVE_URL = props.saveUrl;
  125. self.RUN_URL = props.runUrl;
  126. self.STOP_URL = props.stopUrl;
  127. self.COPY_URL = props.copyUrl;
  128. self.DELETE_URL = props.deleteUrl;
  129. self.isLoading = ko.observable(false);
  130. self.allSelected = ko.observable(false);
  131. self.submissionVariables = ko.observableArray([]);
  132. self.scripts = ko.observableArray([]);
  133. self.filteredScripts = ko.observableArray(self.scripts());
  134. self.runningScripts = ko.observableArray([]);
  135. self.completedScripts = ko.observableArray([]);
  136. self.isDashboardLoaded = false;
  137. self.isDirty = ko.observable(false);
  138. var _defaultScript = {
  139. id: -1,
  140. name: self.LABELS.NEW_SCRIPT_NAME,
  141. language: 'python',
  142. script: self.LABELS.NEW_SCRIPT_CONTENT['python'],
  143. parameters: self.LABELS.NEW_SCRIPT_PARAMETERS,
  144. resources: self.LABELS.NEW_SCRIPT_RESOURCES,
  145. hadoopProperties: self.LABELS.NEW_SCRIPT_HADOOP_PROPERTIES,
  146. parentModel: self
  147. };
  148. self.currentScript = ko.observable(new SparkScript(_defaultScript));
  149. self.loadingScript = null;
  150. self.currentDeleteType = ko.observable("");
  151. self.selectedScripts = ko.computed(function () {
  152. return ko.utils.arrayFilter(self.scripts(), function (script) {
  153. return script.selected();
  154. });
  155. }, self);
  156. self.selectedScript = ko.computed(function () {
  157. return self.selectedScripts()[0];
  158. }, self);
  159. self.selectAll = function () {
  160. self.allSelected(! self.allSelected());
  161. ko.utils.arrayForEach(self.scripts(), function (script) {
  162. script.selected(self.allSelected());
  163. });
  164. return true;
  165. };
  166. self.getScriptById = function (id) {
  167. var _s = null;
  168. ko.utils.arrayForEach(self.scripts(), function (script) {
  169. if (script.id() == id) {
  170. _s = script;
  171. }
  172. });
  173. return _s;
  174. }
  175. self.filterScripts = function (filter) {
  176. self.filteredScripts(ko.utils.arrayFilter(self.scripts(), function (script) {
  177. return script.isDesign() && script.name().toLowerCase().indexOf(filter.toLowerCase()) > -1
  178. }));
  179. };
  180. self.loadScript = function (id) {
  181. var _s = self.getScriptById(id);
  182. if (_s != null) {
  183. self.currentScript(_s);
  184. }
  185. else {
  186. self.currentScript(new SparkScript(_defaultScript));
  187. }
  188. }
  189. self.confirmNewScript = function () {
  190. if (self.isDirty()) {
  191. showConfirmModal();
  192. }
  193. else {
  194. self.newScript();
  195. }
  196. };
  197. self.confirmScript = function () {
  198. if (self.loadingScript != null){
  199. self.viewScript(self.loadingScript);
  200. }
  201. else {
  202. self.newScript();
  203. }
  204. };
  205. self.newScript = function () {
  206. self.loadingScript = null;
  207. self.currentScript(new SparkScript(_defaultScript));
  208. self.isDirty(false);
  209. $("#confirmModal").modal("hide");
  210. $("#newScriptModal").modal("show");
  211. $(document).trigger("loadEditor");
  212. $(document).trigger("showEditor");
  213. $(document).trigger("clearLogs");
  214. };
  215. self.createAndEditScript = function () {
  216. self.currentScript().script(LABELS.NEW_SCRIPT_CONTENT[self.currentScript().language()]);
  217. $(document).trigger("loadEditor");
  218. $("#newScriptModal").modal("hide");
  219. $(document).trigger("showEditor");
  220. };
  221. self.editScript = function (script) {
  222. $(document).trigger("showEditor");
  223. };
  224. self.editScriptProperties = function (script) {
  225. $(document).trigger("showProperties");
  226. };
  227. self.showScriptLogs = function (script) {
  228. $(document).trigger("showLogs");
  229. };
  230. self.confirmViewScript = function (script) {
  231. if (self.isDirty()) {
  232. self.loadingScript = script;
  233. showConfirmModal();
  234. }
  235. else {
  236. self.viewScript(script);
  237. }
  238. };
  239. self.viewScript = function (script) {
  240. self.loadingScript = null;
  241. self.currentScript(script);
  242. self.isDirty(false);
  243. $("#confirmModal").modal("hide");
  244. $(document).trigger("loadEditor");
  245. $(document).trigger("showEditor");
  246. };
  247. self.saveScript = function () {
  248. if (self.LABELS.NEW_SCRIPT_NAME == self.currentScript().name()){
  249. showNameModal();
  250. }
  251. else {
  252. $("#nameModal").modal("hide");
  253. callSave(self.currentScript());
  254. self.isDirty(false);
  255. }
  256. };
  257. self.runScript = function () {
  258. callRun(self.currentScript());
  259. };
  260. self.copyScript = function () {
  261. callCopy(self.currentScript());
  262. viewModel.isDirty(true);
  263. };
  264. self.confirmDeleteScript = function () {
  265. self.currentDeleteType("single");
  266. showDeleteModal();
  267. };
  268. self.stopScript = function () {
  269. callStop(self.currentScript());
  270. };
  271. self.listRunScript = function () {
  272. self.currentScript(self.selectedScript());
  273. self.runOrShowSubmissionModal();
  274. };
  275. self.listCopyScript = function () {
  276. callCopy(self.selectedScript());
  277. };
  278. self.listConfirmDeleteScripts = function () {
  279. self.currentDeleteType("multiple");
  280. showDeleteModal();
  281. };
  282. self.deleteScripts = function () {
  283. var ids = [];
  284. if (self.currentDeleteType() == "single") {
  285. ids.push(self.currentScript().id());
  286. }
  287. if (self.currentDeleteType() == "multiple") {
  288. $(self.selectedScripts()).each(function (index, script) {
  289. ids.push(script.id());
  290. });
  291. }
  292. callDelete(ids);
  293. };
  294. self.updateScripts = function () {
  295. $.getJSON(self.LIST_SCRIPTS, function (data) {
  296. self.scripts(ko.utils.arrayMap(data, function (script) {
  297. script.parentModel = self;
  298. return new SparkScript(script);
  299. }));
  300. self.filteredScripts(self.scripts());
  301. $(document).trigger("scriptsRefreshed");
  302. });
  303. };
  304. self.updateDashboard = function (workflows) {
  305. self.isDashboardLoaded = true;
  306. var koWorkflows = ko.utils.arrayMap(workflows, function (wf) {
  307. return new Workflow(wf);
  308. });
  309. self.runningScripts(ko.utils.arrayFilter(koWorkflows, function (wf) {
  310. return wf.isRunning
  311. }));
  312. self.completedScripts(ko.utils.arrayFilter(koWorkflows, function (wf) {
  313. return !wf.isRunning
  314. }));
  315. }
  316. self.runOrShowSubmissionModal = function runOrShowSubmissionModal() {
  317. var script = self.currentScript();
  318. if (! $.isEmptyObject(script.getParameters())) {
  319. self.submissionVariables.removeAll();
  320. $.each(script.getParameters(), function (key, value) {
  321. self.submissionVariables.push({'name': key, 'value': value});
  322. });
  323. $("#runScriptBtn").button("reset");
  324. $("#runScriptBtn").attr("data-loading-text", $("#runScriptBtn").text() + " ...");
  325. $("#submitModal").modal({
  326. keyboard: true,
  327. show: true
  328. });
  329. } else {
  330. self.runScript();
  331. }
  332. };
  333. self.showStopModal = function showStopModal() {
  334. $("#stopScriptBtn").button("reset");
  335. $("#stopScriptBtn").attr("data-loading-text", $("#stopScriptBtn").text() + " ...");
  336. $("#stopModal").modal({
  337. keyboard: true,
  338. show: true
  339. });
  340. }
  341. self.showFileChooser = function showFileChooser() {
  342. var inputPath = this;
  343. var path = inputPath.value().substr(0, inputPath.value().lastIndexOf("/"));
  344. $("#filechooser").jHueFileChooser({
  345. initialPath: path,
  346. onFileChoose: function (filePath) {
  347. inputPath.value(filePath);
  348. $("#chooseFile").modal("hide");
  349. },
  350. createFolder: false
  351. });
  352. $("#chooseFile").modal("show");
  353. };
  354. function showDeleteModal() {
  355. $(".deleteMsg").addClass("hide");
  356. if (self.currentDeleteType() == "single") {
  357. $(".deleteMsg.single").removeClass("hide");
  358. }
  359. if (self.currentDeleteType() == "multiple") {
  360. if (self.selectedScripts().length > 1) {
  361. $(".deleteMsg.multiple").removeClass("hide");
  362. }
  363. else {
  364. $(".deleteMsg.single").removeClass("hide");
  365. }
  366. }
  367. $("#deleteModal").modal({
  368. keyboard: true,
  369. show: true
  370. });
  371. }
  372. function showStopModal() {
  373. $(".stopMsg").addClass("hide");
  374. if (self.currentStopType() == "single") {
  375. $(".stopMsg.single").removeClass("hide");
  376. }
  377. if (self.currentStopType() == "multiple") {
  378. if (self.selectedScripts().length > 1) {
  379. $(".stopMsg.multiple").removeClass("hide");
  380. } else {
  381. $(".stopMsg.single").removeClass("hide");
  382. }
  383. }
  384. $("#stopModal").modal({
  385. keyboard: true,
  386. show: true
  387. });
  388. }
  389. function showConfirmModal() {
  390. $("#confirmModal").modal({
  391. keyboard: true,
  392. show: true
  393. });
  394. }
  395. function showNameModal() {
  396. $("#nameModal").modal({
  397. keyboard: true,
  398. show: true
  399. });
  400. }
  401. function callSave(script) {
  402. $(document).trigger("saving");
  403. $.post(self.SAVE_URL,
  404. {
  405. id: script.id(),
  406. name: script.name(),
  407. script: script.script(),
  408. parameters: ko.toJSON(script.parameters()),
  409. resources: ko.toJSON(script.resources()),
  410. hadoopProperties: ko.toJSON(script.hadoopProperties()),
  411. language: ko.toJSON(script.language())
  412. },
  413. function (data) {
  414. self.currentScript().id(data.id);
  415. $(document).trigger("saved");
  416. self.updateScripts();
  417. }, "json");
  418. }
  419. function callRun(script) {
  420. self.currentScript(script);
  421. $(document).trigger("showLogs");
  422. $(document).trigger("running");
  423. $("#submitModal").modal("hide");
  424. $.post(self.RUN_URL,
  425. {
  426. id: script.id(),
  427. name: script.name(),
  428. script: script.script(),
  429. parameters: ko.toJSON(script.parameters()),
  430. submissionVariables: ko.utils.stringifyJson(self.submissionVariables()),
  431. resources: ko.toJSON(script.resources()),
  432. hadoopProperties: ko.toJSON(script.hadoopProperties()),
  433. language: ko.toJSON(script.language())
  434. },
  435. function (data) {
  436. if (data.id && self.currentScript().id() != data.id){
  437. script.id(data.id);
  438. $(document).trigger("loadEditor");
  439. }
  440. script.isRunning(true);
  441. script.watchUrl(data.watchUrl);
  442. $(document).trigger("startLogsRefresh");
  443. $(document).trigger("refreshDashboard");
  444. self.updateScripts();
  445. }, "json").fail( function(xhr, textStatus, errorThrown) {
  446. $(document).trigger("error", xhr.responseText);
  447. });
  448. }
  449. function callStop(script) {
  450. $(document).trigger("stopping");
  451. $.post(self.STOP_URL, {
  452. id: script.id()
  453. },
  454. function (data) {
  455. $(document).trigger("stopped");
  456. $("#stopModal").modal("hide");
  457. }, "json");
  458. }
  459. function callCopy(script) {
  460. $.post(self.COPY_URL,
  461. {
  462. id: script.id()
  463. },
  464. function (data) {
  465. data.parentModel = self;
  466. self.currentScript(new SparkScript(data));
  467. $(document).trigger("loadEditor");
  468. self.updateScripts();
  469. }, "json");
  470. }
  471. function callDelete(ids) {
  472. if (ids.indexOf(self.currentScript().id()) > -1) {
  473. self.currentScript(new SparkScript(_defaultScript));
  474. $(document).trigger("loadEditor");
  475. }
  476. $.post(self.DELETE_URL, {
  477. ids: ids.join(",")
  478. },
  479. function (data) {
  480. self.updateScripts();
  481. $("#deleteModal").modal("hide");
  482. viewModel.isDirty(false);
  483. }, "json");
  484. }
  485. self.viewSubmittedScript = function (workflow) {
  486. self.loadScript(workflow.scriptId);
  487. self.currentScript().script(workflow.scriptContent);
  488. self.currentScript().isRunning(true);
  489. self.currentScript().watchUrl(workflow.watchUrl);
  490. $(document).trigger("loadEditor");
  491. $(document).trigger("clearLogs");
  492. $(document).trigger("startLogsRefresh");
  493. $(document).trigger("showLogs");
  494. };
  495. self.showLogsInterval = -1;
  496. self.showLogsAtEnd = true;
  497. self.showLogs = function (workflow) {
  498. window.clearInterval(self.showLogsInterval);
  499. $("#logsModal pre").scroll(function () {
  500. self.showLogsAtEnd = $(this).scrollTop() + $(this).height() + 20 >= $(this)[0].scrollHeight;
  501. });
  502. if (workflow.isRunning) {
  503. $("#logsModal img").removeClass("hide");
  504. $("#logsModal pre").addClass("hide");
  505. $("#logsModal").modal({
  506. keyboard: true,
  507. show: true
  508. });
  509. $("#logsModal").on("hide", function () {
  510. window.clearInterval(self.showLogsInterval);
  511. });
  512. self.showLogsInterval = window.setInterval(function () {
  513. $.getJSON(workflow.watchUrl, function (data) {
  514. if (data.workflow && !data.workflow.isRunning) {
  515. window.clearInterval(self.showLogsInterval);
  516. }
  517. if (data.logs.spark) {
  518. $("#logsModal img").addClass("hide");
  519. $("#logsModal pre").removeClass("hide");
  520. var _logsEl = $("#logsModal pre");
  521. var _newLinesCount = _logsEl.html() == "" ? 0 : _logsEl.html().split("<br>").length;
  522. var newLines = data.logs.spark.split("\n").slice(_newLinesCount);
  523. if (newLines.length > 0){
  524. _logsEl.html(_logsEl.html() + newLines.join("<br>") + "<br>");
  525. }
  526. if (self.showLogsAtEnd) {
  527. _logsEl.scrollTop(_logsEl[0].scrollHeight - _logsEl.height());
  528. }
  529. }
  530. });
  531. }, 1000);
  532. }
  533. };
  534. };