collections.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615
  1. // Defaults
  2. var SOURCES = [
  3. 'file',
  4. // 'hbase',
  5. // 'hive'
  6. ];
  7. var SOURCE_TYPES = [
  8. // 'log',
  9. 'separated',
  10. // 'morphlines'
  11. ];
  12. var FIELD_TYPES = [
  13. "alphaOnlySort",
  14. "ancestor_path",
  15. "boolean",
  16. "currency",
  17. "date",
  18. "descendent_path",
  19. "double",
  20. "float",
  21. "int",
  22. "location",
  23. "location_rpt",
  24. "long",
  25. "lowercase",
  26. "pdate",
  27. "pdouble",
  28. "pfloat",
  29. "pint",
  30. "plong",
  31. "point",
  32. "random",
  33. "string",
  34. "tdate",
  35. "tdouble",
  36. "text_ar",
  37. "text_bg",
  38. "text_ca",
  39. "text_char_norm",
  40. "text_cjk",
  41. "text_cz",
  42. "text_da",
  43. "text_de",
  44. "text_el",
  45. "text_en",
  46. "text_en_splitting",
  47. "text_en_splitting_tight",
  48. "text_es",
  49. "text_eu",
  50. "text_fa",
  51. "text_fi",
  52. "text_fr",
  53. "text_ga",
  54. "text_general",
  55. "text_general_rev",
  56. "text_gl",
  57. "text_greek",
  58. "text_hi",
  59. "text_hu",
  60. "text_hy",
  61. "text_id",
  62. "text_it",
  63. "text_ja",
  64. "text_lv",
  65. "text_nl",
  66. "text_no",
  67. "text_pt",
  68. "text_ro",
  69. "text_ru",
  70. "text_sv",
  71. "text_th",
  72. "text_tr",
  73. "text_ws",
  74. "tfloat",
  75. "tint",
  76. "tlong"
  77. ];
  78. var FIELD_SEPARATORS = [
  79. ',',
  80. '\t',
  81. ];
  82. var FIELD_QUOTE_CHARACTERS = [
  83. '"',
  84. "'"
  85. ];
  86. // View Models
  87. var HiveViewModel = function() {
  88. var self = this;
  89. self.databases = ko.observableArray();
  90. self.database = ko.observable().extend({'errors': null});
  91. self.table = ko.observable().extend({'errors': null});
  92. self.database.subscribe(function() {
  93. self.loadTables();
  94. });
  95. self.table.subscribe(function() {
  96. self.loadColumns();
  97. });
  98. self.loadDatabases = function() {
  99. return $.get("/beeswax/api/autocomplete").done(function(data) {
  100. if (data.databases.length > 0) {
  101. self.databases($.map(data.databases, function(database) { return new HiveDatabase(database); }));
  102. self.database(self.databases()[0]);
  103. }
  104. })
  105. .fail(function (xhr, textStatus, errorThrown) {
  106. $(document).trigger("error", xhr.responseText);
  107. });
  108. };
  109. self.loadTables = function() {
  110. return $.get("/beeswax/api/autocomplete/" + self.database().name()).done(function(data) {
  111. if (data.tables.length > 0) {
  112. self.database().tables($.map(data.tables, function(table) { return new HiveTable(table); }));
  113. self.table(self.database().tables()[0]);
  114. }
  115. })
  116. .fail(function (xhr, textStatus, errorThrown) {
  117. $(document).trigger("error", xhr.responseText);
  118. });
  119. };
  120. self.loadColumns = function() {
  121. return $.get("/beeswax/api/autocomplete/" + self.database().name() + '/' + self.table().name()).done(function(data) {
  122. if (data.columns.length > 0) {
  123. self.table().columns(data.columns);
  124. }
  125. })
  126. .fail(function (xhr, textStatus, errorThrown) {
  127. $(document).trigger("error", xhr.responseText);
  128. });
  129. };
  130. // self.loadDatabases();
  131. };
  132. var HBaseViewModel = function() {
  133. var self = this;
  134. self.clusters = ko.observableArray();
  135. self.cluster = ko.observable().extend({'errors': null});
  136. self.table = ko.observable().extend({'errors': null});
  137. self.indexMapping = {};
  138. self.cluster.subscribe(function() {
  139. self.loadTables();
  140. });
  141. self.loadClusters = function() {
  142. return $.get("/hbase/api/getClusters").done(function(data) {
  143. if (data.data.length > 0) {
  144. self.clusters($.map(data.data, function(cluster) { return new HBaseCluster(cluster.name); }));
  145. self.cluster(self.clusters()[0]);
  146. }
  147. })
  148. .fail(function (xhr, textStatus, errorThrown) {
  149. $(document).trigger("error", xhr.responseText);
  150. });
  151. };
  152. self.loadTables = function() {
  153. return $.get("/hbase/api/getTableList/" + self.cluster().name()).done(function(data) {
  154. if (data.data.length > 0) {
  155. // @TODO(Abe): Check if enabled
  156. self.cluster().tables($.map(data.data, function(table) { return new HBaseTable(table.name); }));
  157. self.table(self.cluster().tables()[0]);
  158. }
  159. })
  160. .fail(function (xhr, textStatus, errorThrown) {
  161. $(document).trigger("error", xhr.responseText);
  162. });
  163. };
  164. // self.loadClusters();
  165. };
  166. var CreateCollectionViewModel = function() {
  167. var self = this;
  168. // Models
  169. self.collection = new Collection();
  170. self.source = ko.observable(SOURCES[0]).extend({'errors': null});
  171. self.fieldSeparator = ko.observable().extend({'errors': null});
  172. self.fieldQuoteCharacter = ko.observable().extend({'errors': null});
  173. self.file = ko.observable().extend({'errors': null});
  174. self.sourceType = ko.observable().extend({'errors': null});
  175. self.morphlines = ko.mapping.fromJS({'name': 'message', 'expression': '%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST:hostname} %{DATA:program}(?:\[%{POSINT:pid}\])?: %{GREEDYDATA:msg}'});
  176. self.morphlines.name = self.morphlines.name.extend({'errors': null});
  177. self.morphlines.expression = self.morphlines.expression.extend({'errors': null});
  178. // UI
  179. self.hive = new HiveViewModel();
  180. self.hbase = new HBaseViewModel();
  181. self.wizard = new Wizard();
  182. self.isLoading = ko.observable();
  183. self.sources = ko.mapping.fromJS(SOURCES);
  184. self.fieldTypes = ko.mapping.fromJS(FIELD_TYPES);
  185. self.fieldSeparators = ko.mapping.fromJS(FIELD_SEPARATORS);
  186. self.fieldQuoteCharacters = ko.mapping.fromJS(FIELD_QUOTE_CHARACTERS);
  187. self.sourceTypes = ko.mapping.fromJS(SOURCE_TYPES);
  188. self.parseFields = function() {
  189. if (self.source() == 'file') {
  190. self.isLoading(true);
  191. return $.post("/indexer/api/fields/parse/", {
  192. 'source': self.source(),
  193. 'type': self.sourceType(),
  194. 'path': self.file(),
  195. 'separator': self.fieldSeparator(),
  196. 'quote': self.fieldQuoteCharacter(),
  197. 'morphlines': ko.mapping.toJSON(self.morphlines)
  198. }).done(function(data) {
  199. if (data.status == 0) {
  200. self.collection.fields(inferFields(data.data, self.collection));
  201. chooseUniqueKey(self.collection);
  202. chooseDefaultField(self.collection);
  203. } else {
  204. $(document).trigger("error", data.message);
  205. }
  206. self.isLoading(false);
  207. }).fail(function (xhr, textStatus, errorThrown) {
  208. $(document).trigger("error", xhr.responseText);
  209. self.isLoading(false);
  210. });
  211. }
  212. };
  213. self.save = function() {
  214. if (self.wizard.currentPage().validate()) {
  215. self.isLoading(true);
  216. var collection = ko.mapping.toJS(self.collection);
  217. collection.fields = ko.utils.arrayMap(collection.fields, function(field) {
  218. delete field['uniqueKeyField'];
  219. return field;
  220. });
  221. switch(self.source()) {
  222. case 'file':
  223. return $.post("/indexer/api/collections/create/", {
  224. 'collection': ko.mapping.toJSON(collection),
  225. 'type': self.sourceType(),
  226. 'path': self.file(),
  227. 'source': self.source(),
  228. 'separator': self.fieldSeparator(),
  229. 'quote': self.fieldQuoteCharacter()
  230. }).done(function(data) {
  231. if (data.status == 0) {
  232. window.location.href = '/indexer';
  233. } else {
  234. self.isLoading(false);
  235. $(document).trigger("error", data.message);
  236. }
  237. })
  238. .fail(function (xhr, textStatus, errorThrown) {
  239. $(document).trigger("error", xhr.responseText);
  240. self.isLoading(false);
  241. });
  242. case 'hive':
  243. return $.post("/indexer/api/collections/create/", {
  244. 'collection': ko.toJSON(collection),
  245. 'database': self.hive.database().name(),
  246. 'table': self.hive.table().name(),
  247. 'source': self.source()
  248. }).done(function(data) {
  249. if (data.status == 0) {
  250. window.location.href = '/indexer';
  251. } else {
  252. $(document).trigger("error", data.message);
  253. }
  254. self.isLoading(false);
  255. })
  256. .fail(function (xhr, textStatus, errorThrown) {
  257. $(document).trigger("error", xhr.responseText);
  258. self.isLoading(false);
  259. });
  260. }
  261. }
  262. };
  263. };
  264. var EditCollectionViewModel = function() {
  265. var self = this;
  266. // Models
  267. self.collection = ko.observable();
  268. self.source = ko.observable(SOURCES[0]).extend({'errors': null});
  269. self.fieldSeparator = ko.observable().extend({'errors': null});
  270. self.fieldQuoteCharacter = ko.observable().extend({'errors': null});
  271. self.file = ko.observable().extend({'errors': null});
  272. self.sourceType = ko.observable().extend({'errors': null});
  273. // UI
  274. self.wizard = new Wizard();
  275. self.sources = ko.mapping.fromJS(SOURCES);
  276. self.fieldTypes = ko.mapping.fromJS(FIELD_TYPES);
  277. self.sourceTypes = ko.mapping.fromJS(SOURCE_TYPES);
  278. self.fieldSeparators = ko.mapping.fromJS(FIELD_SEPARATORS);
  279. self.fieldQuoteCharacters = ko.mapping.fromJS(FIELD_QUOTE_CHARACTERS);
  280. self.isLoading = ko.observable();
  281. self.fetchFields = function() {
  282. if (self.collection()) {
  283. self.isLoading(true);
  284. return $.get("/indexer/api/collections/" + self.collection().name() + "/fields/").done(function(data) {
  285. if (data.status == 0) {
  286. self.collection().fields(inferFields(data.fields, self.collection()));
  287. self.collection().uniqueKeyField(data.unique_key);
  288. self.sourceType(data.type);
  289. } else {
  290. $(document).trigger("error", data.message);
  291. }
  292. self.isLoading(false);
  293. })
  294. .fail(function (xhr, textStatus, errorThrown) {
  295. $(document).trigger("error", xhr.responseText);
  296. self.isLoading(false);
  297. });
  298. }
  299. };
  300. self.removeCollection = function(collection) {
  301. self.isLoading(true);
  302. var data = [ko.mapping.toJS(self.collection)];
  303. return $.post("/indexer/api/collections/remove/", {
  304. 'collections': ko.mapping.toJSON(data)
  305. }).done(function(data) {
  306. if (data.status == 0) {
  307. window.location.href = '/indexer';
  308. } else {
  309. $(document).trigger("error", data.message);
  310. }
  311. self.isLoading(false);
  312. })
  313. .fail(function (xhr, textStatus, errorThrown) {
  314. $(document).trigger("error", xhr.responseText);
  315. self.isLoading(false);
  316. });
  317. };
  318. self.updateCollection = function() {
  319. self.isLoading(true);
  320. var data = ko.mapping.toJS(self.collection);
  321. return $.post("/indexer/api/collections/" + self.collection().name() + "/update/", {
  322. 'collection': ko.mapping.toJSON(data),
  323. 'type': self.sourceType()
  324. }).done(function(data) {
  325. if (data.status == 0) {
  326. $(document).trigger("info", data.message);
  327. } else {
  328. $(document).trigger("error", data.message);
  329. }
  330. self.isLoading(false);
  331. })
  332. .fail(function (xhr, textStatus, errorThrown) {
  333. $(document).trigger("error", xhr.responseText);
  334. self.isLoading(false);
  335. });
  336. };
  337. self.addData = function() {
  338. self.isLoading(true);
  339. switch(self.source()) {
  340. case 'file':
  341. var data = ko.mapping.toJS(self.collection);
  342. return $.post("/indexer/api/collections/" + self.collection().name() + "/data/", {
  343. 'collection': ko.mapping.toJSON(data),
  344. 'source': self.source(),
  345. 'type': self.sourceType(),
  346. 'path': self.file(),
  347. 'separator': self.fieldSeparator(),
  348. 'quote': self.fieldQuoteCharacter()
  349. }).done(function(data) {
  350. if (data.status == 0) {
  351. window.location.href = '/indexer';
  352. } else {
  353. self.isLoading(false);
  354. $(document).trigger("error", data.message);
  355. }
  356. })
  357. .fail(function (xhr, textStatus, errorThrown) {
  358. self.isLoading(false);
  359. $(document).trigger("error", xhr.responseText);
  360. });
  361. }
  362. };
  363. };
  364. var ManageCollectionsViewModel = function() {
  365. var self = this;
  366. // Models
  367. self.collections = ko.observableArray();
  368. // UI
  369. self.isLoading = ko.observable();
  370. self.hasLoadedOnce = ko.observable(false);
  371. self.showCores = ko.observable(false);
  372. self.filteredCollections = ko.observableArray();
  373. self.displayCollections = ko.computed(function() {
  374. return ko.utils.arrayFilter(self.filteredCollections(), function(collection) {
  375. return ! self.hasCloudCollections() || self.showCores() || ! ko.unwrap(collection).isCoreOnly();
  376. });
  377. });
  378. self.selectedCollections = ko.computed(function() {
  379. return ko.utils.arrayFilter(self.collections(), function(collection) {
  380. return collection.selected();
  381. });
  382. });
  383. self.selectedCloudCollections = ko.computed(function() {
  384. return ko.utils.arrayFilter(self.selectedCollections(), function(collection) {
  385. return !ko.unwrap(collection).isCoreOnly();
  386. });
  387. });
  388. self.hasCloudCollections = ko.computed(function() {
  389. var _arr = ko.utils.arrayFilter(self.collections(), function(collection) {
  390. return !ko.unwrap(collection).isCoreOnly();
  391. });
  392. return _arr.length > 0;
  393. });
  394. self.toggleSelectAll = function() {
  395. var direction = !self.selectedCollections().length;
  396. ko.utils.arrayForEach(self.filteredCollections(), function(collection) {
  397. collection.selected(direction);
  398. });
  399. };
  400. self.toggleCollectionSelect = function(collection, e) {
  401. ko.utils.arrayForEach(self.collections(), function(other_collection) {
  402. if(ko.unwrap(other_collection).name() == collection.name()) {
  403. other_collection.selected(!other_collection.selected());
  404. }
  405. });
  406. };
  407. self.importCollection = function(collection, e) {
  408. self.isLoading(true);
  409. return $.post("/indexer/api/collections/import/", {
  410. 'collection': ko.mapping.toJSON(collection)
  411. }).done(function(data) {
  412. self.isLoading(false);
  413. $(document).trigger("info", data.message);
  414. self.fetchCollections();
  415. })
  416. .fail(function (xhr, textStatus, errorThrown) {
  417. self.isLoading(false);
  418. $(document).trigger("error", xhr.responseText);
  419. });
  420. };
  421. self.filterTest = function(obj, filter) {
  422. return ko.unwrap(obj).name().indexOf(filter) != -1;
  423. };
  424. self.fetchCollections = function() {
  425. self.isLoading(true);
  426. return $.get("/indexer/api/collections/").done(function(data) {
  427. if (data.status == 0) {
  428. var collections = [];
  429. ko.utils.arrayForEach(data.collections, function(collection) {
  430. var new_collection = ko.observable(new Collection(collection.name)).extend({'selectable': null});
  431. new_collection().hasHueCollection(collection.hue);
  432. new_collection().hasSolrCollection(collection.solr);
  433. new_collection().isCoreOnly(collection.isCoreOnly);
  434. collections.push(new_collection);
  435. });
  436. self.collections(collections);
  437. self.hasLoadedOnce(true);
  438. } else {
  439. $(document).trigger("error", data.message);
  440. }
  441. self.isLoading(false);
  442. })
  443. .fail(function (xhr, textStatus, errorThrown) {
  444. $(document).trigger("error", xhr.responseText);
  445. self.isLoading(false);
  446. });
  447. };
  448. self.removeCollections = function(collection) {
  449. self.isLoading(true);
  450. var collections = self.selectedCloudCollections();
  451. if (collections.length == 0) {
  452. collections = ($.isArray(collection)) ? collection : [collection];
  453. }
  454. return $.post("/indexer/api/collections/remove/", {
  455. 'collections': ko.mapping.toJSON(collections)
  456. }).done(function(data) {
  457. if (data.status == 0) {
  458. // Remove collections
  459. collection_names = ko.utils.arrayMap(collections, function(collection) {
  460. return ko.unwrap(collection).name();
  461. });
  462. var index = 0;
  463. var remove = [];
  464. ko.utils.arrayForEach(self.collections(), function(collection) {
  465. collection = ko.unwrap(collection);
  466. if (collection_names.indexOf(collection.name()) != -1) {
  467. remove.push(index);
  468. }
  469. index++;
  470. });
  471. remove.reverse();
  472. ko.utils.arrayForEach(remove, function(index) {
  473. self.collections.splice(index, 1);
  474. });
  475. } else {
  476. $(document).trigger("error", data.message);
  477. }
  478. self.isLoading(false);
  479. })
  480. .fail(function (xhr, textStatus, errorThrown) {
  481. $(document).trigger("error", xhr.responseText);
  482. self.isLoading(false);
  483. });
  484. };
  485. };
  486. var CollectionsViewModel = function(config) {
  487. var self = this;
  488. var _config = config || {};
  489. // Models
  490. self.page = ko.observable();
  491. self.breadcrumb = ko.observable();
  492. self.breadcrumb.list = ko.computed(function() {
  493. var breadcrum_config = _config.breadcrumb || {};
  494. var labels = breadcrum_config.labels || {};
  495. var skip = breadcrum_config.skip || [];
  496. var breadcrums = [];
  497. var crum = '';
  498. breadcrums.push({
  499. 'url': '/',
  500. 'label': labels[crum] || crum,
  501. 'crum': crum
  502. });
  503. if (self.breadcrumb()) {
  504. var crums = [];
  505. var newcrums = ko.utils.arrayMap( self.breadcrumb().split('/'), function(crum) {
  506. crums.push(crum);
  507. if (skip.indexOf(crum) != -1) {
  508. return null;
  509. }
  510. return {
  511. 'url': crums.join('/'),
  512. 'label': labels[crum] || crum,
  513. 'crum': crum
  514. };
  515. });
  516. breadcrums.push.apply( breadcrums, ko.utils.arrayFilter(newcrums, function(breadcrumb) {
  517. return !!breadcrumb;
  518. }) );
  519. }
  520. return breadcrums;
  521. });
  522. // UI
  523. self.create = new CreateCollectionViewModel();
  524. self.manage = new ManageCollectionsViewModel();
  525. self.edit = new EditCollectionViewModel();
  526. self.isLoading = ko.computed(function() {
  527. return self.create.isLoading() || self.manage.isLoading() || self.edit.isLoading();
  528. });
  529. };
  530. // Utils
  531. function getFileBrowseButton(inputElement) {
  532. return $("<button>").addClass("btn").addClass("fileChooserBtn").text("..").click(function (e) {
  533. e.preventDefault();
  534. $("#filechooser").jHueFileChooser({
  535. initialPath: inputElement.val(),
  536. onFileChoose: function (filePath) {
  537. inputElement.val(filePath);
  538. inputElement.trigger("change");
  539. $("#chooseFile").modal("hide");
  540. },
  541. selectFolder: false,
  542. createFolder: false
  543. });
  544. $("#chooseFile").modal("show");
  545. });
  546. }
  547. function inferFields(field_data, collection) {
  548. var fields = [];
  549. $.each(field_data, function(index, value) {
  550. // 0 => name
  551. // 1 => type
  552. // 2 => required
  553. // 3 => indexed
  554. // 4 => stored
  555. var field = new Field(collection, value[0], value[1], value[2], value[3]);
  556. field.saved(true);
  557. fields.push(field);
  558. });
  559. return fields;
  560. }