indexer.mako 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034
  1. ## Licensed to Cloudera, Inc. under one
  2. ## or more contributor license agreements. See the NOTICE file
  3. ## distributed with this work for additional information
  4. ## regarding copyright ownership. Cloudera, Inc. licenses this file
  5. ## to you under the Apache License, Version 2.0 (the
  6. ## "License"); you may not use this file except in compliance
  7. ## with the License. You may obtain a copy of the License at
  8. ##
  9. ## http://www.apache.org/licenses/LICENSE-2.0
  10. ##
  11. ## Unless required by applicable law or agreed to in writing, software
  12. ## distributed under the License is distributed on an "AS IS" BASIS,
  13. ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. ## See the License for the specific language governing permissions and
  15. ## limitations under the License.
  16. <%!
  17. from desktop import conf
  18. from desktop.views import commonheader, commonfooter, commonshare, commonimportexport
  19. from django.utils.translation import ugettext as _
  20. %>
  21. <%namespace name="actionbar" file="actionbar.mako" />
  22. <%namespace name="assist" file="/assist.mako" />
  23. <%namespace name="tableStats" file="/table_stats.mako" />
  24. <%namespace name="require" file="/require.mako" />
  25. ${ commonheader(_("Solr Indexes"), "search", user, "60px") | n,unicode }
  26. ${ require.config() }
  27. ${ tableStats.tableStats() }
  28. ${ assist.assistPanel() }
  29. <link rel="stylesheet" href="${ static('notebook/css/notebook.css') }">
  30. <link rel="stylesheet" href="${ static('desktop/css/wizard.css') }">
  31. <style type="text/css">
  32. % if conf.CUSTOM.BANNER_TOP_HTML.get():
  33. .show-assist {
  34. top: 110px!important;
  35. }
  36. .main-content {
  37. top: 112px!important;
  38. }
  39. % endif
  40. .path {
  41. margin-bottom: 0!important;
  42. border-right: none!important;
  43. }
  44. .step1 label div {
  45. width: 50px;
  46. display: inline-block;
  47. }
  48. .step2.form-inline input[type='text'], .step2.form-inline select {
  49. margin-right: 10px;
  50. margin-left: 3px;
  51. }
  52. .step2.form-inline input[type='checkbox'] {
  53. margin-left: 10px!important;
  54. margin-right: 4px!important;
  55. }
  56. .field {
  57. padding: 4px;
  58. padding-left: 10px;
  59. border-left: 4px solid #DBE8F1;
  60. }
  61. .operation {
  62. border-left: 4px solid #EEE;
  63. padding-left: 10px;
  64. margin-top: 5px;
  65. margin-bottom: 5px;
  66. margin-left: 10px;
  67. }
  68. .right-panel {
  69. overflow-x: hidden;
  70. }
  71. .fileChooserBtn {
  72. margin-top: -2px;
  73. }
  74. .form-actions {
  75. position: fixed;
  76. bottom: 0;
  77. margin: 0;
  78. }
  79. #notebook {
  80. height: 5px;
  81. margin-top: 10px;
  82. }
  83. </style>
  84. <script src="${ static('desktop/js/jquery.hiveautocomplete.js') }" type="text/javascript" charset="utf-8"></script>
  85. <script src="${ static('desktop/js/jquery.hdfsautocomplete.js') }" type="text/javascript" charset="utf-8"></script>
  86. <div class="navbar navbar-inverse navbar-fixed-top">
  87. <div class="navbar-inner">
  88. <div class="container-fluid">
  89. <div class="nav-collapse">
  90. <ul class="nav">
  91. <li class="currentApp">
  92. <a href="/indexer/indexer">
  93. <i class="fa fa-database app-icon"></i> ${_('Indexes')}</a>
  94. </a>
  95. </li>
  96. </ul>
  97. </div>
  98. </div>
  99. </div>
  100. </div>
  101. <a title="${_('Toggle Assist')}" class="pointer show-assist" data-bind="visible: !$root.isLeftPanelVisible() && $root.assistAvailable(), click: function() { $root.isLeftPanelVisible(true); }">
  102. <i class="fa fa-chevron-right"></i>
  103. </a>
  104. <div class="main-content">
  105. <div class="vertical-full container-fluid" data-bind="style: { 'padding-left' : $root.isLeftPanelVisible() ? '0' : '20px' }">
  106. <div class="vertical-full">
  107. <div class="vertical-full row-fluid panel-container">
  108. <div class="assist-container left-panel" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable()">
  109. <a title="${_('Toggle Assist')}" class="pointer hide-assist" data-bind="click: function() { $root.isLeftPanelVisible(false) }">
  110. <i class="fa fa-chevron-left"></i>
  111. </a>
  112. <div class="assist" data-bind="component: {
  113. name: 'assist-panel',
  114. params: {
  115. user: '${user.username}',
  116. onlySql: false,
  117. sql: {
  118. sourceTypes: [{
  119. name: 'hive',
  120. type: 'hive'
  121. }],
  122. navigationSettings: {
  123. openItem: false,
  124. showStats: true
  125. }
  126. },
  127. visibleAssistPanels: ['sql', 'hdfs', 'documents']
  128. }
  129. }"></div>
  130. </div>
  131. <div class="resizer" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable(), splitDraggable : { appName: 'notebook', leftPanelVisible: $root.isLeftPanelVisible }"><div class="resize-bar">&nbsp;</div></div>
  132. <div class="right-panel">
  133. <div style="margin: 10px; margin-bottom: 100px">
  134. <!-- ko template: 'create-index-wizard' --><!-- /ko -->
  135. </div>
  136. </div>
  137. </div>
  138. </div>
  139. </div>
  140. </div>
  141. <div id="chooseFile" class="modal hide fade">
  142. <div class="modal-header">
  143. <a href="#" class="close" data-dismiss="modal">&times;</a>
  144. <h3>${_('Choose a file')}</h3>
  145. </div>
  146. <div class="modal-body">
  147. <div id="filechooser"></div>
  148. </div>
  149. <div class="modal-footer"></div>
  150. </div>
  151. <script type="text/html" id="create-index-wizard">
  152. <div data-bind="visible: createWizard.show">
  153. <ol class="list-inline text-center step-indicator">
  154. <li data-bind="css: { 'active': currentStep() == 1, 'complete': currentStep() > 1 }, click: function() { currentStep(1) }">
  155. <div class="step">
  156. <!-- ko if: currentStep() == 1 -->
  157. <!-- ko if: createWizard.isGuessingFormat -->
  158. <span class="fa fa-spinner fa-spin"></span>
  159. <!-- /ko -->
  160. <!-- ko ifnot: createWizard.isGuessingFormat -->
  161. 1
  162. <!-- /ko -->
  163. <!-- /ko -->
  164. <!-- ko ifnot: currentStep() == 1 -->
  165. <span class="fa fa-check"></span>
  166. <!-- /ko -->
  167. </div>
  168. <div class="caption">${ _('Pick it') }</div>
  169. </li>
  170. <li data-bind="css: { 'inactive': currentStep() == 1, 'active': currentStep() == 2, 'complete': currentStep() == 3 }, click: function() { currentStep(2) }">
  171. <div class="step">
  172. <!-- ko if: currentStep() < 3 -->
  173. <!-- ko if: createWizard.isGuessingFieldTypes -->
  174. <span class="fa fa-spinner fa-spin"></span>
  175. <!-- /ko -->
  176. <!-- ko ifnot: createWizard.isGuessingFieldTypes -->
  177. 2
  178. <!-- /ko -->
  179. <!-- /ko -->
  180. <!-- ko if: currentStep() == 3 -->
  181. <span class="fa fa-check"></span>
  182. <!-- /ko -->
  183. </div>
  184. <div class="caption">${ _('Tweak it') }</div>
  185. </li>
  186. <li data-bind="css: { 'inactive': currentStep() < 3, 'active': currentStep() == 3, 'error': createWizard.indexingError, 'complete': createWizard.indexingSuccess }, click: function() { currentStep(3) }">
  187. <div class="step">
  188. <!-- ko if: createWizard.isIndexing -->
  189. <span class="fa fa-spinner fa-spin"></span>
  190. <!-- /ko -->
  191. <!-- ko ifnot: createWizard.isIndexing -->
  192. <!-- ko if: !createWizard.indexingError() && !createWizard.indexingSuccess() -->
  193. 3
  194. <!-- /ko -->
  195. <!-- ko if: createWizard.indexingError -->
  196. <span class="fa fa-exclamation-triangle"></span>
  197. <!-- /ko -->
  198. <!-- ko if: createWizard.indexingSuccess -->
  199. <span class="fa fa-check"></span>
  200. <!-- /ko -->
  201. <!-- /ko -->
  202. </div>
  203. <div class="caption">${ _('Index it!') }</div>
  204. </li>
  205. </ol>
  206. <!-- ko if: currentStep() == 1 -->
  207. <div class="card step1">
  208. <h3 class="card-heading simple">${_('Collection details')}</h3>
  209. <div class="card-body">
  210. <form class="form-inline">
  211. <div class="control-group">
  212. <label for="collectionName" class="control-label"><div>${ _('Name') }</div>
  213. <input type="text" class="form-control" id="collectionName" data-bind="value: createWizard.fileFormat().name, valueUpdate: 'afterkeydown'" placeholder="${ _('Collection name') }">
  214. <span class="help-inline muted" data-bind="visible: createWizard.isNameAvailable()">${ _('A new collection will be created') }</span>
  215. <span class="help-inline muted" data-bind="visible: !createWizard.isNameAvailable() && createWizard.fileFormat().name().length > 0">
  216. ${_('Adding data to this existing collection') }
  217. <a href="javascript:void(0)" data-bind="attr: {href: '${ url("indexer:collections") }' +'#edit/' + createWizard.fileFormat().name() }, text: createWizard.fileFormat().name" target="_blank"></a>
  218. </span>
  219. </label>
  220. </div>
  221. <div data-bind="visible: createWizard.fileFormat().name().length > 0">
  222. <div class="control-group">
  223. <label for="collectionType" class="control-label"><div>${ _('Type') }</div>
  224. <select id="collectionType" data-bind="options: createWizard.fileFormat().inputFormats, value: createWizard.fileFormat().inputFormat"></select>
  225. </label>
  226. </div>
  227. <div class="control-group" data-bind="visible: createWizard.fileFormat().inputFormat() == 'file'">
  228. <label for="path" class="control-label"><div>${ _('Path') }</div>
  229. <input type="text" class="form-control path" data-bind="value: createWizard.fileFormat().path, filechooser: createWizard.fileFormat().path, filechooserOptions: { skipInitialPathIfEmpty: true }">
  230. </label>
  231. </div>
  232. <div class="control-group" data-bind="visible: createWizard.fileFormat().inputFormat() == 'table'">
  233. <label for="path" class="control-label"><div>${ _('Table') }</div>
  234. <input type="text" data-bind="value: createWizard.fileFormat().table, hivechooser: createWizard.fileFormat().table, skipColumns: true">
  235. </label>
  236. </div>
  237. <div class="control-group" data-bind="visible: createWizard.fileFormat().inputFormat() == 'query'">
  238. <label for="path" class="control-label"><div>${ _('Query') }</div>
  239. <select data-bind="options: createWizard.fileFormat().queries, value: createWizard.fileFormat().query, optionsText: 'name', optionsAfterRender: createWizard.fileFormat().selectQuery"></select>
  240. </label>
  241. </div>
  242. </div>
  243. </form>
  244. </div>
  245. </div>
  246. <!-- /ko -->
  247. <!-- ko if: currentStep() == 2 -->
  248. <div class="card step2">
  249. <h3 class="card-heading simple">${_('Format')}</h3>
  250. <div class="card-body">
  251. <form class="form-inline">
  252. <label>${_('File Type')} <select data-bind="options: $root.createWizard.fileTypes, optionsText: 'description', value: $root.createWizard.fileType"></select></label>
  253. <span data-bind="with: createWizard.fileFormat().format, visible: createWizard.fileFormat().show">
  254. <!-- ko template: {name: 'format-settings'}--><!-- /ko -->
  255. </span>
  256. </form>
  257. </div>
  258. </div>
  259. <div class="card step2">
  260. <h3 class="card-heading simple">${_('Fields')}</h3>
  261. <div class="card-body">
  262. <!-- ko if: createWizard.isGuessingFieldTypes -->
  263. <i class="fa fa-spinner fa-spin"></i>
  264. <!-- /ko -->
  265. <form class="form-inline" data-bind="foreach: createWizard.fileFormat().columns">
  266. <div data-bind="template: { name:'field-template', data:$data}" class="margin-top-10 field"></div>
  267. </form>
  268. </div>
  269. </div>
  270. <!-- /ko -->
  271. <!-- ko if: currentStep() == 3 -->
  272. <div class="card step3">
  273. <h3 class="card-heading simple">${_('Preview')}</h3>
  274. <div class="card-body">
  275. <!-- ko if: createWizard.isGuessingFieldTypes -->
  276. <i class="fa fa-spinner fa-spin"></i>
  277. <!-- /ko -->
  278. <div style="overflow: auto">
  279. <table class="table table-striped table-condensed" style="margin:auto;text-align:left">
  280. <thead>
  281. <tr data-bind="foreach: createWizard.fileFormat().columns">
  282. <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  283. </tr>
  284. </thead>
  285. <tbody data-bind="foreach: createWizard.sample">
  286. <tr data-bind="foreach: $data">
  287. <!-- ko if: $index() < $root.createWizard.fileFormat().columns().length -->
  288. <td data-bind="visible: $root.createWizard.fileFormat().columns()[$index()].keep, text: $data">
  289. </td>
  290. <!-- ko with: $root.createWizard.fileFormat().columns()[$index()] -->
  291. <!-- ko template: 'output-generated-field-data-template' --> <!-- /ko -->
  292. <!-- /ko -->
  293. <!-- /ko -->
  294. </tr>
  295. </tbody>
  296. </table>
  297. </div>
  298. </div>
  299. </div>
  300. <!-- /ko -->
  301. <div class="form-actions">
  302. <!-- ko if: previousStepVisible -->
  303. <button class="btn" data-bind="click: previousStep">${ _('Previous') }</button>
  304. <!-- /ko -->
  305. <!-- ko if: currentStep() == 1 && createWizard.fileFormat().name().length > 0 -->
  306. <button class="btn" data-bind="click: createWizard.guessFormat, visible: createWizard.fileFormat().inputFormat() == 'file', enable: createWizard.fileFormat().path() !== '' && !createWizard.isGuessingFormat()">${_('Guess File Format')} <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.isGuessingFormat"></i> </button>
  307. <button class="btn" data-bind="click: createWizard.guessFormat, visible: createWizard.fileFormat().inputFormat() == 'table', enable: createWizard.fileFormat().table() !== '' && !createWizard.isGuessingFormat()">${_('Process Table')} <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.isGuessingFormat"></i></button>
  308. <button class="btn" data-bind="click: createWizard.guessFormat, visible: createWizard.fileFormat().inputFormat() == 'query', enable: createWizard.fileFormat().query() !== '' && !createWizard.isGuessingFormat()">${_('Process Query')} <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.isGuessingFormat"></i></button>
  309. <!-- /ko -->
  310. <!-- ko if: currentStep() > 1 -->
  311. <!-- ko if: nextStepVisible -->
  312. <button class="btn" data-bind="click: nextStep">${ _('Next') }</button>
  313. <!-- /ko -->
  314. <!-- /ko -->
  315. <!-- ko if: currentStep() == 3 -->
  316. <button href="javascript:void(0)" class="btn btn-primary disable-feedback" data-bind="click: createWizard.indexFile, enable: createWizard.readyToIndex() && !createWizard.indexingStarted()">
  317. ${_('Index it!')} <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.indexingStarted"></i>
  318. </button>
  319. <!-- /ko -->
  320. <span data-bind="visible: createWizard.editorId">
  321. <a href="javascript:void(0)" class="btn btn-success" data-bind="attr: {href: '${ url('notebook:editor') }?editor=' + createWizard.editorId() }" target="_blank" title="${ _('Open') }">
  322. ${_('View indexing status')}
  323. </a>
  324. ${ _('View collection') } <a href="javascript:void(0)" data-bind="attr: {href: '${ url("indexer:collections") }' +'#edit/' + createWizard.fileFormat().name() }, text: createWizard.fileFormat().name" target="_blank"></a>
  325. </span>
  326. <div id="notebook"></div>
  327. </div>
  328. </div>
  329. </script>
  330. <script type="text/html" id="format-settings">
  331. <!-- ko foreach: {data: getArguments(), as: 'argument'} -->
  332. <!-- ko template: {name: 'arg-'+argument.type, data:{description: argument.description, value: $parent[argument.name]}}--><!-- /ko -->
  333. <!-- /ko -->
  334. </script>
  335. <script type="text/html" id="field-template">
  336. <label>${ _('Name') }
  337. <input type="text" class="input-large" placeholder="${ _('Field name') }" data-bind="value: name">
  338. </label>
  339. <label>${ _('Type') }
  340. <select data-bind="options: $root.createWizard.fieldTypes, value: type"></select>
  341. </label>
  342. <label class="checkbox">
  343. <input type="checkbox" data-bind="checked: unique"> ${_('Unique')}
  344. </label>
  345. <label class="checkbox">
  346. <input type="checkbox" data-bind="checked: keep"> ${_('Keep in index')}
  347. </label>
  348. <label class="checkbox">
  349. <input type="checkbox" data-bind="checked: required"> ${_('Required')}
  350. </label>
  351. <!-- ko if: operations().length == 0 -->
  352. <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Add Operation')}</a>
  353. <!-- /ko -->
  354. <div data-bind="foreach: operations">
  355. <div data-bind="template: { name:'operation-template',data:{operation: $data, list: $parent.operations}}"></div>
  356. </div>
  357. <!-- ko if: operations().length > 0 -->
  358. <a class="pointer" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Add Operation to')} <span data-bind="text: name"></span></a>
  359. <!-- /ko -->
  360. </script>
  361. <script type="text/html" id="operation-template">
  362. <div class="operation">
  363. <select data-bind="options: $root.createWizard.operationTypes.map(function(o){return o.name});, value: operation.type"></select>
  364. <!-- ko template: "args-template" --><!-- /ko -->
  365. <!-- ko if: operation.settings().outputType() == "custom_fields" -->
  366. <label> ${ _('Number of expected fields') }
  367. <input type="number" class="input-mini" data-bind="value: operation.numExpectedFields">
  368. </label>
  369. <!-- /ko -->
  370. <a class="pointer margin-left-20" data-bind="click: function(){$root.createWizard.removeOperation(operation, list)}" title="${ _('Remove') }"><i class="fa fa-times"></i></a>
  371. <div class="margin-left-20" data-bind="foreach: operation.fields">
  372. <div data-bind="template: { name:'field-template', data:$data}" class="margin-top-10 field"></div>
  373. </div>
  374. </div>
  375. </script>
  376. <script type="text/html" id="args-template">
  377. <!-- ko foreach: {data: operation.settings().getArguments(), as: 'argument'} -->
  378. <!-- ko template: {name: 'arg-'+argument.type, data:{description: argument.description, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
  379. <!-- /ko -->
  380. </script>
  381. <script type="text/html" id="arg-text">
  382. <label>
  383. <span data-bind="text: description"></span>
  384. <input type="text" class="input-small" data-bind="attr: {placeholder: description}, value: value">
  385. </label>
  386. </script>
  387. <script type="text/html" id="arg-checkbox">
  388. <label>
  389. <input type="checkbox" data-bind="checked: value">
  390. <span data-bind="text: description"></span>
  391. </label>
  392. </script>
  393. <script type="text/html" id="arg-mapping">
  394. <!-- ko foreach: value-->
  395. <div>
  396. <input type="text" data-bind="value: key, attr: {placeholder: 'key'}">
  397. <input type="text" data-bind="value: value, attr: {placeholder: 'value'}">
  398. <button class="btn" data-bind="click: function(){$parent.value.remove($data)}">${_('Remove Pair')}</button>
  399. </div>
  400. <!-- /ko -->
  401. <button class="btn" data-bind="click: function(){value.push({key: ko.observable(''), value: ko.observable('')})}">${_('Add Pair')}</button>
  402. <br>
  403. </script>
  404. <script type="text/html" id="field-preview-header-template">
  405. <th data-bind="visible: keep, text: name" style="padding-right:60px"></th>
  406. <!-- ko foreach: operations -->
  407. <!--ko foreach: fields -->
  408. <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  409. <!-- /ko -->
  410. <!--/ko -->
  411. </script>
  412. <script type="text/html" id="output-generated-field-data-template">
  413. <!-- ko foreach: operations -->
  414. <!--ko foreach: fields -->
  415. <td data-bind="visible: keep">[[${_('generated')}]]</td>
  416. <!-- ko template: 'output-generated-field-data-template' --><!-- /ko -->
  417. <!-- /ko -->
  418. <!--/ko -->
  419. </script>
  420. <script type="text/html" id="notebook-progress">
  421. <!-- ko with: selectedNotebook -->
  422. <!-- ko foreach: snippets -->
  423. <div class="progress-snippet progress active" data-bind="css: {
  424. 'progress-starting': progress() == 0 && status() == 'running',
  425. 'progress-warning': progress() > 0 && progress() < 100,
  426. 'progress-success': progress() == 100,
  427. 'progress-danger': progress() == 0 && errors().length > 0}" style="background-color: #FFF; width: 100%; height: 4px">
  428. <div class="bar" data-bind="style: {'width': (errors().length > 0 ? 100 : Math.max(2,progress())) + '%'}"></div>
  429. </div>
  430. <!-- /ko -->
  431. <!-- /ko -->
  432. </script>
  433. <script type="text/javascript" charset="utf-8">
  434. require([
  435. "knockout",
  436. "ko.charts",
  437. "desktop/js/apiHelper",
  438. "notebook/js/notebook.ko",
  439. "assistPanel",
  440. "tableStats",
  441. "knockout-mapping",
  442. "knockout-sortable",
  443. "ko.editable",
  444. "ko.hue-bindings"
  445. ], function (ko, charts, ApiHelper, EditorViewModel) {
  446. ko.options.deferUpdates = true;
  447. var fieldNum = 0;
  448. var getNewFieldName = function () {
  449. fieldNum++;
  450. return "new_field_" + fieldNum
  451. }
  452. var createDefaultField = function () {
  453. var defaultField = ko.mapping.fromJS(${default_field_type | n});
  454. defaultField.name = ko.observable(getNewFieldName());
  455. return defaultField;
  456. };
  457. var Operation = function (type) {
  458. var self = this;
  459. var createArgumentValue = function (arg) {
  460. if (arg.type == "mapping") {
  461. return ko.observableArray([]);
  462. }
  463. else if (arg.type == "checkbox") {
  464. return ko.observable(false);
  465. }
  466. else {
  467. return ko.observable("");
  468. }
  469. }
  470. var constructSettings = function (type) {
  471. var settings = {};
  472. var operation = viewModel.createWizard.operationTypes.find(function (currOperation) {
  473. return currOperation.name == type;
  474. });
  475. for (var i = 0; i < operation.args.length; i++) {
  476. var argVal = createArgumentValue(operation.args[i]);
  477. if (operation.args[i].type == "checkbox" && operation.outputType == "checkbox_fields") {
  478. argVal.subscribe(function (newVal) {
  479. if (newVal) {
  480. self.fields.push(createDefaultField());
  481. }
  482. else {
  483. self.fields.pop();
  484. }
  485. });
  486. }
  487. settings[operation.args[i].name] = argVal;
  488. }
  489. settings.getArguments = function () {
  490. return operation.args
  491. };
  492. settings.outputType = function () {
  493. return operation.outputType;
  494. }
  495. return settings;
  496. };
  497. var init = function () {
  498. self.fields([]);
  499. self.numExpectedFields(0);
  500. self.numExpectedFields.subscribe(function (numExpectedFields) {
  501. if (numExpectedFields < self.fields().length) {
  502. self.fields(self.fields().slice(0, numExpectedFields));
  503. }
  504. else if (numExpectedFields > self.fields().length) {
  505. var difference = numExpectedFields - self.fields().length;
  506. for (var i = 0; i < difference; i++) {
  507. self.fields.push(createDefaultField());
  508. }
  509. }
  510. });
  511. self.settings(constructSettings(self.type()));
  512. }
  513. self.load = function (data) {
  514. self.numExpectedFields(data.numExpectedFields);
  515. var newSettings = constructSettings(data.type);
  516. for (var key in data.settings) {
  517. newSettings[key] = ko.mapping.fromJS(data.settings[key]);
  518. }
  519. self.settings(newSettings);
  520. data.fields.forEach(function (field) {
  521. self.fields.push(loadField(field));
  522. });
  523. }
  524. self.type = ko.observable(type);
  525. self.fields = ko.observableArray();
  526. self.numExpectedFields = ko.observable();
  527. self.settings = ko.observable();
  528. init();
  529. self.type.subscribe(function (newType) {
  530. init();
  531. });
  532. }
  533. var FileType = function (typeName, args) {
  534. var self = this;
  535. var type;
  536. var init = function () {
  537. self.type = ko.observable(typeName);
  538. var types = viewModel.createWizard.fileTypes;
  539. for (var i = 0; i < types.length; i++) {
  540. if (types[i].name == typeName) {
  541. type = types[i];
  542. break;
  543. }
  544. }
  545. for (var i = 0; i < type.args.length; i++) {
  546. self[type.args[i].name] = ko.observable();
  547. }
  548. if (args) loadFromObj(args);
  549. for (var i = 0; i < type.args.length; i++) {
  550. self[type.args[i].name].subscribe(viewModel.createWizard.guessFieldTypes);
  551. }
  552. }
  553. var loadFromObj = function (args) {
  554. for (var attr in args) {
  555. self[attr] = ko.mapping.fromJS(args[attr]);
  556. }
  557. }
  558. self.getArguments = function () {
  559. return type.args;
  560. }
  561. self.isCustomizable = function () {
  562. return type.isCustomizable;
  563. }
  564. init();
  565. }
  566. var IndexerFormat = function (vm) {
  567. var self = this;
  568. self.name = ko.observable('');
  569. self.inputFormat = ko.observable('file');
  570. self.inputFormat.subscribe(function(val) {
  571. if (val == 'query') {
  572. self.getDocuments();
  573. }
  574. });
  575. self.inputFormats = ko.observableArray(['file', 'table', 'query']);
  576. self.draggedQuery = ko.observable();
  577. // File
  578. self.path = ko.observable('');
  579. // Table
  580. self.table = ko.observable('');
  581. self.tableName = ko.computed(function() {
  582. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[1] : self.table();
  583. });
  584. self.databaseName = ko.computed(function() {
  585. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[0] : 'default';
  586. });
  587. // Queries
  588. self.query = ko.observable('');
  589. self.queries = ko.observableArray([]);
  590. self.getDocuments = function() {
  591. $.get('/desktop/api2/docs/', {
  592. type: 'query-hive',
  593. include_trashed: false,
  594. sort: '-last_modified',
  595. limit: 100
  596. }, function(data) {
  597. if (data && data.documents) {
  598. var queries = [];
  599. $.each(data.documents, function(index, query) {
  600. queries.push(ko.mapping.fromJS(query));
  601. });
  602. self.queries(queries);
  603. }
  604. });
  605. };
  606. var waitForRendered = -1;
  607. self.selectQuery = function () {
  608. window.clearTimeout(waitForRendered);
  609. waitForRendered = window.setTimeout(function(){
  610. if (self.draggedQuery()){
  611. self.query(ko.utils.arrayFilter(self.queries(), function(q) {
  612. return q.id() === self.draggedQuery();
  613. })[0]);
  614. }
  615. }, 50);
  616. }
  617. self.format = ko.observable();
  618. self.columns = ko.observableArray();
  619. self.show = ko.computed(function() {
  620. if (self.inputFormat() == 'file') {
  621. return self.path().length > 0 && self.columns().length > 0;
  622. } else if (self.inputFormat() == 'table') {
  623. return self.table().length > 0;
  624. } else if (self.inputFormat() == 'query') {
  625. return self.query() && self.query().length > 0;
  626. }
  627. });
  628. };
  629. var CreateWizard = function (vm) {
  630. var self = this;
  631. var guessFieldTypesXhr;
  632. self.fileType = ko.observable();
  633. self.fileType.subscribe(function (newType) {
  634. if (self.fileFormat().format()) self.fileFormat().format().type(newType.name);
  635. });
  636. self.operationTypes = ${operators_json | n};
  637. self.fieldTypes = ${fields_json | n};
  638. self.fileTypes = ${file_types_json | n};
  639. self.show = ko.observable(true);
  640. self.showCreate = ko.observable(false);
  641. self.fileFormat = ko.observable(new IndexerFormat(vm));
  642. self.sample = ko.observableArray();
  643. self.editorId = ko.observable();
  644. self.editorVM = null;
  645. self.indexingStarted = ko.observable(false);
  646. self.isNameAvailable = ko.computed(function () {
  647. var name = self.fileFormat().name();
  648. return viewModel && viewModel.collectionNameAvailable(name) && name.length > 0;
  649. });
  650. self.readyToIndex = ko.computed(function () {
  651. var validFields = self.fileFormat().columns().length;
  652. return self.fileFormat().name().length > 0 && validFields;
  653. });
  654. self.fileFormat().format.subscribe(function () {
  655. for (var i = 0; i < self.fileTypes.length; i++) {
  656. if (self.fileTypes[i].name == self.fileFormat().format().type()) {
  657. self.fileType(self.fileTypes[i]);
  658. break;
  659. }
  660. }
  661. if (self.fileFormat().format().type) {
  662. self.fileFormat().format().type.subscribe(function (newType) {
  663. self.fileFormat().format(new FileType(newType));
  664. self.fileFormat().columns.removeAll();
  665. self.guessFieldTypes();
  666. });
  667. }
  668. });
  669. self.isGuessingFormat = ko.observable(false);
  670. self.guessFormat = function () {
  671. self.isGuessingFormat(true);
  672. self.fileFormat().columns([]);
  673. $.post("${ url('indexer:guess_format') }", {
  674. "fileFormat": ko.mapping.toJSON(self.fileFormat)
  675. }, function (resp) {
  676. var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
  677. self.fileFormat().format(newFormat);
  678. self.guessFieldTypes();
  679. self.isGuessingFormat(false);
  680. viewModel.wizardEnabled(true);
  681. viewModel.currentStep(2);
  682. }).fail(function (xhr, textStatus, errorThrown) {
  683. $(document).trigger("error", xhr.responseText);
  684. viewModel.isLoading(false);
  685. });
  686. };
  687. self.isGuessingFieldTypes = ko.observable(false);
  688. self.guessFieldTypes = function () {
  689. if (guessFieldTypesXhr) {
  690. guessFieldTypesXhr.abort();
  691. }
  692. self.isGuessingFieldTypes(true);
  693. guessFieldTypesXhr = $.post("${ url('indexer:guess_field_types') }", {
  694. "fileFormat": ko.mapping.toJSON(self.fileFormat)
  695. }, function (resp) {
  696. resp.columns.forEach(function (entry, i, arr) {
  697. arr[i] = loadField(entry);
  698. });
  699. self.fileFormat().columns(resp.columns);
  700. self.isGuessingFieldTypes(false);
  701. self.sample(resp.sample);
  702. }).fail(function (xhr, textStatus, errorThrown) {
  703. $(document).trigger("error", xhr.responseText);
  704. self.isGuessingFieldTypes(false);
  705. viewModel.isLoading(false);
  706. });
  707. };
  708. self.isIndexing = ko.observable(false);
  709. self.indexingError = ko.observable(false);
  710. self.indexingSuccess = ko.observable(false);
  711. self.indexFile = function () {
  712. if (!self.readyToIndex()) return;
  713. self.indexingStarted(true);
  714. viewModel.isLoading(true);
  715. self.isIndexing(true);
  716. $.post("${ url('indexer:index_file') }", {
  717. "fileFormat": ko.mapping.toJSON(self.fileFormat)
  718. }, function (resp) {
  719. self.showCreate(true);
  720. self.editorId(resp.history_id);
  721. $('#notebook').html($('#notebook-progress').html());
  722. self.editorVM = new EditorViewModel(resp.history_uuid, '', {
  723. user: '${ user.username }',
  724. userId: ${ user.id },
  725. languages: [{name: "Java SQL", type: "java"}],
  726. snippetViewSettings: {
  727. java : {
  728. snippetIcon: 'fa-file-archive-o '
  729. }
  730. }
  731. });
  732. ko.cleanNode($("#notebook")[0]);
  733. ko.applyBindings(self.editorVM, $("#notebook")[0]);
  734. self.editorVM.openNotebook(resp.history_uuid, null, true, function(){
  735. self.editorVM.selectedNotebook().snippets()[0].progress.subscribe(function(val){
  736. if (val == 100){
  737. self.indexingStarted(false);
  738. self.isIndexing(false);
  739. self.indexingSuccess(true);
  740. }
  741. });
  742. self.editorVM.selectedNotebook().snippets()[0].status.subscribe(function(val){
  743. if (val == 'failed'){
  744. self.isIndexing(false);
  745. self.indexingStarted(false);
  746. self.indexingError(true);
  747. }
  748. });
  749. });
  750. viewModel.isLoading(false);
  751. }).fail(function (xhr, textStatus, errorThrown) {
  752. $(document).trigger("error", xhr.responseText);
  753. viewModel.isLoading(false);
  754. self.indexingStarted(false);
  755. self.isIndexing(false);
  756. });
  757. }
  758. self.removeOperation = function (operation, operationList) {
  759. operationList.remove(operation);
  760. }
  761. self.addOperation = function (field) {
  762. field.operations.push(new Operation("split"));
  763. }
  764. self.load = function (state) {
  765. self.fileFormat().name(state.name);
  766. self.fileFormat().show(state.show);
  767. self.fileFormat().path(state.path);
  768. self.fileFormat().columns.removeAll();
  769. if (state.format && 'type' in state.format) {
  770. var koFormat = ko.mapping.fromJS(new FileType(state.format.type, state.format));
  771. self.fileFormat().format(koFormat);
  772. }
  773. if (state.columns) state.columns.forEach(function (currCol) {
  774. self.fileFormat().columns.push(loadField(currCol));
  775. });
  776. }
  777. };
  778. var loadField = function (currField) {
  779. var koField = ko.mapping.fromJS(currField);
  780. koField.operations.removeAll();
  781. currField.operations.forEach(function (operationData) {
  782. var operation = new Operation(operationData.type);
  783. operation.load(operationData);
  784. koField.operations.push(operation);
  785. });
  786. return koField;
  787. }
  788. var IndexerViewModel = function (options) {
  789. var self = this;
  790. self.apiHelper = ApiHelper.getInstance(options);
  791. self.assistAvailable = ko.observable(true);
  792. self.isLeftPanelVisible = ko.observable();
  793. self.apiHelper.withTotalStorage('assist', 'assist_panel_visible', self.isLeftPanelVisible, true);
  794. // wizard related
  795. self.wizardEnabled = ko.observable(false);
  796. self.currentStep = ko.observable(1);
  797. self.previousStepVisible = ko.pureComputed(function(){
  798. return self.currentStep() > 1;
  799. });
  800. self.nextStepVisible = ko.pureComputed(function(){
  801. return self.currentStep() < 3 && self.wizardEnabled();
  802. });
  803. self.nextStep = function () {
  804. if (self.nextStepVisible()){
  805. self.currentStep(self.currentStep()+1);
  806. }
  807. }
  808. self.previousStep = function () {
  809. if (self.previousStepVisible()){
  810. self.currentStep(self.currentStep()-1);
  811. }
  812. }
  813. self.collections = ${ indexes_json | n }.
  814. filter(function (index) {
  815. return index.type == 'collection';
  816. });
  817. self.createWizard = new CreateWizard(self);
  818. self.isLoading = ko.observable(false);
  819. self.collectionNameAvailable = function (name) {
  820. var matchingCollections = self.collections.filter(function (collection) {
  821. return collection.name == name;
  822. });
  823. return matchingCollections.length == 0;
  824. }
  825. };
  826. var viewModel;
  827. $(document).ready(function () {
  828. var options = {
  829. user: '${ user.username }',
  830. i18n: {
  831. errorLoadingDatabases: "${ _('There was a problem loading the databases') }",
  832. errorLoadingTablePreview: "${ _('There was a problem loading the table preview.') }"
  833. }
  834. }
  835. viewModel = new IndexerViewModel(options);
  836. ko.applyBindings(viewModel);
  837. var draggableMeta = {};
  838. huePubSub.subscribe('draggable.text.meta', function (meta) {
  839. draggableMeta = meta;
  840. });
  841. huePubSub.subscribe('split.panel.resized', function () {
  842. $('.form-actions').width($('.right-panel').width() - 50);
  843. });
  844. $('.form-actions').width($('.right-panel').width() - 50);
  845. $('.right-panel').droppable({
  846. accept: ".draggableText",
  847. drop: function (e, ui) {
  848. var text = ui.helper.text();
  849. var generatedName = 'idx';
  850. switch (draggableMeta.type){
  851. case 'sql':
  852. if (draggableMeta.table !== ''){
  853. generatedName += draggableMeta.table;
  854. viewModel.createWizard.fileFormat().inputFormat('table');
  855. viewModel.createWizard.fileFormat().table(draggableMeta.table);
  856. }
  857. break;
  858. case 'hdfs':
  859. generatedName += draggableMeta.definition.name;
  860. viewModel.createWizard.fileFormat().inputFormat('file');
  861. viewModel.createWizard.fileFormat().path(draggableMeta.definition.path);
  862. break;
  863. case 'document':
  864. if (draggableMeta.definition.type === 'query-hive'){
  865. generatedName += draggableMeta.definition.name;
  866. viewModel.createWizard.fileFormat().inputFormat('query');
  867. viewModel.createWizard.fileFormat().draggedQuery(draggableMeta.definition.id);
  868. }
  869. break;
  870. }
  871. if (generatedName !== 'idx' && viewModel.createWizard.fileFormat().name() === ''){
  872. viewModel.createWizard.fileFormat().name(generatedName);
  873. }
  874. else {
  875. if (draggableMeta.table !== ''){
  876. viewModel.createWizard.fileFormat().selectQuery();
  877. }
  878. }
  879. }
  880. });
  881. });
  882. });
  883. </script>
  884. ${ commonfooter(request, messages) | n,unicode }