importer.mako 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445
  1. ## Licensed to Cloudera, Inc. under one
  2. ## or more contributor license agreements. See the NOTICE file
  3. ## distributed with this work for additional information
  4. ## regarding copyright ownership. Cloudera, Inc. licenses this file
  5. ## to you under the Apache License, Version 2.0 (the
  6. ## "License"); you may not use this file except in compliance
  7. ## with the License. You may obtain a copy of the License at
  8. ##
  9. ## http://www.apache.org/licenses/LICENSE-2.0
  10. ##
  11. ## Unless required by applicable law or agreed to in writing, software
  12. ## distributed under the License is distributed on an "AS IS" BASIS,
  13. ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. ## See the License for the specific language governing permissions and
  15. ## limitations under the License.
  16. <%!
  17. from desktop import conf
  18. from desktop.views import commonheader, commonfooter, commonshare, commonimportexport
  19. from django.utils.translation import ugettext as _
  20. %>
  21. <%namespace name="actionbar" file="actionbar.mako" />
  22. <%namespace name="assist" file="/assist.mako" />
  23. %if not is_embeddable:
  24. ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
  25. # Todo lot of those
  26. <script src="${ static('desktop/js/autocomplete/sql.js') }"></script>
  27. <script src="${ static('desktop/js/sqlAutocompleter.js') }"></script>
  28. <script src="${ static('desktop/js/sqlAutocompleter2.js') }"></script>
  29. <script src="${ static('desktop/js/hdfsAutocompleter.js') }"></script>
  30. <script src="${ static('desktop/js/autocompleter.js') }"></script>
  31. <script src="${ static('desktop/js/hue.json.js') }"></script>
  32. <script src="${ static('desktop/js/jquery.hiveautocomplete.js') }" type="text/javascript" charset="utf-8"></script>
  33. <script src="${ static('desktop/ext/js/jquery/plugins/jquery-ui-1.10.4.custom.min.js') }"></script>
  34. <script src="${ static('desktop/js/jquery.huedatatable.js') }"></script>
  35. <script src="${ static('desktop/ext/js/d3.v3.js') }" type="text/javascript" charset="utf-8"></script>
  36. <script src="${ static('desktop/ext/js/selectize.min.js') }"></script>
  37. <script src="${ static('desktop/js/apiHelper.js') }"></script>
  38. <script src="${ static('metastore/js/metastore.ko.js') }"></script>
  39. <script src="${ static('desktop/js/ko.charts.js') }"></script>
  40. <script src="${ static('desktop/ext/js/knockout-sortable.min.js') }"></script>
  41. <script src="${ static('desktop/js/ko.editable.js') }"></script>
  42. <script src="${ static('desktop/ext/js/selectize.min.js') }"></script>
  43. <script src="${ static('desktop/ext/js/knockout-selectize.js') }"></script>
  44. ${ assist.assistJSModels() }
  45. <script src="${ static('notebook/js/notebook.ko.js') }"></script>
  46. <link rel="stylesheet" href="${ static('notebook/css/notebook.css') }">
  47. <link rel="stylesheet" href="${ static('notebook/css/notebook-layout.css') }">
  48. ${ assist.assistPanel() }
  49. <link rel="stylesheet" href="${ static('desktop/ext/css/selectize.css') }">
  50. %endif
  51. <link rel="stylesheet" href="${ static('desktop/css/wizard.css') }">
  52. <style type="text/css">
  53. % if conf.CUSTOM.BANNER_TOP_HTML.get():
  54. .show-assist {
  55. top: 110px!important;
  56. }
  57. .main-content {
  58. top: 112px!important;
  59. }
  60. % endif
  61. .path {
  62. margin-bottom: 0!important;
  63. border-right: none!important;
  64. }
  65. .step1 label div {
  66. width: 50px;
  67. display: inline-block;
  68. }
  69. .step2.form-inline input[type='text'], .step2.form-inline select {
  70. margin-right: 10px;
  71. margin-left: 3px;
  72. }
  73. .step2.form-inline input[type='checkbox'] {
  74. margin-left: 10px!important;
  75. margin-right: 4px!important;
  76. }
  77. .field {
  78. padding: 4px;
  79. padding-left: 10px;
  80. border-left: 4px solid #DBE8F1;
  81. }
  82. .operation {
  83. border-left: 4px solid #EEE;
  84. padding-left: 10px;
  85. margin-top: 5px;
  86. margin-bottom: 5px;
  87. margin-left: 10px;
  88. }
  89. .content-panel {
  90. overflow-x: hidden;
  91. }
  92. .fileChooserBtn {
  93. height: 29px;
  94. }
  95. .form-control.path {
  96. vertical-align: top;
  97. }
  98. .form-actions {
  99. position: fixed;
  100. bottom: 0;
  101. margin: 0;
  102. }
  103. #importerNotebook {
  104. height: 5px;
  105. margin-top: 10px;
  106. }
  107. </style>
  108. <span id="importerComponents" class="notebook">
  109. <div class="navbar navbar-inverse navbar-fixed-top">
  110. <div class="navbar-inner">
  111. <div class="container-fluid">
  112. <div class="nav-collapse">
  113. <ul class="nav">
  114. <li class="currentApp">
  115. <a href="${ url('indexer:importer') }">
  116. <i class="fa fa-database app-icon"></i> ${_('Importer')}</a>
  117. </a>
  118. </li>
  119. </ul>
  120. </div>
  121. </div>
  122. </div>
  123. </div>
  124. %if not is_embeddable:
  125. <a title="${_('Toggle Assist')}" class="pointer show-assist" data-bind="visible: !$root.isLeftPanelVisible() && $root.assistAvailable(), click: function() { $root.isLeftPanelVisible(true); }">
  126. <i class="fa fa-chevron-right"></i>
  127. </a>
  128. %endif
  129. <div class="main-content">
  130. <div class="vertical-full container-fluid" data-bind="style: { 'padding-left' : $root.isLeftPanelVisible() ? '0' : '20px' }">
  131. <div class="vertical-full">
  132. <div class="vertical-full row-fluid panel-container">
  133. %if not is_embeddable:
  134. <div class="assist-container left-panel" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable()">
  135. <a title="${_('Toggle Assist')}" class="pointer hide-assist" data-bind="click: function() { $root.isLeftPanelVisible(false) }">
  136. <i class="fa fa-chevron-left"></i>
  137. </a>
  138. <div class="assist" data-bind="component: {
  139. name: 'assist-panel',
  140. params: {
  141. user: '${user.username}',
  142. onlySql: false,
  143. sql: {
  144. sourceTypes: [{
  145. name: 'hive',
  146. type: 'hive'
  147. }],
  148. navigationSettings: {
  149. openItem: false,
  150. showStats: true
  151. }
  152. },
  153. visibleAssistPanels: ['sql', 'hdfs', 'documents']
  154. }
  155. }"></div>
  156. </div>
  157. <div class="resizer" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable(), splitDraggable : { appName: 'notebook', leftPanelVisible: $root.isLeftPanelVisible }"><div class="resize-bar">&nbsp;</div></div>
  158. %endif
  159. <div class="content-panel">
  160. <div style="margin: 10px; margin-bottom: 100px">
  161. <!-- ko template: 'create-index-wizard' --><!-- /ko -->
  162. </div>
  163. </div>
  164. </div>
  165. </div>
  166. </div>
  167. </div>
  168. <div id="chooseFile" class="modal hide fade">
  169. <div class="modal-header">
  170. <a href="#" class="close" data-dismiss="modal">&times;</a>
  171. <h3>${_('Choose a file')}</h3>
  172. </div>
  173. <div class="modal-body">
  174. <div id="filechooser"></div>
  175. </div>
  176. <div class="modal-footer"></div>
  177. </div>
  178. <script type="text/html" id="create-index-wizard">
  179. <div data-bind="visible: createWizard.show">
  180. <ol class="list-inline text-center step-indicator">
  181. <li data-bind="css: { 'active': currentStep() == 1, 'active': currentStep() > 1 }, click: function() { currentStep(1) }">
  182. <div class="step" title="${ _('Go to Step 1') }">
  183. <!-- ko if: currentStep() == 1 -->
  184. <!-- ko if: createWizard.isGuessingFormat -->
  185. <span class="fa fa-spinner fa-spin"></span>
  186. <!-- /ko -->
  187. <!-- ko ifnot: createWizard.isGuessingFormat -->
  188. 1
  189. <!-- /ko -->
  190. <!-- /ko -->
  191. <!-- ko ifnot: currentStep() == 1 -->
  192. <span class="fa fa-check"></span>
  193. <!-- /ko -->
  194. </div>
  195. <div class="caption">${ _('Pick data from ') }<span data-bind="text: createWizard.source.inputFormat"></span></div>
  196. </li>
  197. <li data-bind="css: { 'inactive': currentStep() == 1, 'active': currentStep() == 2, 'complete': currentStep() == 3 }, click: function() { currentStep(2) }">
  198. <div class="step" title="${ _('Go to Step 2') }">
  199. <!-- ko if: currentStep() < 2 -->
  200. <!-- ko if: createWizard.isGuessingFieldTypes -->
  201. <span class="fa fa-spinner fa-spin"></span>
  202. <!-- /ko -->
  203. <!-- ko ifnot: createWizard.isGuessingFieldTypes -->
  204. 2
  205. <!-- /ko -->
  206. <!-- /ko -->
  207. </div>
  208. <div class="caption">${ _('Move it to ') }<span data-bind="text: createWizard.destination.ouputFormat"></span></div>
  209. </li>
  210. </ol>
  211. <!-- ko if: currentStep() == 1 -->
  212. <div class="card step1">
  213. <h3 class="card-heading simple">${_('Source')}</h3>
  214. <div class="card-body">
  215. <form class="form-inline">
  216. <div>
  217. <div class="control-group">
  218. <label for="collectionType" class="control-label"><div>${ _('Type') }</div>
  219. <select id="collectionType" data-bind="options: createWizard.source.inputFormats, value: createWizard.source.inputFormat, optionsText: 'name', optionsValue: 'value'"></select>
  220. </label>
  221. </div>
  222. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'file'">
  223. <label for="path" class="control-label"><div>${ _('Path') }</div>
  224. <input type="text" class="form-control path input-xxlarge" data-bind="value: createWizard.source.path, filechooser: createWizard.source.path, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }" placeholder="${ _('Click or drag & drop') }">
  225. </label>
  226. </div>
  227. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'table'">
  228. <label for="path" class="control-label"><div>${ _('Table') }</div>
  229. <input type="text" data-bind="value: createWizard.source.table, hivechooser: createWizard.source.table, skipColumns: true" placeholder="${ _('Table name or <database>.<table>') }">
  230. </label>
  231. </div>
  232. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'query'">
  233. <label for="path" class="control-label"><div>${ _('Query') }</div>
  234. <select data-bind="options: createWizard.source.queries, value: createWizard.source.query, optionsText: 'name', optionsAfterRender: createWizard.source.selectQuery"></select>
  235. </label>
  236. </div>
  237. </div>
  238. <!-- ko if: createWizard.source.show() && createWizard.source.inputFormat() != 'manual' -->
  239. <h3 class="card-heading simple">${_('Format')}</h3>
  240. <div class="card-body">
  241. <label data-bind="visible: ! createWizard.prefill.source_type">${_('File Type')} <select data-bind="options: $root.createWizard.fileTypes, optionsText: 'description', value: $root.createWizard.fileType"></select></label>
  242. <span data-bind="with: createWizard.source.format, visible: createWizard.source.show">
  243. <!-- ko template: {name: 'format-settings'} --> <!-- /ko -->
  244. </span>
  245. </div>
  246. <h3 class="card-heading simple">${_('Preview')}</h3>
  247. <div class="card-body">
  248. <!-- ko if: createWizard.isGuessingFieldTypes -->
  249. <i class="fa fa-spinner fa-spin"></i>
  250. <!-- /ko -->
  251. <div style="overflow: auto">
  252. <table class="table table-striped table-condensed" style="margin:auto;text-align:left">
  253. <thead>
  254. <tr data-bind="foreach: createWizard.source.sampleCols">
  255. ##<!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  256. <th data-bind="text: name" style="padding-right:60px"></th>
  257. ## TODO nested
  258. </tr>
  259. </thead>
  260. <tbody data-bind="foreach: createWizard.source.sample">
  261. <tr data-bind="foreach: $data">
  262. ##<!-- ko if: $index() < $root.createWizard.source.columns().length -->
  263. ##<td data-bind="visible: $root.createWizard.source.columns()[$index()].keep, text: $data"></td>
  264. <td data-bind="text: $data"></td>
  265. ##<!-- ko with: $root.createWizard.source.columns()[$index()] -->
  266. ## <!-- ko template: 'output-generated-field-data-template' --> <!-- /ko -->
  267. ##<!-- /ko -->
  268. ##<!-- /ko -->
  269. </tr>
  270. </tbody>
  271. </table>
  272. </div>
  273. </div>
  274. <!-- /ko -->
  275. </form>
  276. </div>
  277. </div>
  278. <!-- /ko -->
  279. <!-- ko if: currentStep() == 2 -->
  280. <div class="card step2">
  281. <!-- ko with: createWizard.destination -->
  282. <div class="control-group">
  283. <label for="collectionType" class="control-label" data-bind="visible: ! $parent.createWizard.prefill.target_type"><div>${ _('Type') }</div>
  284. <select id="collectionType" data-bind="options: ouputFormats, value: ouputFormat, optionsValue: 'value', optionsText: 'name'"></select>
  285. </label>
  286. <label for="collectionName" class="control-label"><div>${ _('Name') }</div>
  287. <!-- ko if: ouputFormat() != 'table' && ouputFormat() != 'database' -->
  288. <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="value: name, valueUpdate: 'afterkeydown'" placeholder="${ _('Name') }">
  289. <!-- /ko -->
  290. <!-- ko if: ouputFormat() == 'table' || ouputFormat() == 'database' -->
  291. <label for="path" class="control-label">
  292. <input type="text" data-bind="value: name, hivechooser: name, skipColumns: true, valueUpdate: 'afterkeydown'" placeholder="${ _('Table name or <database>.<table>') }">
  293. </label>
  294. <!-- /ko -->
  295. <span class="help-inline muted" data-bind="visible: ! isTargetExisting()">
  296. ${ _('Create a new ') } <span data-bind="text: ouputFormat"></span>
  297. </span>
  298. <span class="help-inline muted" data-bind="visible: isTargetExisting()">
  299. ${ _('Adding data to the existing ') } <span data-bind="text: ouputFormat"></span>
  300. <a href="javascript:void(0)" data-bind="attr: { href: existingTargetUrl() }, text: name" target="_blank"></a>
  301. </span>
  302. </label>
  303. </div>
  304. <div class="card-body">
  305. ##<!-- ko if: createWizard.isGuessingFieldTypes -->
  306. ## <i class="fa fa-spinner fa-spin"></i>
  307. ##<!-- /ko -->
  308. <!-- ko if: ouputFormat() == 'table' -->
  309. <h3 class="card-heading simple">${_('Properties')}</h3>
  310. <input type="text" class="form-control input-xlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
  311. <div class="control-group">
  312. <label for="collectionType" class="control-label"><div>${ _('Format') }</div>
  313. <select id="collectionType" data-bind="options: tableFormats, value: tableFormat, optionsValue: 'value', optionsText: 'name'"></select>
  314. </label>
  315. </div>
  316. <label class="checkbox">
  317. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Default location')}
  318. </label>
  319. <span data-bind="visible: ! useDefaultLocation()">
  320. <label for="path" class="control-label"><div>${ _('External location') }</div>
  321. <input type="text" class="form-control path input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  322. </label>
  323. </span>
  324. <label class="checkbox">
  325. <input type="checkbox" data-bind="checked: importData, disable: ! useDefaultLocation() && $parent.createWizard.source.path() == nonDefaultLocation();"> ${_('Import data')}
  326. </label>
  327. <label class="checkbox">
  328. <input type="checkbox" data-bind="checked: useCustomDelimiters"> ${_('Custom char delimiters')}
  329. </label>
  330. <span data-bind="visible: useCustomDelimiters">
  331. <div class="control-group">
  332. <label for="fieldDelimiter" class="control-label"><div>${ _('Field') }</div>
  333. <select id="fieldDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { create: true, maxLength: 2 }, value: customFieldDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  334. </label>
  335. </div>
  336. <div class="control-group">
  337. <label for="collectionDelimiter" class="control-label"><div>${ _('Array, Map') }</div>
  338. <select id="collectionDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { create: true, maxLength: 2 }, value: customCollectionDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  339. </label>
  340. </div>
  341. <div class="control-group">
  342. <label for="structDelimiter" class="control-label"><div>${ _('Struct') }</div>
  343. <select id="structDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { create: true, maxLength: 2 }, value: customMapDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  344. </label>
  345. </div>
  346. <input type="text" data-bind="value: customRegexp"> ${_('Regexp')}
  347. </span>
  348. <div class="control-group" data-bind="visible: tableFormat() == 'kudu'">
  349. <label for="kuduPks" class="control-label"><div>${ _('Primary keys') }</div>
  350. ## At least one selected
  351. <select id="kuduPks" data-bind="options: columns, selectedOptions: primaryKeys, optionsValue: 'name', optionsText: 'name'" size="3" multiple="true"></select>
  352. </label>
  353. </div>
  354. <div class="row" style="margin-left: 8px">
  355. <div class="span3">
  356. <input type="checkbox" data-bind="checked: hasHeader">
  357. ${_('Use first row as column names')} <a class="btn disable-feedback"><i class="fa fa-outdent"></i></a>
  358. </div>
  359. <div class="span3" data-bind="click: function() { alert('Hello'); }">
  360. ${ _('Bulk edit column names') }<a class="btn"><i class="fa fa-edit"></i></a>
  361. </div>
  362. </div>
  363. <label for="tablePartitions" class="control-label"><div>${ _('Partitions') }</div>
  364. <!-- ko if: tableFormat() != 'kudu' -->
  365. <div class="form-inline" data-bind="foreach: partitionColumns">
  366. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field"></div>
  367. <a data-bind="click: function() { $parent.partitionColumns.remove($data); }"><i class="fa fa-minus"></i></a>
  368. </div>
  369. <a data-bind="click: function() { partitionColumns.push($root.loadField({operations: [], nested: [], name: '', level: 0, type: '', showProperties: false, isPartition: true})); }" class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Add partition')}</a>
  370. <!-- /ko -->
  371. <!-- ko if: tableFormat() == 'kudu' -->
  372. <div class="form-inline" data-bind="foreach: kuduPartitionColumns">
  373. ${ _('Columns') }
  374. <select id="kuduPks" data-bind="options: $parent.primaryKeys, selectedOptions: columns" size="3" multiple="true"></select>
  375. <select data-bind="options: ['RANGE BY', 'HASH'], value: name"></select>
  376. <!-- ko if: name() == 'HASH' -->
  377. <input type="number" data-bind="value: int_val">
  378. <!-- /ko -->
  379. <!-- ko if: name() == 'RANGE BY' -->
  380. <div class="form-inline" data-bind="foreach: range_partitions">
  381. <!-- ko if: name() == 'VALUES' -->
  382. <input type="input" data-bind="value: lower_val">
  383. <input type="checkbox" data-bind="checked: include_lower_val">
  384. <
  385. <span data-bind="text: '=', visible: include_lower_val"></span>
  386. <select data-bind="options: ['VALUES', 'VALUE'], value: name"></select>
  387. <
  388. <span data-bind="text: '=', visible: include_upper_val"></span>
  389. <input type="input" data-bind="value: upper_val">
  390. <input type="checkbox" data-bind="checked: include_upper_val">
  391. <!-- /ko -->
  392. <!-- ko if: name() == 'VALUE' -->
  393. <select data-bind="options: ['VALUES', 'VALUE'], value: name"></select>
  394. <div class="form-inline" data-bind="foreach: values">
  395. <input type="text" data-bind="value: $data">
  396. <a data-bind="click: function() { $parent.values.remove($data); }"><i class="fa fa-minus"></i></a>
  397. </div>
  398. <a data-bind="click: function() { values.push(''); }"><i class="fa fa-plus"></i></a>
  399. <!-- /ko -->
  400. <a data-bind="click: function() { $parent.range_partitions.remove($data); }"><i class="fa fa-minus"></i></a>
  401. </div>
  402. <a data-bind="click: function() { range_partitions.push(ko.mapping.fromJS({values: [''], name: 'VALUES', lower_val: 0, include_lower_val: true, upper_val: 1, include_upper_val: true})); }"><i class="fa fa-plus"></i></a>
  403. <!-- /ko -->
  404. <a data-bind="click: function() { $parent.kuduPartitionColumns.remove($data); }"><i class="fa fa-minus"></i></a>
  405. <a data-bind="click: function() { $parent.kuduPartitionColumns.push(ko.mapping.fromJS({columns: [], range_partitions: [], name: 'HASH', int_val: 16})); }" class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Add partition')}</a>
  406. <!-- /ko -->
  407. </div>
  408. <a data-bind="click: function() { kuduPartitionColumns.push(ko.mapping.fromJS({columns: [], range_partitions: [], name: 'HASH', int_val: 16})); }" class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Add partition')}</a>
  409. <!-- /ko -->
  410. </label>
  411. <!-- /ko -->
  412. <!-- ko if: ouputFormat() == 'table' || ouputFormat() == 'index' -->
  413. <h3 class="card-heading simple">${_('Fields')}</h3>
  414. <form class="form-inline" data-bind="foreach: columns">
  415. <!-- ko if: $parent.ouputFormat() == 'table' -->
  416. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field"></div>
  417. <!-- /ko -->
  418. <!-- ko if: $parent.ouputFormat() == 'index' -->
  419. <div data-bind="template: { name: 'index-field-template', data: $data }" class="margin-top-10 field"></div>
  420. <!-- /ko -->
  421. </form>
  422. <!-- ko if: $parent.createWizard.source.inputFormat() == 'manual' && ouputFormat() == 'table' -->
  423. <a data-bind="click: function() { columns.push($root.loadField({operations: [], nested: [], name: '', level: 0, type: '', showProperties: false, isPartition: true})); }" class="pointer margin-left-20" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Add Field')}</a>
  424. <!-- /ko -->
  425. <!-- /ko -->
  426. <!-- ko if: ouputFormat() == 'database' -->
  427. <input type="text" class="form-control input-xlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
  428. <label class="checkbox">
  429. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Default location')}
  430. </label>
  431. <span data-bind="visible: ! useDefaultLocation()">
  432. <label for="path" class="control-label"><div>${ _('External location') }</div>
  433. <input type="text" class="form-control path input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  434. </label>
  435. </span>
  436. <!-- /ko -->
  437. </div>
  438. <!-- /ko -->
  439. </div>
  440. <!-- /ko -->
  441. <div class="form-actions">
  442. <!-- ko if: previousStepVisible -->
  443. <button class="btn" data-bind="click: previousStep">${ _('Back') }</button>
  444. <!-- /ko -->
  445. <!-- ko if: currentStep() == 1 && createWizard.source.show -->
  446. <button class="btn" data-bind="click: function() { currentStep(2); }">
  447. ${_('Next')} <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.isGuessingFormat"></i>
  448. </button>
  449. <!-- /ko -->
  450. <!-- ko if: currentStep() == 2 -->
  451. <button href="javascript:void(0)" class="btn btn-primary disable-feedback" data-bind="click: createWizard.indexFile, enable: createWizard.readyToIndex() && ! createWizard.indexingStarted()">
  452. ${ _('Submit') } <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.indexingStarted"></i>
  453. </button>
  454. <!-- /ko -->
  455. <span data-bind="visible: createWizard.editorId">
  456. <a href="javascript:void(0)" class="btn btn-success" data-bind="attr: {href: '${ url('notebook:editor') }?editor=' + createWizard.editorId() }" target="_blank" title="${ _('Open') }">
  457. ${_('Status')}
  458. </a>
  459. ${ _('View collection') } <a href="javascript:void(0)" data-bind="attr: {href: '${ url("indexer:collections") }' +'#edit/' + createWizard.source.name() }, text: createWizard.source.name" target="_blank"></a>
  460. </span>
  461. <div id="importerNotebook"></div>
  462. </div>
  463. </div>
  464. </script>
  465. <script type="text/html" id="format-settings">
  466. <!-- ko foreach: {data: getArguments(), as: 'argument'} -->
  467. <!-- ko template: {name: 'arg-' + argument.type, data:{description: argument.description, value: $parent[argument.name]}}--><!-- /ko -->
  468. <!-- /ko -->
  469. </script>
  470. <script type="text/html" id="table-field-template">
  471. <label data-bind="visible: level() == 0 || ($parent.type() != 'array' && $parent.type() != 'map')">${ _('Name') }
  472. <input type="text" class="input-large" placeholder="${ _('Field name') }" data-bind="value: name">
  473. </label>
  474. <label>${ _('Type') }
  475. <!-- ko if: ! (level() > 0 && $parent.type() == 'map') -->
  476. <select class="input-small" data-bind="options: $root.createWizard.hiveFieldTypes, value: type"></select>
  477. <!-- /ko -->
  478. <!-- ko if: level() > 0 && $parent.type() == 'map' -->
  479. <select class="input-small" data-bind="options: $root.createWizard.hivePrimitiveFieldTypes, value: keyType"></select>
  480. <select class="input-small" data-bind="options: $root.createWizard.hiveFieldTypes, value: type"></select>
  481. <!-- /ko -->
  482. <input type="number" class="input-small" placeholder="${ _('Length') }" data-bind="value: length, visible: type() == 'varchar' || type() == 'char'">
  483. <!-- ko if: type() == 'array' || type() == 'map' || type() == 'struct' -->
  484. <div data-bind="template: { name: 'table-field-template', foreach: nested }">
  485. </div>
  486. <a data-bind="click: function() { nested.push($root.loadField({operations: [], nested: [], name: '', type: '', 'keyType': '', level: level() + 1})); }, visible: type() == 'struct'"><i class="fa fa-plus"></i></a>
  487. <!-- /ko -->
  488. </label>
  489. <span data-bind="visible: level() == 0 || ($parent.type() != 'array' && $parent.type() != 'map')">
  490. ${_('Comment')}
  491. </span>
  492. <!-- ko if: level() > 0 && $parent.type() == 'struct' && $parent.nested().length > 1 -->
  493. <a data-bind="click: function() { $parent.nested.remove($data); }"><i class="fa fa-minus"></i></a>
  494. <!-- /ko -->
  495. <!-- ko if: level() == 0 && typeof isPartition === 'undefined'-->
  496. <label data-bind="text: $root.createWizard.source.sample()[0][$index()]"></label>
  497. <label data-bind="text: $root.createWizard.source.sample()[1][$index()]"></label>
  498. <!-- /ko -->
  499. </script>
  500. <script type="text/html" id="index-field-template">
  501. <label>${ _('Name') }
  502. <input type="text" class="input-large" placeholder="${ _('Field name') }" data-bind="value: name">
  503. </label>
  504. <label>${ _('Type') }
  505. <select class="input-small" data-bind="options: $root.createWizard.fieldTypes, value: type"></select>
  506. </label>
  507. <a href="javascript:void(0)" title="${ _('Show field properties') }" data-bind="click: function() {showProperties(! showProperties()) }">
  508. <i class="fa fa-sliders"></i>
  509. </a>
  510. <span data-bind="visible: showProperties">
  511. <label class="checkbox">
  512. <input type="checkbox" data-bind="checked: unique"> ${_('Unique')}
  513. </label>
  514. <label class="checkbox">
  515. <input type="checkbox" data-bind="checked: keep"> ${_('Keep in index')}
  516. </label>
  517. <label class="checkbox">
  518. <input type="checkbox" data-bind="checked: required"> ${_('Required')}
  519. </label>
  520. </span>
  521. <!-- ko if: operations().length == 0 -->
  522. <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation')}</a>
  523. <!-- /ko -->
  524. <div data-bind="foreach: operations">
  525. <div data-bind="template: { name:'operation-template',data:{operation: $data, list: $parent.operations}}"></div>
  526. </div>
  527. <!-- ko if: operations().length > 0 -->
  528. <a class="pointer" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation to')} <span data-bind="text: name"></span></a>
  529. <!-- /ko -->
  530. </script>
  531. <script type="text/html" id="operation-template">
  532. <div class="operation">
  533. <select data-bind="options: $root.createWizard.operationTypes.map(function(o){return o.name}), value: operation.type"></select>
  534. <!-- ko template: "args-template" --><!-- /ko -->
  535. <!-- ko if: operation.settings().outputType() == "custom_fields" -->
  536. <label> ${ _('Number of expected fields') }
  537. <input type="number" class="input-mini" data-bind="value: operation.numExpectedFields">
  538. </label>
  539. <!-- /ko -->
  540. <a class="pointer margin-left-20" data-bind="click: function(){$root.createWizard.removeOperation(operation, list)}" title="${ _('Remove') }"><i class="fa fa-times"></i></a>
  541. <div class="margin-left-20" data-bind="foreach: operation.fields">
  542. <div data-bind="template: { name:'index-field-template', data:$data }" class="margin-top-10 field"></div>
  543. </div>
  544. </div>
  545. </script>
  546. <script type="text/html" id="args-template">
  547. <!-- ko foreach: {data: operation.settings().getArguments(), as: 'argument'} -->
  548. <!-- ko template: {name: 'arg-' + argument.type, data: {description: argument.description, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
  549. <!-- /ko -->
  550. </script>
  551. <script type="text/html" id="arg-text">
  552. <label>
  553. <span data-bind="text: description"></span>
  554. <input type="text" class="input-small" data-bind="attr: {placeholder: description}, value: value">
  555. </label>
  556. </script>
  557. <script type="text/html" id="arg-text-delimiter">
  558. <label>
  559. <span data-bind="text: description"></span>
  560. <select id="fieldDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { create: true, maxLength: 2 }, value: value, optionsValue: 'value', optionsText: 'name', attr: {placeholder: description}"></select>
  561. </label>
  562. </script>
  563. <script type="text/html" id="arg-checkbox">
  564. <label>
  565. <input type="checkbox" data-bind="checked: value">
  566. <span data-bind="text: description"></span>
  567. </label>
  568. </script>
  569. <script type="text/html" id="arg-mapping">
  570. <!-- ko foreach: value-->
  571. <div>
  572. <input type="text" data-bind="value: key, attr: {placeholder: 'key'}">
  573. <input type="text" data-bind="value: value, attr: {placeholder: 'value'}">
  574. <button class="btn" data-bind="click: function(){$parent.value.remove($data)}">${_('Remove Pair')}</button>
  575. </div>
  576. <!-- /ko -->
  577. <button class="btn" data-bind="click: function(){value.push({key: ko.observable(''), value: ko.observable('')})}">${_('Add Pair')}</button>
  578. <br>
  579. </script>
  580. <script type="text/html" id="field-preview-header-template">
  581. <th data-bind="visible: keep, text: name" style="padding-right:60px"></th>
  582. <!-- ko foreach: operations -->
  583. <!--ko foreach: fields -->
  584. <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  585. <!-- /ko -->
  586. <!--/ko -->
  587. </script>
  588. <script type="text/html" id="output-generated-field-data-template">
  589. <!-- ko foreach: operations -->
  590. <!--ko foreach: fields -->
  591. <td data-bind="visible: keep">[[${_('generated')}]]</td>
  592. <!-- ko template: 'output-generated-field-data-template' --><!-- /ko -->
  593. <!-- /ko -->
  594. <!--/ko -->
  595. </script>
  596. <script type="text/html" id="importerNotebook-progress">
  597. <!-- ko with: selectedNotebook -->
  598. <!-- ko foreach: snippets -->
  599. <div class="progress-snippet progress active" data-bind="css: {
  600. 'progress-starting': progress() == 0 && status() == 'running',
  601. 'progress-warning': progress() > 0 && progress() < 100,
  602. 'progress-success': progress() == 100,
  603. 'progress-danger': progress() == 0 && errors().length > 0}" style="background-color: #FFF; width: 100%; height: 4px">
  604. <div class="bar" data-bind="style: {'width': (errors().length > 0 ? 100 : Math.max(2, progress())) + '%'}"></div>
  605. </div>
  606. <!-- /ko -->
  607. <!-- /ko -->
  608. </script>
  609. <script type="text/javascript" charset="utf-8">
  610. (function () {
  611. ko.options.deferUpdates = true;
  612. var fieldNum = 0;
  613. var getNewFieldName = function () {
  614. fieldNum++;
  615. return "new_field_" + fieldNum
  616. };
  617. var createDefaultField = function () {
  618. var defaultField = ko.mapping.fromJS(${default_field_type | n});
  619. defaultField.name = ko.observable(getNewFieldName());
  620. return defaultField;
  621. };
  622. var Operation = function (type) {
  623. var self = this;
  624. var createArgumentValue = function (arg) {
  625. if (arg.type == "mapping") {
  626. return ko.observableArray([]);
  627. }
  628. else if (arg.type == "checkbox") {
  629. return ko.observable(false);
  630. }
  631. else {
  632. return ko.observable("");
  633. }
  634. }
  635. var constructSettings = function (type) {
  636. var settings = {};
  637. var operation = viewModel.createWizard.operationTypes.find(function (currOperation) {
  638. return currOperation.name == type;
  639. });
  640. for (var i = 0; i < operation.args.length; i++) {
  641. var argVal = createArgumentValue(operation.args[i]);
  642. if (operation.args[i].type == "checkbox" && operation.outputType == "checkbox_fields") {
  643. argVal.subscribe(function (newVal) {
  644. if (newVal) {
  645. self.fields.push(createDefaultField());
  646. }
  647. else {
  648. self.fields.pop();
  649. }
  650. });
  651. }
  652. settings[operation.args[i].name] = argVal;
  653. }
  654. settings.getArguments = function () {
  655. return operation.args
  656. };
  657. settings.outputType = function () {
  658. return operation.outputType;
  659. }
  660. return settings;
  661. };
  662. var init = function () {
  663. self.fields([]);
  664. self.numExpectedFields(0);
  665. self.numExpectedFields.subscribe(function (numExpectedFields) {
  666. if (numExpectedFields < self.fields().length) {
  667. self.fields(self.fields().slice(0, numExpectedFields));
  668. }
  669. else if (numExpectedFields > self.fields().length) {
  670. var difference = numExpectedFields - self.fields().length;
  671. for (var i = 0; i < difference; i++) {
  672. self.fields.push(createDefaultField());
  673. }
  674. }
  675. });
  676. self.settings(constructSettings(self.type()));
  677. }
  678. self.load = function (data) {
  679. self.numExpectedFields(data.numExpectedFields);
  680. var newSettings = constructSettings(data.type);
  681. for (var key in data.settings) {
  682. newSettings[key] = ko.mapping.fromJS(data.settings[key]);
  683. }
  684. self.settings(newSettings);
  685. data.fields.forEach(function (field) {
  686. self.fields.push(loadField(field));
  687. });
  688. }
  689. self.type = ko.observable(type);
  690. self.fields = ko.observableArray();
  691. self.numExpectedFields = ko.observable();
  692. self.settings = ko.observable();
  693. init();
  694. self.type.subscribe(function (newType) {
  695. init();
  696. });
  697. }
  698. var FileType = function (typeName, args) {
  699. var self = this;
  700. var type;
  701. var init = function () {
  702. self.type = ko.observable(typeName);
  703. var types = viewModel.createWizard.fileTypes;
  704. for (var i = 0; i < types.length; i++) {
  705. if (types[i].name == typeName) {
  706. type = types[i];
  707. break;
  708. }
  709. }
  710. for (var i = 0; i < type.args.length; i++) {
  711. self[type.args[i].name] = ko.observable();
  712. }
  713. if (args) loadFromObj(args);
  714. for (var i = 0; i < type.args.length; i++) {
  715. self[type.args[i].name].subscribe(viewModel.createWizard.guessFieldTypes);
  716. }
  717. }
  718. var loadFromObj = function (args) {
  719. for (var attr in args) {
  720. self[attr] = ko.mapping.fromJS(args[attr]);
  721. }
  722. }
  723. self.getArguments = function () {
  724. return type.args;
  725. }
  726. self.isCustomizable = function () {
  727. return type.isCustomizable;
  728. }
  729. init();
  730. }
  731. var Source = function (vm, wizard) {
  732. var self = this;
  733. self.name = ko.observable('');
  734. self.sample = ko.observableArray();
  735. self.sampleCols = ko.observableArray();
  736. self.inputFormat = ko.observable('file');
  737. self.inputFormat.subscribe(function(val) {
  738. if (val == 'query') {
  739. self.getDocuments();
  740. }
  741. });
  742. self.inputFormats = ko.observableArray([
  743. {'value': 'file', 'name': 'File'},
  744. {'value': 'table', 'name': 'Table'},
  745. {'value': 'text', 'name': 'Copy paste text'},
  746. {'value': 'query', 'name': 'SQL Query'},
  747. {'value': 'dbms', 'name': 'DBMS'},
  748. {'value': 'manual', 'name': 'Manually'},
  749. ]);
  750. if (wizard.prefill.source_type) {
  751. self.inputFormats([
  752. {'value': 'file', 'name': 'File'},
  753. {'value': 'manual', 'name': 'Manually'},
  754. ]);
  755. if (wizard.prefill.source_type() == 'manual') {
  756. self.inputFormat(wizard.prefill.source_type());
  757. }
  758. }
  759. // File
  760. self.path = ko.observable('');
  761. self.path.subscribe(function(val) {
  762. if (val) {
  763. vm.createWizard.guessFormat();
  764. vm.createWizard.destination.nonDefaultLocation(val);
  765. }
  766. })
  767. // Table
  768. self.table = ko.observable('');
  769. self.tableName = ko.computed(function() {
  770. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[1] : self.table();
  771. });
  772. self.databaseName = ko.computed(function() {
  773. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[0] : 'default';
  774. });
  775. // Queries
  776. self.query = ko.observable('');
  777. self.queries = ko.observableArray([]);
  778. self.draggedQuery = ko.observable();
  779. self.getDocuments = function() {
  780. $.get('/desktop/api2/docs/', {
  781. type: 'query-hive',
  782. include_trashed: false,
  783. sort: '-last_modified',
  784. limit: 100
  785. }, function(data) {
  786. if (data && data.documents) {
  787. var queries = [];
  788. $.each(data.documents, function(index, query) {
  789. queries.push(ko.mapping.fromJS(query));
  790. });
  791. self.queries(queries);
  792. }
  793. });
  794. };
  795. var waitForRendered = -1;
  796. self.selectQuery = function () {
  797. window.clearTimeout(waitForRendered);
  798. waitForRendered = window.setTimeout(function(){
  799. if (self.draggedQuery()){
  800. self.query(ko.utils.arrayFilter(self.queries(), function(q) {
  801. return q.id() === self.draggedQuery();
  802. })[0]);
  803. }
  804. }, 50);
  805. }
  806. self.format = ko.observable();
  807. self.show = ko.computed(function() {
  808. if (self.inputFormat() == 'file') {
  809. return self.path().length > 0;
  810. } else if (self.inputFormat() == 'table') {
  811. return self.table().length > 0;
  812. } else if (self.inputFormat() == 'query') {
  813. return self.query();
  814. } else if (self.inputFormat() == 'manual') {
  815. return true;
  816. }
  817. });
  818. self.defaultName = ko.computed(function() {
  819. var name = ''
  820. if (self.inputFormat() == 'file') {
  821. if (self.path()) {
  822. name = self.path().split('/').pop().split('.')[0];
  823. }
  824. } else if (self.inputFormat() == 'table') {
  825. if (self.table().split('.', 2).length == 2) {
  826. name = self.table();
  827. }
  828. } else if (self.inputFormat() == 'query') {
  829. if (self.query()) {
  830. name = val.name();
  831. }
  832. } else if (self.inputFormat() == 'manual') {
  833. name = wizard.prefill.target_path ? wizard.prefill.target_path() + '.' : '';
  834. }
  835. return name.replace(' ', '_');
  836. });
  837. self.defaultName.subscribe(function(newVal) {
  838. vm.createWizard.destination.name(newVal);
  839. });
  840. };
  841. var Destination = function (vm, wizard) {
  842. var self = this;
  843. self.name = ko.observable('');
  844. self.name.subscribe(function (name) {
  845. var exists = false;
  846. if (name.length == 0) {
  847. self.isTargetExisting(false);
  848. }
  849. else if (self.ouputFormat() == 'file') {
  850. // Todo
  851. // self.path()
  852. }
  853. else if (self.ouputFormat() == 'table') {
  854. $.get("/beeswax/api/autocomplete/" + self.databaseName() + '/' + self.tableName(), function (data) {
  855. self.isTargetExisting(data.code != 500);
  856. }).fail(function (xhr, textStatus, errorThrown) { self.isTargetExisting(false); });
  857. }
  858. else if (self.ouputFormat() == 'index') {
  859. $.post("/search/get_collection", {
  860. name: self.name()
  861. }, function (data) {
  862. self.isTargetExisting(data.status == 0);
  863. }).fail(function (xhr, textStatus, errorThrown) { self.isTargetExisting(false); });
  864. }
  865. });
  866. self.description = ko.observable('');
  867. self.ouputFormat = ko.observable('table');
  868. self.ouputFormatsList = ko.observableArray([
  869. {'name': 'Table', 'value': 'table'},
  870. {'name': 'Solr index', 'value': 'index'},
  871. {'name': 'File', 'value': 'file'},
  872. {'name': 'Database', 'value': 'database'},
  873. ]);
  874. self.ouputFormats = ko.computed(function() {
  875. return $.grep(self.ouputFormatsList(), function(format) {
  876. if (format.value == 'database' && wizard.source.inputFormat() != 'manual') {
  877. return false;
  878. }
  879. else if (format.value == 'file' && wizard.source.inputFormat() != 'manual') {
  880. return false;
  881. }
  882. return true;
  883. })
  884. });
  885. if (wizard.prefill.target_type) {
  886. self.ouputFormat(wizard.prefill.target_type());
  887. if (wizard.prefill.target_type() == 'database') {
  888. vm.currentStep(2);
  889. };
  890. }
  891. self.format = ko.observable();
  892. self.columns = ko.observableArray();
  893. self.isTargetExisting = ko.observable();
  894. self.existingTargetUrl = ko.computed(function() { // Should open generic sample popup instead
  895. if (self.isTargetExisting()) {
  896. if (self.ouputFormat() == 'file') {
  897. // Todo
  898. return '';
  899. }
  900. else if (self.ouputFormat() == 'table') {
  901. return '/metastore/table/' + self.databaseName() + '/' + self.tableName();
  902. }
  903. else if (self.ouputFormat() == 'index') {
  904. return '${ url("indexer:collections") }#edit/' + self.name();
  905. }
  906. }
  907. return '';
  908. });
  909. // Table
  910. self.tableName = ko.computed(function() {
  911. return self.ouputFormat() == 'table' && self.name().indexOf('.') > 0 ? self.name().split('.', 2)[1] : self.name();
  912. });
  913. self.databaseName = ko.computed(function() {
  914. return self.ouputFormat() == 'table' && self.name().indexOf('.') > 0 ? self.name().split('.', 2)[0] : 'default';
  915. });
  916. self.tableFormat = ko.observable('text');
  917. self.tableFormats = ko.observableArray([
  918. {'value': 'text', 'name': 'Text'},
  919. {'value': 'parquet', 'name': 'Parquet'},
  920. {'value': 'json', 'name': 'Json'},
  921. {'value': 'kudu', 'name': 'Kudu'},
  922. {'value': 'orc', 'name': 'ORC'},
  923. {'value': 'avro', 'name': 'Avro'},
  924. {'value': 'rcfile', 'name': 'RCFile'},
  925. {'value': 'sequencefile', 'name': 'SequenceFile'}
  926. ]);
  927. self.partitionColumns = ko.observableArray();
  928. self.kuduPartitionColumns = ko.observableArray();
  929. self.primaryKeys = ko.observableArray();
  930. self.importData = ko.observable(true);
  931. self.useDefaultLocation = ko.observable(true);
  932. self.nonDefaultLocation = ko.observable('');
  933. self.hasHeader = ko.observable(true);
  934. self.useCustomDelimiters = ko.observable(false);
  935. self.customFieldDelimiter = ko.observable(',');
  936. self.customCollectionDelimiter = ko.observable('\\002');
  937. self.customMapDelimiter = ko.observable('\\003');
  938. self.customRegexp = ko.observable('');
  939. // Index
  940. };
  941. var CreateWizard = function (vm) {
  942. var self = this;
  943. var guessFieldTypesXhr;
  944. self.fileType = ko.observable();
  945. self.fileType.subscribe(function (newType) {
  946. if (self.source.format()) {
  947. self.source.format().type(newType.name);
  948. }
  949. });
  950. self.operationTypes = ${operators_json | n};
  951. self.fieldTypes = ${fields_json | n}.solr;
  952. self.hivePrimitiveFieldTypes = ${fields_json | n}.hivePrimitive;
  953. self.hiveFieldTypes = ${fields_json | n}.hive;
  954. self.fileTypes = ${file_types_json | n};
  955. self.prefill = ko.mapping.fromJS(${prefill | n});
  956. self.show = ko.observable(true);
  957. self.showCreate = ko.observable(false);
  958. self.source = new Source(vm, self);
  959. self.destination = new Destination(vm, self);
  960. self.customDelimiters = ko.observable([
  961. {'value': ',', 'name': 'Comma (,)'},
  962. {'value': '\\t', 'name': '^Tab (\\t)'},
  963. {'value': '\\n', 'name': 'New line'},
  964. {'value': ' ', 'name': 'Space'},
  965. {'value': '"', 'name': 'Double Quote'},
  966. {'value': '\\001', 'name': '^A (\\001)'},
  967. {'value': '\\002', 'name': '^B (\\002)'},
  968. {'value': '\\003', 'name': '^C (\\003)'},
  969. ]);
  970. self.editorId = ko.observable();
  971. self.jobId = ko.observable();
  972. self.editorVM = null;
  973. self.indexingStarted = ko.observable(false);
  974. self.readyToIndex = ko.computed(function () {
  975. var validFields = self.destination.columns().length || self.destination.ouputFormat() == 'database';
  976. return self.destination.name().length > 0 && validFields;
  977. });
  978. self.source.format.subscribe(function () {
  979. for (var i = 0; i < self.fileTypes.length; i++) {
  980. if (self.fileTypes[i].name == self.source.format().type()) {
  981. self.fileType(self.fileTypes[i]);
  982. break;
  983. }
  984. }
  985. if (self.source.format().type) {
  986. self.source.format().type.subscribe(function (newType) {
  987. self.source.format(new FileType(newType));
  988. self.destination.columns.removeAll();
  989. self.guessFieldTypes();
  990. });
  991. }
  992. });
  993. self.isGuessingFormat = ko.observable(false);
  994. self.guessFormat = function () {
  995. self._guessIndexFormat();
  996. }
  997. self._guessIndexFormat = function () {
  998. self.isGuessingFormat(true);
  999. self.destination.columns.removeAll();
  1000. $.post("${ url('indexer:guess_format') }", {
  1001. "fileFormat": ko.mapping.toJSON(self.source)
  1002. }, function (resp) {
  1003. var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
  1004. self.source.format(newFormat);
  1005. self.guessFieldTypes();
  1006. self.isGuessingFormat(false);
  1007. viewModel.wizardEnabled(true);
  1008. }).fail(function (xhr, textStatus, errorThrown) {
  1009. $(document).trigger("error", xhr.responseText);
  1010. viewModel.isLoading(false);
  1011. });
  1012. };
  1013. self.isGuessingFieldTypes = ko.observable(false);
  1014. self.guessFieldTypes = function () {
  1015. self._guessIndexFieldTypes();
  1016. }
  1017. self._guessIndexFieldTypes = function () {
  1018. if (guessFieldTypesXhr) {
  1019. guessFieldTypesXhr.abort();
  1020. }
  1021. self.isGuessingFieldTypes(true);
  1022. guessFieldTypesXhr = $.post("${ url('indexer:guess_field_types') }", {
  1023. "fileFormat": ko.mapping.toJSON(self.source)
  1024. }, function (resp) {
  1025. resp.columns.forEach(function (entry, i, arr) {
  1026. arr[i] = loadField(entry);
  1027. });
  1028. self.source.sampleCols(resp.sample_cols);
  1029. self.source.sample(resp.sample);
  1030. self.destination.columns(resp.columns);
  1031. self.isGuessingFieldTypes(false);
  1032. }).fail(function (xhr, textStatus, errorThrown) {
  1033. $(document).trigger("error", xhr.responseText);
  1034. self.isGuessingFieldTypes(false);
  1035. viewModel.isLoading(false);
  1036. });
  1037. };
  1038. self.isIndexing = ko.observable(false);
  1039. self.indexingError = ko.observable(false);
  1040. self.indexingSuccess = ko.observable(false);
  1041. self.indexFile = function () {
  1042. if (! self.readyToIndex()) {
  1043. return;
  1044. }
  1045. self.indexingStarted(true);
  1046. viewModel.isLoading(true);
  1047. self.isIndexing(true);
  1048. $.post("${ url('indexer:importer_submit') }", {
  1049. "source": ko.mapping.toJSON(self.source),
  1050. "destination": ko.mapping.toJSON(self.destination)
  1051. }, function (resp) {
  1052. self.showCreate(true);
  1053. self.editorId(resp.history_id);
  1054. self.jobId(resp.handle.id);
  1055. $('#importerNotebook').html($('#importerNotebook-progress').html());
  1056. self.editorVM = new EditorViewModel(resp.history_uuid, '', {
  1057. user: '${ user.username }',
  1058. userId: ${ user.id },
  1059. languages: [{name: "Java", type: "java"}, {name: "Hive SQL", type: "hive"}], // TODO reuse
  1060. snippetViewSettings: {
  1061. java : {
  1062. snippetIcon: 'fa-file-archive-o '
  1063. },
  1064. hive: {
  1065. placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
  1066. aceMode: 'ace/mode/hive',
  1067. snippetImage: '${ static("beeswax/art/icon_beeswax_48.png") }',
  1068. sqlDialect: true
  1069. },
  1070. impala: {
  1071. placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
  1072. aceMode: 'ace/mode/impala',
  1073. snippetImage: '${ static("impala/art/icon_impala_48.png") }',
  1074. sqlDialect: true
  1075. }
  1076. }
  1077. });
  1078. self.editorVM.editorMode(true);
  1079. ko.cleanNode($("#importerNotebook")[0]);
  1080. ko.applyBindings(self.editorVM, $("#importerNotebook")[0]);
  1081. self.editorVM.openNotebook(resp.history_uuid, null, true, function(){
  1082. self.editorVM.selectedNotebook().snippets()[0].progress.subscribe(function(val){
  1083. if (val == 100){
  1084. self.indexingStarted(false);
  1085. self.isIndexing(false);
  1086. self.indexingSuccess(true);
  1087. }
  1088. });
  1089. self.editorVM.selectedNotebook().snippets()[0].status.subscribe(function(val){
  1090. if (val == 'failed'){
  1091. self.isIndexing(false);
  1092. self.indexingStarted(false);
  1093. self.indexingError(true);
  1094. } else if (val == 'available') {
  1095. var snippet = self.editorVM.selectedNotebook().snippets()[0]; // Could be native to editor at some point
  1096. if (! snippet.result.handle().has_more_statements) {
  1097. if (self.editorVM.selectedNotebook().onSuccessUrl()) {
  1098. window.location.href = self.editorVM.selectedNotebook().onSuccessUrl();
  1099. }
  1100. } else { // Perform last DROP statement execute
  1101. snippet.execute();
  1102. }
  1103. }
  1104. });
  1105. self.editorVM.selectedNotebook().snippets()[0].checkStatus();
  1106. });
  1107. viewModel.isLoading(false);
  1108. }).fail(function (xhr, textStatus, errorThrown) {
  1109. $(document).trigger("error", xhr.responseText);
  1110. viewModel.isLoading(false);
  1111. self.indexingStarted(false);
  1112. self.isIndexing(false);
  1113. });
  1114. }
  1115. self.removeOperation = function (operation, operationList) {
  1116. operationList.remove(operation);
  1117. }
  1118. self.addOperation = function (field) {
  1119. field.operations.push(new Operation("split"));
  1120. }
  1121. self.load = function (state) {
  1122. self.source.name(state.name);
  1123. self.source.show(state.show);
  1124. self.source.path(state.path);
  1125. self.destination.columns.removeAll();
  1126. if (state.format && 'type' in state.format) {
  1127. var koFormat = ko.mapping.fromJS(new FileType(state.format.type, state.format));
  1128. self.source.format(koFormat);
  1129. }
  1130. if (state.columns) state.columns.forEach(function (currCol) {
  1131. self.destination.columns.push(loadField(currCol));
  1132. });
  1133. }
  1134. };
  1135. var loadField = function (currField) {
  1136. var koField = ko.mapping.fromJS(currField);
  1137. koField.operations.removeAll();
  1138. currField.operations.forEach(function (operationData) {
  1139. var operation = new Operation(operationData.type);
  1140. operation.load(operationData);
  1141. koField.operations.push(operation);
  1142. });
  1143. var autoExpand = function(newVal) {
  1144. if ((newVal == 'array' || newVal == 'map' || newVal == 'struct') && koField.nested().length == 0) {
  1145. koField.nested.push(loadField({operations: [], nested: [], name: '', type: '', keyType: '', level: koField.level() + 1}));
  1146. }
  1147. };
  1148. koField.type.subscribe(autoExpand);
  1149. koField.keyType.subscribe(autoExpand);
  1150. return koField;
  1151. }
  1152. var IndexerViewModel = function (options) {
  1153. var self = this;
  1154. self.apiHelper = ApiHelper.getInstance(options);
  1155. self.assistAvailable = ko.observable(true);
  1156. self.isLeftPanelVisible = ko.observable();
  1157. self.apiHelper.withTotalStorage('assist', 'assist_panel_visible', self.isLeftPanelVisible, true);
  1158. self.loadField = loadField;
  1159. // Wizard related
  1160. self.wizardEnabled = ko.observable(false);
  1161. self.currentStep = ko.observable(1);
  1162. self.previousStepVisible = ko.pureComputed(function(){
  1163. return self.currentStep() > 1;
  1164. });
  1165. self.nextStepVisible = ko.pureComputed(function(){
  1166. return self.currentStep() < 3 && self.wizardEnabled();
  1167. });
  1168. self.nextStep = function () {
  1169. if (self.nextStepVisible()){
  1170. self.currentStep(self.currentStep() + 1);
  1171. }
  1172. }
  1173. self.previousStep = function () {
  1174. if (self.previousStepVisible()){
  1175. self.currentStep(self.currentStep() - 1);
  1176. }
  1177. }
  1178. self.createWizard = new CreateWizard(self);
  1179. self.isLoading = ko.observable(false);
  1180. };
  1181. var viewModel;
  1182. $(document).ready(function () {
  1183. var options = {
  1184. user: '${ user.username }',
  1185. i18n: {
  1186. errorLoadingDatabases: "${ _('There was a problem loading the databases') }",
  1187. errorLoadingTablePreview: "${ _('There was a problem loading the table preview.') }"
  1188. }
  1189. }
  1190. viewModel = new IndexerViewModel(options);
  1191. ko.applyBindings(viewModel, $('#importerComponents')[0]);
  1192. var draggableMeta = {};
  1193. huePubSub.subscribe('draggable.text.meta', function (meta) {
  1194. draggableMeta = meta;
  1195. });
  1196. huePubSub.subscribe('split.panel.resized', function () {
  1197. $('.form-actions').width($('.content-panel').width() - 50);
  1198. });
  1199. $('.form-actions').width($('.content-panel').width() - 50);
  1200. $('.content-panel').droppable({
  1201. accept: ".draggableText",
  1202. drop: function (e, ui) {
  1203. var text = ui.helper.text();
  1204. var generatedName = 'idx';
  1205. switch (draggableMeta.type){
  1206. case 'sql':
  1207. if (draggableMeta.table !== ''){
  1208. generatedName += draggableMeta.table;
  1209. viewModel.createWizard.source.inputFormat('table');
  1210. viewModel.createWizard.source.table(draggableMeta.table);
  1211. }
  1212. break;
  1213. case 'hdfs':
  1214. generatedName += draggableMeta.definition.name;
  1215. viewModel.createWizard.source.inputFormat('file');
  1216. viewModel.createWizard.source.path(draggableMeta.definition.path);
  1217. break;
  1218. case 'document':
  1219. if (draggableMeta.definition.type === 'query-hive'){
  1220. generatedName += draggableMeta.definition.name;
  1221. viewModel.createWizard.source.inputFormat('query');
  1222. viewModel.createWizard.source.draggedQuery(draggableMeta.definition.id);
  1223. }
  1224. break;
  1225. }
  1226. if (generatedName !== 'idx' && viewModel.createWizard.source.name() === ''){
  1227. viewModel.createWizard.source.name(generatedName);
  1228. }
  1229. else {
  1230. if (draggableMeta.table !== ''){
  1231. viewModel.createWizard.source.selectQuery();
  1232. }
  1233. }
  1234. }
  1235. });
  1236. });
  1237. })();
  1238. </script>
  1239. </span>
  1240. %if not is_embeddable:
  1241. ${ commonfooter(request, messages) | n,unicode }
  1242. %endif