importer.mako 142 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089
  1. ## Licensed to Cloudera, Inc. under one
  2. ## or more contributor license agreements. See the NOTICE file
  3. ## distributed with this work for additional information
  4. ## regarding copyright ownership. Cloudera, Inc. licenses this file
  5. ## to you under the Apache License, Version 2.0 (the
  6. ## "License"); you may not use this file except in compliance
  7. ## with the License. You may obtain a copy of the License at
  8. ##
  9. ## http://www.apache.org/licenses/LICENSE-2.0
  10. ##
  11. ## Unless required by applicable law or agreed to in writing, software
  12. ## distributed under the License is distributed on an "AS IS" BASIS,
  13. ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. ## See the License for the specific language governing permissions and
  15. ## limitations under the License.
  16. <%!
  17. from django.utils.translation import ugettext as _
  18. from desktop import conf
  19. from desktop.views import commonheader, commonfooter, commonshare, commonimportexport, _ko
  20. from filebrowser.conf import SHOW_UPLOAD_BUTTON
  21. from notebook.conf import ENABLE_SQL_INDEXER
  22. from indexer.conf import ENABLE_NEW_INDEXER, ENABLE_SQOOP, ENABLE_KAFKA, CONFIG_INDEXER_LIBS_PATH, ENABLE_SCALABLE_INDEXER, ENABLE_ALTUS, ENABLE_ENVELOPE, ENABLE_FIELD_EDITOR
  23. %>
  24. <%namespace name="actionbar" file="actionbar.mako" />
  25. <%namespace name="assist" file="/assist.mako" />
  26. %if not is_embeddable:
  27. ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
  28. ## TODO lot of those re-imported
  29. <script src="${ static('desktop/js/autocomplete/sqlParseSupport.js') }"></script>
  30. <script src="${ static('desktop/js/autocomplete/sqlAutocompleteParser.js') }"></script>
  31. <script src="${ static('desktop/js/sqlAutocompleter2.js') }"></script>
  32. <script src="${ static('desktop/js/hdfsAutocompleter.js') }"></script>
  33. <script src="${ static('desktop/js/autocompleter.js') }"></script>
  34. <script src="${ static('desktop/js/hue.json.js') }"></script>
  35. <script src="${ static('desktop/js/jquery.hiveautocomplete.js') }" type="text/javascript" charset="utf-8"></script>
  36. <script src="${ static('desktop/ext/js/jquery/plugins/jquery-ui-1.10.4.custom.min.js') }"></script>
  37. <script src="${ static('desktop/ext/js/selectize.min.js') }"></script>
  38. <script src="${ static('desktop/ext/js/selectize-plugin-clear.js') }"></script>
  39. <script src="${ static('metastore/js/metastore.ko.js') }"></script>
  40. <script src="${ static('desktop/ext/js/knockout-sortable.min.js') }"></script>
  41. <script src="${ static('desktop/js/ko.editable.js') }"></script>
  42. <script src="${ static('desktop/ext/js/selectize.min.js') }"></script>
  43. <script src="${ static('desktop/js/ko.selectize.js') }"></script>
  44. ${ assist.assistJSModels() }
  45. <script src="${ static('notebook/js/notebook.ko.js') }"></script>
  46. <link rel="stylesheet" href="${ static('notebook/css/notebook.css') }">
  47. <link rel="stylesheet" href="${ static('notebook/css/notebook-layout.css') }">
  48. ${ assist.assistPanel() }
  49. %endif
  50. <link rel="stylesheet" href="${ static('indexer/css/importer.css') }" type="text/css">
  51. <style type="text/css">
  52. % if conf.CUSTOM.BANNER_TOP_HTML.get():
  53. .show-assist {
  54. top: 110px!important;
  55. }
  56. .main-content {
  57. top: 112px!important;
  58. }
  59. % endif
  60. </style>
  61. <span id="importerComponents" class="notebook importer-main" data-bind="dropzone: { url: '/filebrowser/upload/file?dest=' + DROPZONE_HOME_DIR, params: {dest: DROPZONE_HOME_DIR}, paramName: 'hdfs_file', onComplete: function(path){ createWizard.source.path(path); }, disabled: '${ not SHOW_UPLOAD_BUTTON.get() }' === 'True' }">
  62. <div class="dz-message" data-dz-message></div>
  63. <div class="navbar hue-title-bar">
  64. <div class="navbar-inner">
  65. <div class="container-fluid">
  66. <div class="nav-collapse">
  67. <ul class="nav">
  68. <li class="app-header">
  69. <a href="${ url('indexer:importer') }">
  70. <!-- ko if: createWizard.prefill.target_type().length == 0 -->
  71. <i class="fa fa-database app-icon"></i> ${_('Importer')}
  72. <!-- /ko -->
  73. <!-- ko ifnot: createWizard.prefill.target_type().length == 0 -->
  74. <!-- ko if: createWizard.prefill.target_type() === 'index' -->
  75. <i class="fa fa-search app-icon"></i> ${_('Import to index')}
  76. <!-- /ko -->
  77. <!-- ko if: createWizard.prefill.target_type() === 'table' -->
  78. <i class="fa fa-database app-icon"></i> ${_('Import to table')}
  79. <!-- /ko -->
  80. <!-- ko if: createWizard.prefill.target_type() === 'database' -->
  81. <i class="fa fa-database app-icon"></i> ${_('Create a new database')}
  82. <!-- /ko -->
  83. <!-- /ko -->
  84. </a>
  85. </li>
  86. </ul>
  87. </div>
  88. </div>
  89. </div>
  90. </div>
  91. %if not is_embeddable:
  92. <a title="${_('Toggle Assist')}" class="pointer show-assist" data-bind="visible: !$root.isLeftPanelVisible() && $root.assistAvailable(), click: function() { $root.isLeftPanelVisible(true); }">
  93. <i class="fa fa-chevron-right"></i>
  94. </a>
  95. %endif
  96. <div class="main-content">
  97. <div class="vertical-full container-fluid" data-bind="style: { 'padding-left' : $root.isLeftPanelVisible() ? '0' : '20px' }">
  98. <div class="vertical-full">
  99. <div class="vertical-full row-fluid panel-container">
  100. %if not is_embeddable:
  101. <div class="assist-container left-panel" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable()">
  102. <a title="${_('Toggle Assist')}" class="pointer hide-assist" data-bind="click: function() { $root.isLeftPanelVisible(false) }">
  103. <i class="fa fa-chevron-left"></i>
  104. </a>
  105. <div class="assist" data-bind="component: {
  106. name: 'assist-panel',
  107. params: {
  108. user: '${user.username}',
  109. onlySql: false,
  110. sql: {
  111. navigationSettings: {
  112. openItem: false,
  113. showStats: true
  114. }
  115. },
  116. visibleAssistPanels: ['sql', 'hdfs', 'documents']
  117. }
  118. }"></div>
  119. </div>
  120. <div class="resizer" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable(), splitDraggable : { appName: 'notebook', leftPanelVisible: $root.isLeftPanelVisible }"><div class="resize-bar">&nbsp;</div></div>
  121. %endif
  122. <div class="content-panel importer-droppable">
  123. <div class="content-panel-inner">
  124. <!-- ko template: 'create-index-wizard' --><!-- /ko -->
  125. </div>
  126. </div>
  127. </div>
  128. </div>
  129. </div>
  130. </div>
  131. <script type="text/html" id="create-index-wizard">
  132. <div data-bind="visible: createWizard.show">
  133. <div class="step-indicator-fixed">
  134. <ol class="list-inline text-center step-indicator">
  135. <li data-bind="css: { 'active': currentStep() == 1, 'complete': currentStep() > 1, 'pointer': currentStep() > 1 }, click: function() { currentStep(1) }">
  136. <div class="step" title="${ _('Go to Step 1') }">
  137. <!-- ko if: currentStep() == 1 -->
  138. <!-- ko if: createWizard.isGuessingFormat -->
  139. <span class="fa fa-spinner fa-spin"></span>
  140. <!-- /ko -->
  141. <!-- ko ifnot: createWizard.isGuessingFormat -->
  142. 1
  143. <!-- /ko -->
  144. <!-- /ko -->
  145. <!-- ko ifnot: currentStep() == 1 -->
  146. <span class="fa fa-check"></span>
  147. <!-- /ko -->
  148. </div>
  149. <div class="caption">
  150. <!-- ko if: createWizard.source.inputFormat() != 'manual' -->
  151. ${ _('Pick data from ') }<span data-bind="text: createWizard.source.inputFormat"></span>
  152. <!-- ko if: createWizard.source.inputFormat() == 'file' -->
  153. <span data-bind="text: createWizard.source.path"></span>
  154. <!-- /ko -->
  155. <!-- ko if: createWizard.source.inputFormat() == 'stream' -->
  156. <!-- ko if: createWizard.source.streamSelection() == 'kafka' -->
  157. <span data-bind="text: createWizard.source.kafkaSelectedTopics"></span>
  158. <!-- /ko -->
  159. <!-- ko if: createWizard.source.streamSelection() != 'kafka' -->
  160. <span data-bind="text: createWizard.source.streamSelection"></span>
  161. <!-- /ko -->
  162. <!-- /ko -->
  163. <!-- /ko -->
  164. <!-- ko if: createWizard.source.inputFormat() == 'manual' -->
  165. ${ _('No source data') }
  166. <!-- /ko -->
  167. </div>
  168. </li>
  169. <li class="arrow muted"><i class="fa fa-fw fa-angle-double-right"></i></li>
  170. <li data-bind="css: { 'inactive': currentStep() == 1, 'active': currentStep() == 2, 'complete': currentStep() == 3, 'pointer': currentStep() == 1 && !createWizard.isGuessingFormat() && createWizard.source.show() }, click: function() { if (!createWizard.isGuessingFormat() && createWizard.source.show()){ currentStep(2); }}">
  171. <div class="step" title="${ _('Go to Step 2') }">
  172. <!-- ko if: currentStep() < 3 -->
  173. <!-- ko if: createWizard.isGuessingFieldTypes -->
  174. <span class="fa fa-spinner fa-spin"></span>
  175. <!-- /ko -->
  176. <!-- ko ifnot: createWizard.isGuessingFieldTypes -->
  177. 2
  178. <!-- /ko -->
  179. <!-- /ko -->
  180. </div>
  181. <div class="caption">
  182. <!-- ko if: createWizard.source.inputFormat() != 'manual' -->
  183. ${ _('Move it to ') }
  184. <!-- /ko -->
  185. <!-- ko if: createWizard.source.inputFormat() == 'manual' -->
  186. ${ _('Create') }
  187. <!-- /ko -->
  188. <span data-bind="text: createWizard.destination.outputFormat"></span>
  189. <span data-bind="text: createWizard.destination.name"></span>
  190. <!-- ko if: createWizard.destination.outputFormat() == 'altus' -->
  191. ${ _('in') } <span data-bind="text: createWizard.destination.namespace"></span>
  192. <!-- /ko -->
  193. </div>
  194. </li>
  195. </ol>
  196. </div>
  197. <div class="vertical-spacer"></div>
  198. <!-- ko if: currentStep() == 1 -->
  199. <div class="card step">
  200. <h4>${_('Source')}</h4>
  201. <div class="card-body">
  202. <div>
  203. <div class="control-group" data-bind="visible: createWizard.prefill.target_type().length == 0 || createWizard.prefill.source_type() == 'all'">
  204. <label for="sourceType" class="control-label"><div>${ _('Type') }</div>
  205. <select id="sourceType" data-bind="selectize: createWizard.source.inputFormats, value: createWizard.source.inputFormat, optionsText: 'name', optionsValue: 'value'"></select>
  206. </label>
  207. </div>
  208. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'rdbms'">
  209. <label for="rdbmsMode" class="control-label"><div>${ _('Mode') }</div>
  210. <label class="radio inline-block margin-right-20">
  211. <input type="radio" name="rdbmsMode" value="customRdbms" data-bind="checked: createWizard.source.rdbmsMode" /> ${_('Custom')}
  212. </label>
  213. <label class="radio inline-block">
  214. <input type="radio" name="rdbmsMode" value="configRdbms" data-bind="checked: createWizard.source.rdbmsMode" /> ${_('Configured')}
  215. </label>
  216. </label>
  217. </div>
  218. <div class="control-group" data-bind="visible: createWizard.prefill.target_type() == 'database'">
  219. <label for="sourceType" class="control-label">${ _('No source is needed for creating a database.') }</label>
  220. </div>
  221. <div class="control-group input-append" data-bind="visible: createWizard.source.inputFormat() == 'file'">
  222. <label for="path" class="control-label"><div>${ _('Path') }</div>
  223. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: createWizard.source.path, filechooser: createWizard.source.path, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true, openOnFocus: true, selectFolder: false }" placeholder="${ _('Click or drag from the assist') }">
  224. </label>
  225. <!-- ko if: createWizard.source.path().length > 0 -->
  226. <a data-bind="storageContextPopover: { path: createWizard.source.path(), offset: { right: 5 } }" title="${ _('Preview') }" style="font-size: 14px" class="margin-left-10">
  227. <i class="fa fa-fw fa-info"></i>
  228. </a>
  229. <!-- /ko -->
  230. </div>
  231. <!-- ko if: createWizard.source.inputFormat() == 'rdbms' -->
  232. <!-- ko if: createWizard.source.rdbmsMode() -->
  233. <div class="control-group">
  234. <label for="rdbmsType" class="control-label"><div>${ _('Driver') }</div>
  235. <select id="rdbmsType" data-bind="selectize: createWizard.source.rdbmsTypes, value: createWizard.source.rdbmsType, optionsText: 'name', optionsValue: 'value'"></select>
  236. </label>
  237. </div>
  238. <!-- /ko -->
  239. <!-- ko if: createWizard.source.rdbmsMode() == 'customRdbms' -->
  240. <!-- ko if: createWizard.source.rdbmsType() != 'jdbc' -->
  241. <div class="control-group">
  242. <label for="rdbmsHostname" class="control-label"><div>${ _('Hostname') }</div>
  243. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsHostname" placeholder="${ _('Enter host/ip here e.g. mysql.domain.com or 123.123.123.123') }">
  244. </label>
  245. </div>
  246. <!-- /ko -->
  247. <!-- ko if: createWizard.source.rdbmsType() == 'jdbc' -->
  248. <div class="control-group">
  249. <label for="rdbmsHostname" class="control-label"><div>${ _('Url') }</div>
  250. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsHostname" placeholder="${ _('jdbc:mysql://localhost:3306') }">
  251. </label>
  252. </div>
  253. <!-- /ko -->
  254. <!-- ko if: createWizard.source.rdbmsType() == 'jdbc' -->
  255. <div class="control-group">
  256. <label for="rdbmsJdbcDriver" class="control-label"><div>${ _('JDBC Driver') }</div>
  257. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsJdbcDriver">
  258. </label>
  259. </div>
  260. <!-- /ko -->
  261. <!-- ko if: createWizard.source.rdbmsType() != 'jdbc' -->
  262. <div class="control-group">
  263. <label for="rdbmsPort" class="control-label"><div>${ _('Port') }</div>
  264. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsPort" placeholder="${ _('Enter port number here e.g. 3306') }">
  265. </label>
  266. </div>
  267. <!-- /ko -->
  268. <div class="control-group">
  269. <label for="rdbmsUsername" class="control-label"><div>${ _('Username') }</div>
  270. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsUsername" placeholder="${ _('Enter username here') }">
  271. </label>
  272. </div>
  273. <div class="control-group no-margin-bottom">
  274. <label for="rdbmsPassword" class="control-label"><div>${ _('Password') }</div>
  275. <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsPassword" placeholder="${ _('Enter password here') }">
  276. </label>
  277. </div>
  278. <div class="control-group">
  279. <button class="btn" data-bind="click: createWizard.source.rdbmsCheckConnection">
  280. ${_('Test Connection')}
  281. </button>
  282. </div>
  283. <!-- /ko -->
  284. <!-- ko if: createWizard.source.rdbmsMode() == 'configRdbms' || (createWizard.source.rdbmsMode() == 'customRdbms' && createWizard.source.rdbmsDbIsValid()) -->
  285. <!-- ko if: createWizard.source.rdbmsMode() == 'configRdbms' && createWizard.source.rdbmsType() == 'jdbc' -->
  286. <div class="control-group">
  287. <label for="rdbmsJdbcDriverName" class="control-label"><div>${ _('Options') }</div>
  288. <select id="rdbmsJdbcDriverName" data-bind="selectize: createWizard.source.rdbmsJdbcDriverNames, value: createWizard.source.rdbmsJdbcDriverName, optionsText: 'name', optionsValue: 'value'"></select>
  289. <!-- ko hueSpinner: { spin: createWizard.source.isFetchingDriverNames, inline: true } --><!-- /ko -->
  290. </label>
  291. </div>
  292. <!-- /ko -->
  293. <!-- ko if: createWizard.source.rdbmsTypes -->
  294. <div class="control-group input-append">
  295. <label for="rdbmsDatabaseName" class="control-label"><div>${ _('Database Name') }</div>
  296. <select id="rdbmsDatabaseName" data-bind="selectize: createWizard.source.rdbmsDatabaseNames, value: createWizard.source.rdbmsDatabaseName, optionsText: 'name', optionsValue: 'value'"></select>
  297. <!-- ko hueSpinner: { spin: createWizard.source.isFetchingDatabaseNames, inline: true } --><!-- /ko -->
  298. </label>
  299. </div>
  300. <!-- /ko -->
  301. <!-- ko if: createWizard.source.rdbmsDatabaseName -->
  302. <div class="control-group">
  303. <div>
  304. <label class="checkbox inline-block">
  305. <input type="checkbox" data-bind="checked: createWizard.source.rdbmsIsAllTables"> ${_('All Tables')}
  306. </label>
  307. </div>
  308. <!-- ko ifnot: createWizard.source.rdbmsIsAllTables() -->
  309. <label for="rdbmsTableName" class="control-label"><div>${ _('Table Name') }</div>
  310. <select id="rdbmsTableName" multiple="multiple" data-bind="selectize: createWizard.source.rdbmsTableNames, selectedObjects: createWizard.source.tables, selectedOptions: createWizard.source.tablesNames, optionsText: 'name', optionsValue: 'value'"></select>
  311. <!-- ko hueSpinner: { spin: createWizard.source.isFetchingTableNames, inline: true } --><!-- /ko -->
  312. </label>
  313. <!-- /ko -->
  314. </div>
  315. <!-- /ko -->
  316. <!-- /ko -->
  317. <!-- /ko -->
  318. <!-- ko if: createWizard.source.inputFormat() == 'stream' -->
  319. <div class="control-group">
  320. <label class="control-label"><div>${ _('List') }</div>
  321. <select data-bind="selectize: createWizard.source.publicStreams, value: createWizard.source.streamSelection, optionsText: 'name', optionsValue: 'value'" placeholder="${ _('The list of streams to consume, e.g. SFDC, Jiras...') }"></select>
  322. </label>
  323. </div>
  324. <!-- ko if: createWizard.source.streamSelection() == 'kafka' -->
  325. <div data-bind="template: { name: 'kafka-topic-template', data: $data }" class="margin-top-10 field inline-block"></div>
  326. <!-- /ko -->
  327. <!-- ko if: createWizard.source.streamSelection() == 'flume' -->
  328. <div class="control-group">
  329. <label class="control-label"><div>${ _('Type') }</div>
  330. <select class="input-medium" data-bind="selectize: createWizard.source.channelSourceTypes, value: createWizard.source.channelSourceType, optionsText: 'name', optionsValue: 'value'"></select>
  331. </label>
  332. <!-- ko if: ['directory', 'exec', 'syslogs'].indexOf(createWizard.source.channelSourceType()) != -1 -->
  333. <label class="control-label"><div>${ _('Hosts') }</div>
  334. <select class="input-xxlarge" data-bind="selectize: createWizard.source.channelSourceHosts, selectedOptions: createWizard.source.channelSourceSelectedHosts" multiple="true"></select>
  335. </label>
  336. <!-- /ko -->
  337. <!-- ko if: createWizard.source.channelSourceType() == 'directory' -->
  338. <label class="control-label"><div>${ _('Path') }</div>
  339. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.channelSourcePath" placeholder="${ _('The path to watch and consume') }">
  340. </label>
  341. <!-- /ko -->
  342. <!-- ko if: createWizard.source.channelSourceType() -->
  343. <input data-bind="click: function() { createWizard.source.channelSourceType(null); }" class="btn" value="${ _('Clear') }"/>
  344. <!-- /ko -->
  345. </div>
  346. <!-- /ko -->
  347. <!-- /ko -->
  348. <!-- ko if: createWizard.source.inputFormat() == 'connector' -->
  349. <div class="control-group">
  350. <label class="control-label"><div>${ _('List') }</div>
  351. <select data-bind="selectize: createWizard.source.connectorList, value: createWizard.source.connectorSelection, optionsText: 'name', optionsValue: 'value'" placeholder="${ _('The connector to use, e.g. SFDC, Jiras...') }"></select>
  352. </label>
  353. </div>
  354. <!-- ko if: createWizard.source.connectorSelection() == 'sfdc' -->
  355. <div class="control-group">
  356. <label class="control-label"><div>${ _('Username') }</div>
  357. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamUsername" placeholder="user@company.com">
  358. </label>
  359. <label class="control-label"><div>${ _('Password') }</div>
  360. <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.streamPassword">
  361. </label>
  362. <label class="control-label"><div>${ _('Token') }</div>
  363. <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.streamToken">
  364. </label>
  365. <label class="control-label"><div>${ _('End point URL') }</div>
  366. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamEndpointUrl">
  367. </label>
  368. <br/>
  369. <!-- ko if: createWizard.source.streamUsername() && createWizard.source.streamPassword() && createWizard.source.streamToken() -->
  370. <label class="control-label"><div>${ _('Object') }</div>
  371. <select class="input-xxlarge" data-bind="options: createWizard.source.streamObjects,
  372. value: createWizard.source.streamObject,
  373. optionsCaption: '${ _("Choose...") }'"
  374. placeholder="${ _('The SFDC object to import, e.g. Account, Opportunity') }"></select>
  375. </label>
  376. <!-- /ko -->
  377. </div>
  378. <!-- /ko -->
  379. <!-- /ko -->
  380. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'table'">
  381. <!-- ko foreach: $root.createWizard.source.tables -->
  382. <div>
  383. <label class="control-label">
  384. <div><!-- ko if: $index() === 0 -->${ _('Table') }<!-- /ko --></div>
  385. <input type="text" class="input-xlarge" data-bind="hivechooser: name, skipInvalids:true, pathChangeLevel: 'table', skipColumns: true, apiHelperUser: '${ user }', namespace: $root.createWizard.source.namespace, compute: $root.createWizard.source.compute, apiHelperType: $root.createWizard.source.sourceType, mainScrollable: $(MAIN_SCROLLABLE)" placeholder="${ _('Table name or <database>.<table>') }">
  386. <a class="pointer pull-right margin-left-5" style="margin-top: 7px" data-bind="click: function() { $root.createWizard.source.tables.remove(this); }, visible: $root.createWizard.source.tables().length > 1"><i class="fa fa-minus"></i></a>
  387. </label>
  388. </div>
  389. <!-- /ko -->
  390. <div>
  391. <label class="control-label">
  392. <div>&nbsp;</div>
  393. <a class="pointer" data-bind="click: function() { createWizard.source.tables.push(ko.mapping.fromJS({name: ''})); }" title="${_('Add Table')}"><i class="fa fa-plus fa-padding-top"></i> ${_('Add table')}</a>
  394. </label>
  395. </div>
  396. </div>
  397. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'query'">
  398. <label for="path" class="control-label"><div>${ _('Query') }
  399. <!-- ko if: createWizard.source.query() && createWizard.source.query().name -->
  400. <span data-bind="text: createWizard.source.query().name"></span>
  401. <!-- /ko -->
  402. </div>
  403. ## No dropdown for now
  404. ## <select placeholder="${ _('Search your documents...') }" data-bind="documentChooser: { dependentValue: createWizard.source.draggedQuery, mappedDocument: createWizard.source.query }"></select>
  405. </label>
  406. </div>
  407. </div>
  408. </div>
  409. </div>
  410. <!-- ko if: createWizard.source.show() && createWizard.source.inputFormat() != 'manual' -->
  411. <div class="card step" data-bind="visible: createWizard.source.inputFormat() == 'file' || createWizard.source.inputFormat() == 'stream'">
  412. <!-- ko if: createWizard.isGuessingFormat -->
  413. <h4>${_('Guessing format...')} <i class="fa fa-spinner fa-spin"></i></h4>
  414. <!-- /ko -->
  415. <!-- ko ifnot: createWizard.isGuessingFormat -->
  416. <h4>${_('Format')}</h4>
  417. <div class="card-body">
  418. <label data-bind="visible: (createWizard.prefill.source_type().length == 0 || createWizard.prefill.target_type() == 'index') && (createWizard.source.inputFormat() == 'file' || createWizard.source.inputFormat() == 'stream')">
  419. <div>${_('File Type')}</div>
  420. <select data-bind="selectize: $root.createWizard.fileTypes, value: $root.createWizard.fileTypeName, optionsText: 'description', optionsValue: 'name'"></select>
  421. </label>
  422. <span class="inline-labels" data-bind="with: createWizard.source.format, visible: createWizard.source.show">
  423. <span data-bind="foreach: getArguments()">
  424. <!-- ko template: {name: 'arg-' + $data.type, data: {description: $data.description, value: $parent[$data.name]}}--><!-- /ko -->
  425. </span>
  426. </span>
  427. </div>
  428. <!-- /ko -->
  429. </div>
  430. <!-- ko ifnot: createWizard.source.inputFormat() == 'rdbms' && createWizard.source.rdbmsIsAllTables() -->
  431. <div class="card step" style="min-height: 310px;">
  432. <!-- ko if: $root.createWizard.source.inputFormat() === 'table' || ($root.createWizard.source.inputFormat() !== 'table' && !createWizard.isGuessingFormat()) -->
  433. <h4>${_('Preview')}
  434. <!-- ko if: $root.createWizard.source.inputFormat() !== 'table' && createWizard.isGuessingFieldTypes() -->
  435. <i class="fa fa-spinner fa-spin"></i>
  436. <!-- /ko -->
  437. </h4>
  438. <!-- /ko -->
  439. <!-- ko if: createWizard.source.inputFormat() == 'query' && createWizard.source.sample().length == 0 -->
  440. ${ _('Add sample data') } <i class="fa fa-fw fa-play"></i>
  441. <!-- /ko -->
  442. <div class="card-body">
  443. <!-- ko if: ['table', 'rdbms'].indexOf($root.createWizard.source.inputFormat()) >= 0 && $root.createWizard.source.tables().length > 1 -->
  444. <ul class="nav nav-tabs" style="margin-top: -14px">
  445. <!-- ko foreach: $root.createWizard.source.tables -->
  446. <li data-bind="css: { 'active': $root.createWizard.source.selectedTableIndex() === $index() }"><a href="#" data-bind="text: name, click: function(){ $root.createWizard.source.selectedTableIndex($index()) }"></a></li>
  447. <!-- /ko -->
  448. </ul>
  449. <!-- /ko -->
  450. <!-- ko if: ['table', 'rdbms'].indexOf($root.createWizard.source.inputFormat()) >= 0 -->
  451. <!-- ko hueSpinner: { spin: createWizard.isGuessingFormat() || createWizard.isGuessingFieldTypes() , inline: true } --><!-- /ko -->
  452. <!-- /ko -->
  453. <!-- ko ifnot: createWizard.isGuessingFormat -->
  454. <!-- ko ifnot: createWizard.isGuessingFieldTypes -->
  455. <div data-bind="delayedOverflow">
  456. <table class="table table-condensed table-preview">
  457. <thead>
  458. <tr data-bind="foreach: createWizard.source.sampleCols">
  459. ##<!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  460. <th data-bind="truncatedText: name" style="padding-right: 60px"></th>
  461. ## TODO nested
  462. </tr>
  463. </thead>
  464. <tbody data-bind="foreach: createWizard.source.sample">
  465. <tr data-bind="foreach: $data">
  466. <td data-bind="truncatedText: $data"></td>
  467. </tr>
  468. </tbody>
  469. </table>
  470. </div>
  471. <!-- /ko -->
  472. <!-- /ko -->
  473. </div>
  474. </div>
  475. <!-- /ko -->
  476. <!-- /ko -->
  477. <!-- /ko -->
  478. <!-- ko if: currentStep() == 2 -->
  479. <!-- ko with: createWizard.destination -->
  480. <div class="card step">
  481. <h4>${_('Destination')}</h4>
  482. <div class="card-body">
  483. <div class="control-group">
  484. <label for="destinationType" class="control-label" data-bind="visible: $parent.createWizard.prefill.target_type().length == 0"><div>${ _('Type') }</div>
  485. <select id="destinationType" data-bind="selectize: outputFormats, value: outputFormat, optionsValue: 'value', optionsText: 'name'"></select>
  486. </label>
  487. </div>
  488. <div class="control-group">
  489. <!-- ko if: outputFormat() == 'file' -->
  490. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  491. <input type="text" class="form-control name input-xxlarge" id="collectionName" data-bind="value: name, filechooser: name, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true, openOnFocus: true, selectFolder: true, displayOnlyFolders: true, uploadFile: false}" placeholder="${ _('Name') }" title="${ _('Directory must not exist in the path') }">
  492. <!-- /ko -->
  493. <!-- ko if: outputFormat() == 'index' -->
  494. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  495. <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="value: name, valueUpdate: 'afterkeydown'" placeholder="${ _('Name') }">
  496. <!-- /ko -->
  497. <!-- ko if: ['table', 'database'].indexOf(outputFormat()) != -1 -->
  498. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  499. <input type="text" class="input-xxlarge" data-bind="value: name, hivechooser: name, namespace: namespace, compute: compute, skipColumns: true, skipTables: outputFormat() == 'database', valueUpdate: 'afterkeydown', apiHelperUser: '${ user }', apiHelperType: sourceType, mainScrollable: $(MAIN_SCROLLABLE), attr: { 'placeholder': outputFormat() == 'table' ? '${ _ko('Table name or <database>.<table>') }' : '${ _ko('Database name') }' }" pattern="^([a-zA-Z0-9_]+\.)?[a-zA-Z0-9_]*$" title="${ _('Only alphanumeric and underscore characters') }">
  500. <!-- /ko -->
  501. <!-- ko if: outputFormat() == 'altus' -->
  502. <label for="collectionName" class="control-label"><div>${ _('Namespace') }</div></label>
  503. <div class="namespace-selection">
  504. <select data-bind="selectize: namespaces, value: targetNamespaceId, optionsValue: 'id', optionsText: 'name'" class="input-medium"></select>
  505. </div>
  506. <label class="checkbox inline-block"><input type="checkbox"> ${_('Copy Sentry privileges')}</label>
  507. <br/>
  508. <label class="control-label "><div>${ _('Database') }</div></label>
  509. <input type="text" class="form-control input-xlarge" data-bind="value: name" placeholder="${ _('Database') }">
  510. <a href="javascript:void(0);" data-bind="sqlContextPopover: { sourceType: $root.createWizard.source.sourceType, namespace: namespace, compute: compute, path: 'default', offset: { top: -3, left: 3 }}">
  511. <i class="fa fa-info"></i>
  512. </a>
  513. <!-- /ko -->
  514. <!-- ko if: outputFormat() == 'stream' -->
  515. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  516. <!-- ko with: $root -->
  517. <input type="text" data-bind="value: createWizard.source.kafkaSelectedTopics">
  518. ## <div data-bind="template: { name: 'kafka-topic-template' }" class="margin-top-10 field inline-block"></div>
  519. <!-- /ko -->
  520. <!-- /ko -->
  521. <!-- ko if: outputFormat() == 'flume' -->
  522. <h4>${ _('Sink') }</h4>
  523. <div class="row-fluid">
  524. <div>
  525. <label class="control-label"><div>${ _('Type') }</div>
  526. <select class="input-medium" data-bind="selectize: channelSinkTypes, value: channelSinkType, optionsText: 'name', optionsValue: 'value'"></select>
  527. </label>
  528. <!-- ko if: channelSinkType() == 'solr' -->
  529. <label class="control-label"><div>${ _('Collection') }</div>
  530. <select class="input-xxlarge" data-bind="selectize: ['logIndex', 'apacheLogs'], value: channelSinkPath"></select>
  531. </label>
  532. <!-- /ko -->
  533. <!-- ko if: channelSinkType() -->
  534. <input data-bind="click: function() { channelSourceType(null); }" class="btn" value="${ _('Clear') }"/>
  535. <!-- /ko -->
  536. </div>
  537. </div>
  538. <!-- /ko -->
  539. <span class="help-inline muted" data-bind="visible: !isTargetExisting() && isTargetChecking()">
  540. <i class="fa fa-spinner fa-spin"></i>
  541. </span>
  542. <span class="help-inline muted" data-bind="visible: !$parent.createWizard.isValidDestination()">
  543. <i class="fa fa-warning" style="color: #c09853"></i>
  544. <!-- ko if: name() -->
  545. ${ _('Invalid characters') }
  546. <!-- /ko -->
  547. <!-- ko if: !name() -->
  548. ${ _('Empty name') }
  549. <!-- /ko -->
  550. </span>
  551. <span class="help-inline muted" data-bind="visible: isTargetExisting() && outputFormat() != 'altus'">
  552. <i class="fa fa-warning" style="color: #c09853"></i> ${ _('Already existing') } <span data-bind="text: outputFormat"></span>
  553. <a href="javascript:void(0)" data-bind="hueLink: existingTargetUrl(), text: name" title="${ _('Open') }"></a>
  554. </span>
  555. <span class="help-inline muted" data-bind="visible: !isTargetExisting() && $parent.createWizard.source.inputFormat() === 'rdbms' && outputFormat() == 'database' ">
  556. <i class="fa fa-warning" style="color: #c09853"></i> ${ _('Does not exist') } <span data-bind="text: outputFormat"></span>
  557. </span>
  558. </div>
  559. </div>
  560. </div>
  561. <!-- ko if: outputFormat() == 'table' && $root.createWizard.source.inputFormat() != 'rdbms' -->
  562. <div class="card step">
  563. <h4>${_('Properties')}</h4>
  564. <div class="card-body">
  565. <div class="control-group">
  566. <label for="destinationFormat" class="control-label"><div>${ _('Format') }</div>
  567. <select id="destinationFormat" data-bind="selectize: tableFormats, value: tableFormat, optionsValue: 'value', optionsText: 'name'"></select>
  568. </label>
  569. </div>
  570. <div class="control-group">
  571. <label class="control-label"><div>${ _('Extras') }</div>
  572. <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, toggle: showProperties" title="${ _('Show extra properties') }">
  573. <i class="fa fa-sliders fa-padding-top"></i>
  574. </a>
  575. </label>
  576. </div>
  577. <span data-bind="visible: showProperties">
  578. <div class="control-group">
  579. <label class="checkbox inline-block" data-bind="visible: tableFormat() != 'kudu'">
  580. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Store in Default location')}
  581. </label>
  582. </div>
  583. <div class="control-group" data-bind="visible: !useDefaultLocation()">
  584. <label for="path" class="control-label"><div>${ _('External location') }</div>
  585. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  586. </label>
  587. </div>
  588. <div class="control-group">
  589. <label class="checkbox inline-block" data-bind="visible: $root.createWizard.source.inputFormat() != 'manual'">
  590. <input type="checkbox" data-bind="checked: importData, disable: !useDefaultLocation() && $parent.createWizard.source.path() == nonDefaultLocation();"> ${_('Import data')}
  591. </label>
  592. </div>
  593. <div class="control-group">
  594. <label><div>${ _('Description') }</div>
  595. <input type="text" class="form-control input-xxlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
  596. </label>
  597. </div>
  598. <div class="control-group" data-bind="visible: $root.createWizard.source.inputFormat() == 'file'">
  599. <label class="checkbox inline-block">
  600. <input type="checkbox" data-bind="checked: hasHeader"> ${_('Use first row as header')}
  601. </label>
  602. </div>
  603. <div class="control-group" data-bind="visible: tableFormat() == 'text'">
  604. <label class="checkbox inline-block">
  605. <input type="checkbox" data-bind="checked: useCustomDelimiters"> ${_('Custom char delimiters')}
  606. </label>
  607. </div>
  608. <span class="inline-labels" data-bind="visible: tableFormat() == 'text' && useCustomDelimiters()">
  609. <label for="fieldDelimiter" class="control-label"><div>${ _('Field') }</div>
  610. <select id="fieldDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customFieldDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  611. </label>
  612. <label for="collectionDelimiter" class="control-label"><div>${ _('Array, Map') }</div>
  613. <select id="collectionDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customCollectionDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  614. </label>
  615. <label for="structDelimiter" class="control-label"><div>${ _('Struct') }</div>
  616. <select id="structDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customMapDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  617. </label>
  618. </span>
  619. </span>
  620. <div class="control-group" data-bind="visible: tableFormat() == 'regexp'">
  621. <label for="customRegexp" class="control-label"><div>${ _('Regexp') }</div>
  622. <input id="customRegexp" class="input-xxlarge" type="text" data-bind="value: customRegexp" placeholder='([^]*) ([^]*) ([^]*) (-|\\[^\\]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\".*\") ([^ \"]*|\".*\"))?'>
  623. </label>
  624. </div>
  625. <div class="control-group" data-bind="visible: tableFormat() == 'kudu'">
  626. <label for="kuduPksTable" class="control-label"><div>${ _('Primary keys') }</div>
  627. ## At least one selected
  628. <select id="kuduPksTable" data-bind="selectize: columns, selectedOptions: primaryKeys, selectedObjects: primaryKeyObjects, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="3" multiple="true"></select>
  629. </label>
  630. </div>
  631. <label class="control-label"><div>${ _('Partitions') }</div>
  632. <!-- ko if: tableFormat() != 'kudu' && $root.createWizard.source.inputFormat() != 'rdbms' -->
  633. <div class="inline-table">
  634. <div class="form-inline" data-bind="foreach: partitionColumns">
  635. <a class="pointer pull-right margin-top-20" data-bind="click: function() { $parent.partitionColumns.remove($data); }"><i class="fa fa-minus"></i></a>
  636. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field inline-block"></div>
  637. <div class="clearfix"></div>
  638. </div>
  639. <a data-bind="click: function() { partitionColumns.push($root.loadDefaultField({isPartition: true})); }" class="pointer" title="${_('Add Partition')}"><i class="fa fa-plus fa-padding-top"></i> ${_('Add partition')}</a>
  640. </div>
  641. <!-- /ko -->
  642. <!-- ko if: tableFormat() == 'kudu' -->
  643. <div class="form-inline inline-table">
  644. <ul class="unstyled kudu-partitions" data-bind="foreach: kuduPartitionColumns">
  645. <li>
  646. <a class="pointer pull-right" data-bind="click: function() { $parent.kuduPartitionColumns.remove($data); }"><i class="fa fa-minus"></i></a>
  647. <select data-bind="selectize: $parent.primaryKeyObjects, selectedOptions: columns, optionsValue: 'name', optionsText: 'name', selectizeOptions: { placeholder: '${ _ko('Columns...') }' }" size="3" multiple="true"></select>
  648. <select data-bind="selectize: ['RANGE BY', 'HASH'], value: name"></select>
  649. <!-- ko if: name() == 'HASH' -->
  650. <input class="input-small" type="number" data-bind="value: int_val">
  651. <!-- /ko -->
  652. <!-- ko if: name() == 'RANGE BY' -->
  653. <div class="form-inline" data-bind="foreach: range_partitions">
  654. <div class="range-partition">
  655. <a class="pull-right" data-bind="click: function() { $parent.range_partitions.remove($data); }"><i class="fa fa-minus"></i></a>
  656. <!-- ko if: name() == 'VALUES' -->
  657. <input type="text" class="input-small" data-bind="value: lower_val">
  658. <select data-bind="selectize: ['<', '<='], value: include_lower_val"></select>
  659. <select data-bind="selectize: ['VALUES', 'VALUE'], value: name"></select>
  660. <select data-bind="selectize: ['<', '<='], value: include_upper_val"></select>
  661. <input type="text" class="input-small" data-bind="value: upper_val">
  662. <!-- /ko -->
  663. <!-- ko if: name() == 'VALUE' -->
  664. <select data-bind="selectize: ['VALUES', 'VALUE'], value: name"></select>
  665. <div class="inline-block" data-bind="foreach: values" style="max-width: 370px">
  666. <input class="input-small" type="text" data-bind="textInput: value" style="margin-bottom: 5px">
  667. <a data-bind="click: function() { $parent.values.remove($data); }"><i class="fa fa-minus"></i></a>
  668. </div>
  669. <a class="inline-block" data-bind="click: function() { values.push(ko.mapping.fromJS({value: ''})); }" title="${_('Add value')}"><i class="fa fa-plus"></i></a>
  670. <div class="clearfix"></div>
  671. <!-- /ko -->
  672. </div>
  673. </div>
  674. <a data-bind="click: function() { range_partitions.push(ko.mapping.fromJS($parent.KUDU_DEFAULT_RANGE_PARTITION_COLUMN)); }" title="${_('Add column')}"><i class="fa fa-plus"></i></a>
  675. <!-- /ko -->
  676. <!-- /ko -->
  677. </li>
  678. </ul>
  679. <a data-bind="click: function() { kuduPartitionColumns.push(ko.mapping.fromJS(KUDU_DEFAULT_PARTITION_COLUMN)); }" class="pointer" title="${_('Add partition')}"><i class="fa fa-plus"></i> ${_('Add partition')}</a>
  680. </div>
  681. <!-- /ko -->
  682. </label>
  683. </div>
  684. </div>
  685. <!-- /ko -->
  686. <!-- ko if: outputFormat() == 'index' -->
  687. <div class="card step">
  688. <h4>${_('Properties')}</h4>
  689. <div class="card-body">
  690. % if ENABLE_SCALABLE_INDEXER.get():
  691. <div class="control-group">
  692. <label class="checkbox inline-block" title="${ _('Execute a cluster job to index a large dataset.') }" data-bind="visible: $root.createWizard.source.inputFormat() == 'file'">
  693. <input type="checkbox" data-bind="checked: indexerRunJob"> ${_('Index with a job')}
  694. </label>
  695. <label for="path" class="control-label" data-bind="visible: indexerRunJob"><div>${ _('Libs') }</div>
  696. <input type="text" class="form-control path filechooser-input input-xlarge" data-bind="value: indexerJobLibPath, filechooser: indexerJobLibPath, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  697. </label>
  698. <!-- ko if: indexerRunJob() && indexerJobLibPath().length > 0 -->
  699. <a data-bind="hueLink: '/filebrowser/view=' + indexerJobLibPath()" title="${ _('Open') }" style="font-size: 14px" class="margin-left-10">
  700. <i class="fa fa-external-link-square"></i>
  701. </a>
  702. <!-- /ko -->
  703. </div>
  704. % endif
  705. <div class="control-group">
  706. <label for="kuduPksIndex" class="control-label"><div>${ _('Primary key') }</div>
  707. <select id="kuduPksIndex" data-bind="selectize: columns, selectedOptions: indexerPrimaryKey, selectedObjects: indexerPrimaryKeyObject, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="1" multiple="false"></select>
  708. </label>
  709. </div>
  710. <div class="control-group">
  711. <label for="kuduDefaultField" class="control-label"><div>${ _('Default field') }</div>
  712. <select id="kuduDefaultField" data-bind="selectize: columns, selectedOptions: indexerDefaultField, selectedObjects: indexerDefaultFieldObject, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="1" multiple="false"></select>
  713. </label>
  714. </div>
  715. <div class="control-group">
  716. <label class="control-label"><div>${ _('Extras') }</div>
  717. <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, toggle: showProperties" title="${ _('Show extra properties') }">
  718. <i class="fa fa-sliders fa-padding-top"></i>
  719. </a>
  720. </label>
  721. </div>
  722. <span data-bind="visible: showProperties">
  723. <div class="control-group">
  724. <label for="destinationFormatIndex" class="control-label"><div>${ _('Config set') }</div>
  725. <select id="destinationFormatIndex" data-bind="selectize: indexerConfigSets, value: indexerConfigSet, optionsValue: 'value', optionsText: 'name'"></select>
  726. </label>
  727. </div>
  728. <div class="control-group">
  729. <label for="indexerNumShards" class="control-label"><div>${ _('Num shards') }</div>
  730. <input type="number" class="input-small" placeholder="1" data-bind="value: indexerNumShards">
  731. </label>
  732. </div>
  733. <div class="control-group">
  734. <label for="indexerReplicationFactor" class="control-label"><div>${ _('Replication factor') }</div>
  735. <input type="number" class="input-small" placeholder="1" data-bind="value: indexerReplicationFactor">
  736. </label>
  737. </div>
  738. ## Router, maxShardsPer, shards, routerField
  739. <div class="control-group" data-bind="visible: $root.createWizard.source.inputFormat() == 'file'">
  740. <label class="checkbox inline-block">
  741. <input type="checkbox" data-bind="checked: hasHeader"> ${_('Use first row as header')}
  742. </label>
  743. </div>
  744. </span>
  745. </div>
  746. </div>
  747. <!-- /ko -->
  748. <!-- ko if: $root.createWizard.source.inputFormat() == 'rdbms' && ['database', 'file', 'table', 'hbase'].indexOf(outputFormat()) != -1 -->
  749. <div class="card step">
  750. <h4>${_('Properties')}</h4>
  751. <div class="card-body">
  752. <label class="control-label"><div>${ _('Libs') }</div>
  753. <div class="inline-table">
  754. <ul data-bind="sortable: { data: sqoopJobLibPaths, options: { axis: 'y', containment: 'parent', handle: '.move-widget' }}, visible: sqoopJobLibPaths().length" class="unstyled">
  755. <li>
  756. <div class="input-append" style="margin-bottom: 4px">
  757. <input type="text" class="filechooser-input input-xxlarge" data-bind="value: path, valueUpdate:'afterkeydown', filechooser: { value: path, isAddon: true }, filechooserOptions: { skipInitialPathIfEmpty: true }" placeholder="${ _('Path to the file, e.g. hdfs://localhost:8020/user/hue/file.hue') }"/>
  758. <span class="add-on move-widget muted" data-bind="visible: $parent.sqoopJobLibPaths().length > 1"><i class="fa fa-arrows"></i></span>
  759. <a class="add-on muted" href="javascript: void(0);" data-bind="click: function(){ $parent.removeSqoopJobLibPath($data); }"><i class="fa fa-minus"></i></a>
  760. </div>
  761. </li>
  762. </ul>
  763. <div class="config-property-add-value" style="margin-top: 5px;">
  764. <a class="inactive-action pointer" style="padding: 3px 10px 3px 3px;;" data-bind="click: addSqoopJobLibPath">
  765. <i class="fa fa-plus"></i>
  766. </a>
  767. </div>
  768. </div>
  769. </label>
  770. </div>
  771. <div class="control-group">
  772. <label class="control-label"><div>${ _('Extras') }</div>
  773. <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, toggle: showProperties" title="${ _('Show extra properties') }">
  774. <i class="fa fa-sliders fa-padding-top"></i>
  775. </a>
  776. </label>
  777. </div>
  778. <span data-bind="visible: showProperties">
  779. <div class="control-group">
  780. <label><div>${ _('Mappers') }</div>
  781. <input type="number" class="form-control input-small" data-bind="textInput: numMappers">
  782. </label>
  783. </div>
  784. <div class="control-group">
  785. <!-- ko if: !$root.createWizard.source.rdbmsAllTablesSelected() -->
  786. <label for="rdbmsSplitBy" class="control-label"><div>${ _('Split By') }</div>
  787. <select id="rdbmsSplitBy" data-bind="selectize: columns, value: rdbmsSplitByColumn, optionsValue: 'name', optionsText: 'name'"></select>
  788. </label>
  789. <!-- /ko -->
  790. </div>
  791. <div class="control-group" data-bind="visible: outputFormat() == 'file' && !$root.createWizard.source.rdbmsAllTablesSelected()">
  792. <label for="destinationFormat" class="control-label"><div>${ _('Format') }</div>
  793. <select id="destinationFormat" data-bind="selectize: rdbmsFileOutputFormats, value: rdbmsFileOutputFormat, optionsValue: 'value', optionsText: 'name'"></select>
  794. </label>
  795. </div>
  796. <span class="inline-labels" data-bind="visible: rdbmsFileOutputFormat() == 'text' && outputFormat() == 'file' && !$root.createWizard.source.rdbmsAllTablesSelected()">
  797. <label for="fieldDelimiter" class="control-label"><div>${ _('Fields') }</div>
  798. <select id="fieldDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customFieldsDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  799. </label>
  800. <label for="collectionDelimiter" class="control-label"><div>${ _('Line') }</div>
  801. <select id="collectionDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customLineDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  802. </label>
  803. <label for="structDelimiter" class="control-label"><div>${ _('Optionally Enclosed By') }</div>
  804. <select id="structDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customEnclosedByDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  805. </label>
  806. </span>
  807. <!-- ko if: outputFormat() === 'database' -->
  808. <span>
  809. <label class="checkbox">
  810. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Default location')}
  811. </label>
  812. <span data-bind="visible: !useDefaultLocation()">
  813. <label for="path" class="control-label"><div>${ _('External location') }</div>
  814. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  815. </label>
  816. </span>
  817. </span>
  818. <!-- /ko -->
  819. </span>
  820. </div>
  821. <!-- /ko -->
  822. <!-- ko if: $root.createWizard.source.inputFormat() == 'kafka' -->
  823. <div class="card step">
  824. <h4>${_('Properties')}</h4>
  825. <div class="card-body">
  826. <label class="control-label"><div>${ _('Libs') }</div>
  827. <div class="inline-table">
  828. <ul data-bind="sortable: { data: sqoopJobLibPaths, options: { axis: 'y', containment: 'parent', handle: '.move-widget' }}, visible: sqoopJobLibPaths().length" class="unstyled">
  829. <li>
  830. <div class="input-append" style="margin-bottom: 4px">
  831. <input type="text" class="filechooser-input input-xxlarge" data-bind="value: path, valueUpdate:'afterkeydown', filechooser: { value: path, isAddon: true }, filechooserOptions: { skipInitialPathIfEmpty: true }" placeholder="${ _('Path to the file, e.g. hdfs://localhost:8020/user/hue/file.hue') }"/>
  832. <span class="add-on move-widget muted" data-bind="visible: $parent.sqoopJobLibPaths().length > 1"><i class="fa fa-arrows"></i></span>
  833. <a class="add-on muted" href="javascript: void(0);" data-bind="click: function(){ $parent.removeSqoopJobLibPath($data); }"><i class="fa fa-minus"></i></a>
  834. </div>
  835. </li>
  836. </ul>
  837. <div class="config-property-add-value" style="margin-top: 5px;">
  838. <a class="inactive-action pointer" style="padding: 3px 10px 3px 3px;" data-bind="click: addSqoopJobLibPath">
  839. <i class="fa fa-plus"></i>
  840. </a>
  841. </div>
  842. </div>
  843. </label>
  844. </div>
  845. </div>
  846. <!-- /ko -->
  847. <!-- ko if: ['table', 'index', 'hbase'].indexOf(outputFormat()) != -1 -->
  848. <div class="card step">
  849. <h4>
  850. <!-- ko if: fieldEditorEnabled -->
  851. <!-- ko if: useFieldEditor -->
  852. <a class="inactive-action" href="javascript:void(0);" data-bind="toggle: useFieldEditor">${_('Fields')} /</a> ${_('Editor')}
  853. <!-- /ko -->
  854. <!-- ko ifnot: useFieldEditor -->
  855. ${_('Fields')} / <a class="inactive-action" href="javascript:void(0);" data-bind="toggle: useFieldEditor">${_('Editor')}</a>
  856. <!-- /ko -->
  857. <!-- /ko -->
  858. <!-- ko ifnot: fieldEditorEnabled -->
  859. ${_('Fields')} <!-- ko if: $root.createWizard.isGuessingFieldTypes --><i class="fa fa-spinner fa-spin"></i><!-- /ko -->
  860. <a class="inactive-action pointer" data-bind="visible: columns().length > 0, publish: 'importer.show.bulkeditor'" href="javascript:void(0)">
  861. <i class="fa fa-edit"></i>
  862. </a>
  863. <!-- /ko -->
  864. </h4>
  865. <div class="card-body no-margin-top columns-form">
  866. <!-- ko if: useFieldEditor -->
  867. <div class="control-group" style="margin-bottom: 5px;">
  868. <label class="control-label"><div style="width: initial">${ _('Language') }</div>
  869. <select>
  870. <option value="sql">SQL</option>
  871. <option value="morphline">Morphline</option>
  872. </select>
  873. </label>
  874. </div>
  875. <div data-bind="component: { name: 'hue-simple-ace-editor-multi', params: {
  876. value: fieldEditorValue,
  877. placeHolder: fieldEditorPlaceHolder,
  878. autocomplete: { type: sourceType },
  879. lines: 5,
  880. aceOptions: {
  881. minLines: 10,
  882. maxLines: 25
  883. },
  884. database: fieldEditorDatabase,
  885. namespace: namespace,
  886. compute: compute,
  887. temporaryOnly: true,
  888. mode: sourceType
  889. }}"></div>
  890. <!-- /ko -->
  891. <!-- ko ifnot: useFieldEditor -->
  892. <!-- ko if: $root.createWizard.source.inputFormat() === 'manual' -->
  893. <form class="form-inline inline-table" data-bind="foreach: columns">
  894. <!-- ko if: $parent.outputFormat() == 'table' -->
  895. <a class="pointer pull-right margin-top-20" data-bind="click: function() { $parent.columns.remove($data); }"><i class="fa fa-minus"></i></a>
  896. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field inline-block"></div>
  897. <div class="clearfix"></div>
  898. <!-- /ko -->
  899. <!-- ko if: $parent.outputFormat() == 'index' -->
  900. <a class="pointer pull-right margin-top-20" data-bind="click: function() { $parent.columns.remove($data); }"><i class="fa fa-minus"></i></a>
  901. <div data-bind="template: { name: 'index-field-template', data: $data }, css: { 'disabled': !keep() }" class="margin-top-10 field inline-block index-field"></div>
  902. <div class="clearfix"></div>
  903. <!-- /ko -->
  904. </form>
  905. <div class="clearfix"></div>
  906. <!-- ko if: outputFormat() == 'table' || outputFormat() == 'index' -->
  907. <a data-bind="click: function() { columns.push($root.loadDefaultField({})); }" class="pointer" title="${_('Add Field')}"><i class="fa fa-plus"></i> ${_('Add Field')}</a>
  908. <!-- /ko -->
  909. <!-- /ko -->
  910. <!-- ko ifnot: $root.createWizard.source.inputFormat() === 'manual' -->
  911. <form class="form-inline inline-table" data-bind="foreachVisible: { data: columns, minHeight: 54, container: MAIN_SCROLLABLE }">
  912. <!-- ko if: $parent.outputFormat() == 'table' && $root.createWizard.source.inputFormat() != 'rdbms' -->
  913. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field"></div>
  914. <!-- /ko -->
  915. <!-- ko if: (['file', 'table', 'hbase'].indexOf($parent.outputFormat()) != -1 && $root.createWizard.source.inputFormat() == 'rdbms') || $parent.outputFormat() == 'index' -->
  916. <div data-bind="template: { name: 'index-field-template', data: $data }, css: { 'disabled': !keep() }" class="margin-top-10 field index-field"></div>
  917. <!-- /ko -->
  918. </form>
  919. <div class="clearfix"></div>
  920. <!-- /ko -->
  921. <!-- /ko -->
  922. </div>
  923. </div>
  924. <!-- /ko -->
  925. <!-- ko if: outputFormat() == 'database' && $root.createWizard.source.inputFormat() !== 'rdbms' -->
  926. <div class="card step">
  927. <h4>${_('Properties')}</h4>
  928. <div class="card-body">
  929. <label><div>${ _('Description') }</div>
  930. <input type="text" class="form-control input-xlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
  931. </label>
  932. <label class="checkbox">
  933. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Default location')}
  934. </label>
  935. <span data-bind="visible: !useDefaultLocation()">
  936. <label for="path" class="control-label"><div>${ _('External location') }</div>
  937. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  938. </label>
  939. </span>
  940. </div>
  941. </div>
  942. <!-- /ko -->
  943. <!-- /ko -->
  944. <!-- /ko -->
  945. <div class="form-actions">
  946. <!-- ko if: previousStepVisible -->
  947. <button class="btn" data-bind="click: previousStep">${ _('Back') }</button>
  948. <!-- /ko -->
  949. <!-- ko if: currentStep() == 1 -->
  950. <button class="btn" data-bind="enable: !createWizard.isGuessingFormat() && createWizard.source.show(), click: function() { currentStep(2); }">
  951. ${_('Next')}
  952. </button>
  953. <!-- /ko -->
  954. <!-- ko if: currentStep() == 2 -->
  955. <button class="btn btn-primary disable-feedback" data-bind="click: createWizard.indexFile, enable: createWizard.readyToIndex() && !createWizard.indexingStarted()">
  956. ${ _('Submit') } <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.indexingStarted"></i>
  957. </button>
  958. % if ENABLE_ENVELOPE.get():
  959. <button class="btn disable-feedback" data-bind="click: createWizard.showCommands, enable: createWizard.readyToIndex()">
  960. ${ _('Show Commands') }
  961. </button>
  962. <button class="btn disable-feedback" data-bind="click: createWizard.indexFile">
  963. ${ _('Save') }
  964. </button>
  965. % endif
  966. <!-- /ko -->
  967. <span data-bind="visible: createWizard.editorId">
  968. <button class="btn btn-success" data-bind="click: function(){ window.open('${ url('notebook:editor') }?editor=' + createWizard.editorId()) }" title="${ _('Open') }">
  969. ${_('Check status')}
  970. </button>
  971. </span>
  972. <div id="importerNotebook"></div>
  973. </div>
  974. </div>
  975. <div id="fieldsBulkEditor" class="modal hide fade">
  976. <div class="modal-header">
  977. <button type="button" class="close" data-dismiss="modal" aria-label="${ _('Close') }"><span aria-hidden="true">&times;</span></button>
  978. <h2 class="modal-title">${ _('Write or paste comma separated field names') }</h2>
  979. </div>
  980. <div class="modal-body">
  981. <input type="text" class="input-xxlarge" placeholder="${ _('e.g. id, name, salary') }" data-bind="textInput: createWizard.destination.bulkColumnNames">
  982. </div>
  983. <div class="modal-footer">
  984. <button class="btn" data-dismiss="modal" aria-hidden="true">${ _('Cancel') }</button>
  985. <button class="btn btn-primary" data-bind="click: createWizard.destination.processBulkColumnNames">${ _('Change field names') }</button>
  986. </div>
  987. </div>
  988. </script>
  989. <script type="text/html" id="table-field-template">
  990. <div>
  991. <label data-bind="visible: level() == 0 || ($parent.type() != 'array' && $parent.type() != 'map')">${ _('Name') }&nbsp;
  992. <input type="text" class="input-large" placeholder="${ _('Field name') }" required data-bind="textInput: name">
  993. </label>
  994. <label class="margin-left-5">${ _('Type') }&nbsp;
  995. <!-- ko if: !(level() > 0 && $parent.type() == 'map') -->
  996. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.hiveFieldTypes, value: type"></select>
  997. <!-- /ko -->
  998. <!-- ko if: level() > 0 && $parent.type() == 'map' -->
  999. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.hivePrimitiveFieldTypes, value: keyType"></select>
  1000. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.hiveFieldTypes, value: type"></select>
  1001. <!-- /ko -->
  1002. <input type="number" class="input-small" placeholder="${ _('Length') }" data-bind="value: length, visible: type() == 'varchar' || type() == 'char'">
  1003. <!-- ko if: $parent.type() == 'decimal' -->
  1004. <input type="number" class="input-small" placeholder="${ _('Precision') }" data-bind="value: precision">
  1005. <input type="number" class="input-small" placeholder="${ _('Scale') }" data-bind="value: scale">
  1006. <!-- /ko -->
  1007. </label>
  1008. <!-- ko if: $root.createWizard.source.inputFormat() != 'manual' && typeof isPartition !== 'undefined' && isPartition() -->
  1009. <label class="margin-left-5">${ _('Value') }&nbsp;
  1010. <input type="text" class="input-medium margin-left-5" placeholder="${ _('Partition value') }" data-bind="value: partitionValue">
  1011. </label>
  1012. <!-- /ko -->
  1013. <span data-bind="visible: level() == 0 || ($parent.type() != 'array' && $parent.type() != 'map')">
  1014. <a href="javascript:void(0)" title="${ _('Show field properties') }" data-bind="css: {'inactive-action': !showProperties()}, click: function() {showProperties(!showProperties()) }"><i class="fa fa-sliders"></i></a>
  1015. <span data-bind="visible: showProperties">
  1016. <input type="text" class="input-medium margin-left-5" placeholder="${ _('Field comment') }" data-bind="value: comment">
  1017. <label class="checkbox" data-bind="visible: $root.createWizard.destination.tableFormat() == 'kudu'">
  1018. <input type="checkbox" data-bind="checked: keep"> ${_('Keep')}
  1019. </label>
  1020. </span>
  1021. </span>
  1022. <!-- ko if: level() > 0 && $parent.type() == 'struct' && $parent.nested().length > 1 -->
  1023. <a data-bind="click: function() { $parent.nested.remove($data); }"><i class="fa fa-minus"></i></a>
  1024. <!-- /ko -->
  1025. <div class="inline-block" data-bind="template: { name:'field-column-example' }"></div>
  1026. <!-- ko if: type() == 'array' || type() == 'map' || type() == 'struct' -->
  1027. <div class="operation" data-bind="template: { name: 'table-field-template', foreach: nested }"></div>
  1028. <a data-bind="click: function() { nested.push($root.loadDefaultField({level: level() + 1})); }, visible: type() == 'struct'"><i class="fa fa-plus"></i></a>
  1029. <!-- /ko -->
  1030. </div>
  1031. </script>
  1032. <script type="text/html" id="index-field-template">
  1033. <label>${ _('Name') }&nbsp;
  1034. <input type="text" class="input-large" placeholder="${ _('Field name') }" data-bind="value: name, enable: keep" pattern="^(?!_version_)[a-zA-Z_][a-zA-Z0-9_]*$" title="${ _('Only alphanumeric and underscore characters and not _version_') }">
  1035. </label>
  1036. <!-- ko if: $root.createWizard.source.inputFormat() != 'rdbms' -->
  1037. <label class="margin-left-5">${ _('Type') }&nbsp;
  1038. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.fieldTypes, value: type"></select>
  1039. </label>
  1040. <!-- /ko -->
  1041. <!-- ko if: $root.createWizard.source.inputFormat() == 'rdbms' -->
  1042. <label class="margin-left-5">${ _('Type') }&nbsp;
  1043. <input type="text" class="input-small" placeholder="${ _('Field Type') }" data-bind="value: type, enable: false">
  1044. </label>
  1045. <!-- /ko -->
  1046. <a href="javascript:void(0)" title="${ _('Show field properties') }" data-bind="css: {'inactive-action': !showProperties()}, click: function() {showProperties(!showProperties()) }"><i class="fa fa-sliders"></i></a>
  1047. <span data-bind="visible: showProperties" class="field-properties">
  1048. <!-- ko if: $root.createWizard.source.inputFormat() != 'rdbms' -->
  1049. <label class="checkbox">
  1050. <input type="checkbox" data-bind="checked: unique"> ${_('Unique')}
  1051. </label>
  1052. <label class="checkbox">
  1053. <input type="checkbox" data-bind="checked: required"> ${_('Required')}
  1054. </label>
  1055. <!-- /ko -->
  1056. <label class="checkbox">
  1057. <input type="checkbox" data-bind="checked: keep"> ${_('Keep')}
  1058. </label>
  1059. </span>
  1060. <!-- ko if: operations().length == 0 -->
  1061. <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation, visible: $root.createWizard.destination.outputFormat() == 'index'" title="${_('Add Operation')}">
  1062. <i class="fa fa-plus"></i> ${_('Operation')}
  1063. </a>
  1064. <!-- /ko -->
  1065. <div class="inline-block" data-bind="template: { name: 'field-column-example' }"></div>
  1066. <div data-bind="foreach: operations">
  1067. <div data-bind="template: { name:'operation-template',data: { operation: $data, list: $parent.operations } }"></div>
  1068. </div>
  1069. <!-- ko if: operations().length > 0 -->
  1070. <a class="pointer" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation to')} <span data-bind="text: name"></span></a>
  1071. <!-- /ko -->
  1072. </script>
  1073. <script type="text/html" id="field-column-example">
  1074. <!-- ko if: $root.createWizard.source.inputFormat() != 'manual' && level() == 0 && (typeof isPartition === 'undefined' || !isPartition()) -->
  1075. <!-- ko if: $root.createWizard.source.sample() && $root.createWizard.source.sample().length > 0 -->
  1076. <div class="inline-block muted field-content-preview" data-bind="truncatedText: $root.createWizard.source.sample()[0][$index()]"></div>
  1077. <!-- ko if: $root.createWizard.source.sample().length > 1 -->
  1078. <div class="inline-block muted field-content-preview" data-bind="truncatedText: $root.createWizard.source.sample()[1][$index()]"></div>
  1079. <!-- /ko -->
  1080. <!-- /ko -->
  1081. <!-- ko if: !$root.createWizard.source.sample() || $root.createWizard.source.sample().length === 0 -->
  1082. <div class="inline-block muted field-content-preview">${ _("No sample to be shown") }</div>
  1083. <!-- /ko -->
  1084. <!-- /ko -->
  1085. </script>
  1086. <script type="text/html" id="operation-template">
  1087. <div class="operation">
  1088. <select data-bind="browserAwareSelectize: $root.createWizard.operationTypesFiltered, value: operation.type"></select>
  1089. <!-- ko template: "args-template" --><!-- /ko -->
  1090. <!-- ko if: operation.settings().outputType() == "custom_fields" -->
  1091. <label class="margin-left-5">${ _('Number of expected fields') }
  1092. <input type="number" class="input-mini" data-bind="value: operation.numExpectedFields">
  1093. </label>
  1094. <!-- /ko -->
  1095. <a class="pointer margin-left-20" data-bind="click: function(){$root.createWizard.removeOperation(operation, list)}" title="${ _('Remove') }"><i class="fa fa-times"></i></a>
  1096. <div class="margin-left-20" data-bind="foreach: operation.fields">
  1097. <div data-bind="template: { name: 'index-field-template', data: $data }, css:{ 'disabled': !keep() }" class="margin-top-10 field index-field"></div>
  1098. </div>
  1099. </div>
  1100. </script>
  1101. <script type="text/html" id="args-template">
  1102. <!-- ko foreach: {data: operation.settings().getArguments(), as: 'argument'} -->
  1103. <!-- ko template: {name: 'arg-' + argument.type, data: {description: argument.description, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
  1104. <!-- /ko -->
  1105. </script>
  1106. <script type="text/html" id="arg-text">
  1107. <label>
  1108. <div data-bind="text: description"></div>
  1109. <input type="text" class="input" data-bind="attr: {placeholder: description}, value: value">
  1110. </label>
  1111. </script>
  1112. <script type="text/html" id="arg-text-delimiter">
  1113. <label>
  1114. <div data-bind="text: description"></div>
  1115. <select data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: value, optionsValue: 'value', optionsText: 'name', attr: {placeholder: description}"></select>
  1116. </label>
  1117. </script>
  1118. <script type="text/html" id="arg-checkbox">
  1119. <label class="checkbox">
  1120. <input type="checkbox" data-bind="checked: value">
  1121. <span data-bind="text: description" style="vertical-align: middle"></span>
  1122. </label>
  1123. </script>
  1124. <script type="text/html" id="arg-mapping">
  1125. <!-- ko foreach: value-->
  1126. <div>
  1127. <input type="text" data-bind="value: key, attr: {placeholder: 'key'}">
  1128. <input type="text" data-bind="value: value, attr: {placeholder: 'value'}">
  1129. <button class="btn" data-bind="click: function(){$parent.value.remove($data)}">${_('Remove Pair')}</button>
  1130. </div>
  1131. <!-- /ko -->
  1132. <button class="btn" data-bind="click: function(){value.push({key: ko.observable(''), value: ko.observable('')})}">${_('Add Pair')}</button>
  1133. <br>
  1134. </script>
  1135. <script type="text/html" id="field-preview-header-template">
  1136. <th data-bind="visible: keep, text: name" style="padding-right:60px"></th>
  1137. <!-- ko foreach: operations -->
  1138. <!--ko foreach: fields -->
  1139. <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  1140. <!-- /ko -->
  1141. <!--/ko -->
  1142. </script>
  1143. <script type="text/html" id="output-generated-field-data-template">
  1144. <!-- ko foreach: operations -->
  1145. <!--ko foreach: fields -->
  1146. <td data-bind="visible: keep">[[${_('generated')}]]</td>
  1147. <!-- ko template: 'output-generated-field-data-template' --><!-- /ko -->
  1148. <!-- /ko -->
  1149. <!--/ko -->
  1150. </script>
  1151. <script type="text/html" id="importerNotebook-progress">
  1152. <!-- ko with: selectedNotebook -->
  1153. <!-- ko foreach: snippets -->
  1154. <div class="progress-snippet progress" data-bind="css: {
  1155. 'progress-starting': progress() == 0 && status() == 'running',
  1156. 'progress-warning': progress() > 0 && progress() < 100,
  1157. 'progress-success': progress() == 100,
  1158. 'progress-danger': progress() == 0 && errors().length > 0}" style="background-color: #FFF; width: 100%; height: 4px">
  1159. <div class="bar" data-bind="style: {'width': (errors().length > 0 ? 100 : Math.max(2, progress())) + '%'}"></div>
  1160. </div>
  1161. <div class="snippet-error-container alert alert-error alert-error-gradient" data-bind="visible: errors().length > 0">
  1162. <ul class="unstyled" data-bind="foreach: errors">
  1163. <li data-bind="text: message"></li>
  1164. </ul>
  1165. </div>
  1166. <!-- /ko -->
  1167. <!-- /ko -->
  1168. </script>
  1169. <script type="text/html" id="kafka-topic-template">
  1170. <div class="control-group">
  1171. <label class="control-label"><div>${ _('Topics') }</div>
  1172. <select class="input-xxlarge" data-bind="options: createWizard.source.kafkaTopics,
  1173. value: createWizard.source.kafkaSelectedTopics,
  1174. optionsCaption: '${ _("Choose...") }'"
  1175. placeholder="${ _('The list of topics to consume, e.g. orders,returns') }">
  1176. </select>
  1177. ## <select data-bind="selectize: createWizard.source.kafkaTopics, value: createWizard.source.kafkaSelectedTopics" placeholder="${ _('The list of topics to consume, e.g. orders,returns') }"></select>
  1178. </label>
  1179. <br/>
  1180. <label class="control-group" data-bind="visible: createWizard.source.kafkaSelectedTopics">
  1181. <label class="control-label"><div>${ _('Schema') }</div>
  1182. <label class="radio margin-right-10">
  1183. <input type="radio" name="kafkaSchemaManual" value="manual" data-bind="checked: createWizard.source.kafkaSchemaManual" /> ${_('Manual')}
  1184. </label>
  1185. <label class="radio">
  1186. <input type="radio" name="kafkaSchemaManual" value="detect" data-bind="checked: createWizard.source.kafkaSchemaManual" /> ${_('Guess')}
  1187. </label>
  1188. </label>
  1189. <!-- ko if: createWizard.source.kafkaSchemaManual() == 'manual' -->
  1190. ##<label class="control-label"><div>${ _('Encoding') }</div>
  1191. ## <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldType">
  1192. ##</label>
  1193. <br/>
  1194. <label class="control-label"><div>${ _('Type') }</div>
  1195. <select class="input-medium" data-bind="options: ['delimited', 'bitarray'], value: createWizard.source.kafkaFieldType"></select>
  1196. </label>
  1197. <label class="control-label"><div>${ _('Delimiter') }</div>
  1198. <input type="text" class="input-small" data-bind="value: createWizard.source.kafkaFieldDelimiter">
  1199. </label>
  1200. <br/>
  1201. <label class="control-label"><div>${ _('Field names') }</div>
  1202. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldNames" placeholder="${ _('The list of fields to consume, e.g. orders,returns') }">
  1203. </label>
  1204. <br/>
  1205. <label class="control-label"><div>${ _('Field types') }</div>
  1206. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldTypes" placeholder="${ _('The list of field typs, e.g. string,int') }">
  1207. </label>
  1208. <br/>
  1209. <label class="control-label" data-bind="visible: createWizard.source.hasStreamSelected"><div></div>
  1210. <button class="btn" data-bind="click: createWizard.source.streamCheckConnection">${_('Test')}</button>
  1211. </label>
  1212. <!-- /ko -->
  1213. </div>
  1214. </div>
  1215. </script>
  1216. <div id="showCommandsModal" class="modal transparent-modal hide" data-backdrop="true" style="width:980px; margin-left:-510px!important">
  1217. <div class="modal-header">
  1218. <button type="button" class="close" data-dismiss="modal" aria-label="${ _('Close') }"><span aria-hidden="true">&times;</span></button>
  1219. <h2 class="modal-title">${_('Commands')}</h2>
  1220. </div>
  1221. <div class="modal-body">
  1222. <div class="editor-help two-pane">
  1223. <div class="tab-content">
  1224. <div class="tab-pane active" id="help-editor-syntax">
  1225. <ul class="nav nav-tabs">
  1226. <li class="active">
  1227. <a href="#help-editor-syntax-comment" data-bind="click: function(){ $('a[href=\'#help-editor-syntax-comment\']').tab('show'); }">
  1228. ${ _('Commands')}
  1229. </a>
  1230. </li>
  1231. <li>
  1232. <a href="#help-editor-syntax-click" data-bind="click: function(){ $('a[href=\'#help-editor-syntax-click\']').tab('show'); }">
  1233. ${ _('Shell')}
  1234. </a>
  1235. </li>
  1236. <li>
  1237. <a href="#help-editor-syntax-multiquery" data-bind="click: function(){ $('a[href=\'#help-editor-syntax-multiquery\']').tab('show'); }">
  1238. ${ _('REST')}
  1239. </a>
  1240. </li>
  1241. </ul>
  1242. <div class="tab-content">
  1243. <div class="tab-pane active" id="help-editor-syntax-comment">
  1244. <div data-bind="text: createWizard.commands" style="white-space: pre-wrap;">
  1245. </div>
  1246. </div>
  1247. <div class="tab-pane" id="help-editor-syntax-click">
  1248. <ul class="nav help-list-spacing">
  1249. </ul>
  1250. </div>
  1251. <div class="tab-pane" id="help-editor-syntax-multiquery">
  1252. </div>
  1253. </div>
  1254. </div>
  1255. </div>
  1256. </div>
  1257. </div>
  1258. <div class="modal-footer">
  1259. <a href="#" class="btn" data-dismiss="modal">${_('Close')}</a>
  1260. </div>
  1261. </div>
  1262. <script type="text/javascript">
  1263. % if is_embeddable:
  1264. var MAIN_SCROLLABLE = '.page-content';
  1265. % else:
  1266. var MAIN_SCROLLABLE = '.content-panel';
  1267. % endif
  1268. (function () {
  1269. if (ko.options) {
  1270. ko.options.deferUpdates = true;
  1271. }
  1272. var MAPPINGS = {
  1273. SOLR_TO_HIVE: {
  1274. "string": "string",
  1275. "plong": "bigint",
  1276. "pdouble": "double",
  1277. "pdate": "timestamp",
  1278. "long": "bigint",
  1279. "double": "double",
  1280. "date": "timestamp",
  1281. "boolean": "boolean",
  1282. },
  1283. HIVE_TO_SOLR: {
  1284. "bigint": "plong"
  1285. },
  1286. get: function(type, key, defaultValue) {
  1287. return type[key] || defaultValue
  1288. }
  1289. };
  1290. var fieldNum = 0;
  1291. var getNewFieldName = function () {
  1292. fieldNum++;
  1293. return "new_field_" + fieldNum
  1294. };
  1295. var createDefaultField = function () {
  1296. var defaultField = ko.mapping.fromJS(${default_field_type | n});
  1297. defaultField.name = ko.observable(getNewFieldName());
  1298. return defaultField;
  1299. };
  1300. var Operation = function (type) {
  1301. var self = this;
  1302. var createArgumentValue = function (arg) {
  1303. if (arg.type === "mapping") {
  1304. return ko.observableArray([]);
  1305. }
  1306. if (arg.type === "checkbox") {
  1307. return ko.observable(false);
  1308. }
  1309. return ko.observable("");
  1310. };
  1311. var constructSettings = function (type) {
  1312. var settings = {};
  1313. var operation = viewModel.createWizard.operationTypes.find(function (currOperation) {
  1314. return currOperation.name === type;
  1315. });
  1316. for (var i = 0; i < operation.args.length; i++) {
  1317. var argVal = createArgumentValue(operation.args[i]);
  1318. if (operation.args[i].type === "checkbox" && operation.outputType === "checkbox_fields") {
  1319. argVal.subscribe(function (newVal) {
  1320. if (newVal) {
  1321. self.fields.push(createDefaultField());
  1322. } else {
  1323. self.fields.pop();
  1324. }
  1325. });
  1326. }
  1327. settings[operation.args[i].name] = argVal;
  1328. }
  1329. settings.getArguments = function () {
  1330. return operation.args
  1331. };
  1332. settings.outputType = function () {
  1333. return operation.outputType;
  1334. };
  1335. return settings;
  1336. };
  1337. var init = function () {
  1338. self.fields([]);
  1339. self.numExpectedFields(0);
  1340. self.numExpectedFields.subscribe(function (numExpectedFields) {
  1341. if (numExpectedFields < self.fields().length) {
  1342. self.fields(self.fields().slice(0, numExpectedFields));
  1343. } else if (numExpectedFields > self.fields().length) {
  1344. var difference = numExpectedFields - self.fields().length;
  1345. for (var i = 0; i < difference; i++) {
  1346. self.fields.push(createDefaultField());
  1347. }
  1348. }
  1349. });
  1350. self.settings(constructSettings(self.type()));
  1351. };
  1352. self.load = function (data) {
  1353. self.numExpectedFields(data.numExpectedFields);
  1354. var newSettings = constructSettings(data.type);
  1355. for (var key in data.settings) {
  1356. newSettings[key] = ko.mapping.fromJS(data.settings[key]);
  1357. }
  1358. self.settings(newSettings);
  1359. data.fields.forEach(function (field) {
  1360. self.fields.push(loadField(field));
  1361. });
  1362. };
  1363. self.type = ko.observable(type);
  1364. self.fields = ko.observableArray();
  1365. self.numExpectedFields = ko.observable();
  1366. self.settings = ko.observable();
  1367. init();
  1368. self.type.subscribe(function () {
  1369. init();
  1370. });
  1371. };
  1372. var FileType = function (typeName, args) {
  1373. var self = this;
  1374. var type;
  1375. var init = function () {
  1376. self.type = ko.observable(typeName);
  1377. var types = viewModel.createWizard.fileTypes;
  1378. for (var i = 0; i < types.length; i++) {
  1379. if (types[i].name === typeName) {
  1380. type = types[i];
  1381. break;
  1382. }
  1383. }
  1384. if (type) {
  1385. for (var i = 0; i < type.args.length; i++) {
  1386. self[type.args[i].name] = ko.observable();
  1387. }
  1388. if (args) {
  1389. loadFromObj(args);
  1390. }
  1391. }
  1392. };
  1393. var loadFromObj = function (args) {
  1394. for (var attr in args) {
  1395. self[attr] = ko.mapping.fromJS(args[attr]);
  1396. }
  1397. };
  1398. self.getArguments = function () {
  1399. return type ? type.args : [];
  1400. };
  1401. self.isCustomizable = function () {
  1402. return type.isCustomizable;
  1403. };
  1404. init();
  1405. };
  1406. var Source = function (vm, wizard) {
  1407. var self = this;
  1408. self.sourceType = vm.sourceType;
  1409. self.name = ko.observable('');
  1410. self.sample = ko.observableArray();
  1411. self.sampleCols = ko.observableArray();
  1412. self.namespace = wizard.namespace;
  1413. self.compute = wizard.compute;
  1414. var refreshThrottle = -1;
  1415. var sampleColSubDisposals = [];
  1416. var lastStatement = '';
  1417. var refreshTemporaryTable = function (sampleCols) {
  1418. window.clearTimeout(refreshThrottle);
  1419. window.setTimeout(function () {
  1420. while (sampleColSubDisposals.length) {
  1421. sampleColSubDisposals.pop()();
  1422. }
  1423. if (!self.sampleCols().length) {
  1424. return;
  1425. }
  1426. var tableName = 'input';
  1427. switch (self.inputFormat()) {
  1428. case 'stream':
  1429. if (self.streamSelection() === 'kafka' && self.kafkaSelectedTopics()) {
  1430. tableName = self.kafkaSelectedTopics();
  1431. }
  1432. break;
  1433. case 'file' :
  1434. tableName = self.path().split('/').pop();
  1435. break;
  1436. case 'table':
  1437. case 'rdbms':
  1438. tableName = self.tableName();
  1439. }
  1440. var statementCols = [];
  1441. var temporaryColumns = [];
  1442. sampleCols.forEach(function (sampleCol) {
  1443. statementCols.push(SqlUtils.backTickIfNeeded(self.sourceType, sampleCol.name()));
  1444. var col = {
  1445. name: sampleCol.name(),
  1446. type: sampleCol.type()
  1447. };
  1448. temporaryColumns.push(col);
  1449. var colNameSub = sampleCol.name.subscribe(function () {
  1450. refreshTemporaryTable(self.sampleCols())
  1451. });
  1452. var colTypeSub = sampleCol.type.subscribe(function () {
  1453. refreshTemporaryTable(self.sampleCols())
  1454. });
  1455. sampleColSubDisposals.push(function () {
  1456. colNameSub.dispose();
  1457. colTypeSub.dispose();
  1458. })
  1459. });
  1460. var statement = 'SELECT ';
  1461. statement += statementCols.join(',\n ');
  1462. statement += '\n FROM ' + SqlUtils.backTickIfNeeded(self.sourceType, tableName) + ';';
  1463. if (!wizard.destination.fieldEditorValue() || wizard.destination.fieldEditorValue() === lastStatement) {
  1464. wizard.destination.fieldEditorValue(statement);
  1465. }
  1466. lastStatement = statement;
  1467. wizard.destination.fieldEditorPlaceHolder('${ _('Example: SELECT') }' + ' * FROM ' + SqlUtils.backTickIfNeeded(self.sourceType, tableName));
  1468. var handle = DataCatalog.addTemporaryTable({
  1469. sourceType: self.sourceType,
  1470. namespace: self.namespace(),
  1471. compute: self.compute(),
  1472. name: tableName,
  1473. columns: temporaryColumns
  1474. });
  1475. sampleColSubDisposals.push(function () {
  1476. handle.delete();
  1477. })
  1478. }, 500)
  1479. };
  1480. self.sampleCols.subscribe(refreshTemporaryTable);
  1481. self.inputFormat = ko.observable(wizard.prefill.source_type() ? wizard.prefill.source_type() : 'file');
  1482. self.inputFormat.subscribe(function(val) {
  1483. wizard.destination.columns.removeAll();
  1484. if (self.sample()) {
  1485. self.sample.removeAll();
  1486. }
  1487. self.path('');
  1488. resizeElements();
  1489. if (val === 'stream') {
  1490. if (self.streamSelection() === 'kafka') {
  1491. wizard.guessFormat();
  1492. wizard.destination.tableFormat('kudu');
  1493. } else {
  1494. wizard.destination.tableFormat('text');
  1495. }
  1496. } else if (val === 'table') {
  1497. wizard.destination.outputFormat('altus');
  1498. } else if (val === 'rdbms') {
  1499. self.rdbmsMode('customRdbms');
  1500. }
  1501. });
  1502. self.inputFormatsAll = ko.observableArray([
  1503. {'value': 'file', 'name': 'File'},
  1504. % if ENABLE_SQOOP.get():
  1505. {'value': 'rdbms', 'name': 'External Database'},
  1506. % endif
  1507. % if ENABLE_KAFKA.get():
  1508. {'value': 'stream', 'name': 'Stream'},
  1509. % endif
  1510. % if ENABLE_ALTUS.get():
  1511. {'value': 'table', 'name': 'Table'},
  1512. % endif
  1513. % if ENABLE_SQL_INDEXER.get():
  1514. {'value': 'query', 'name': 'SQL Query'},
  1515. % endif
  1516. ## TODO: Rename to 'Connectors'
  1517. % if ENABLE_ENVELOPE.get():
  1518. {'value': 'connector', 'name': 'Connectors'},
  1519. % endif
  1520. {'value': 'manual', 'name': 'Manually'}
  1521. ##{'value': 'text', 'name': 'Paste Text'},
  1522. ]);
  1523. self.inputFormatsManual = ko.observableArray([
  1524. {'value': 'manual', 'name': 'Manually'}
  1525. ]);
  1526. self.inputFormats = ko.pureComputed(function() {
  1527. if (wizard.prefill.source_type() === 'manual') {
  1528. return self.inputFormatsManual();
  1529. }
  1530. return self.inputFormatsAll();
  1531. });
  1532. // File
  1533. self.path = ko.observable('');
  1534. self.path.subscribe(function(val) {
  1535. if (val) {
  1536. wizard.guessFormat();
  1537. wizard.destination.nonDefaultLocation(val);
  1538. }
  1539. resizeElements();
  1540. });
  1541. self.isObjectStore = ko.pureComputed(function() {
  1542. return self.inputFormat() === 'file' && /^(s3a|adl):\/.*$/.test(self.path());
  1543. });
  1544. self.isObjectStore.subscribe(function(newVal) {
  1545. wizard.destination.useDefaultLocation(!newVal);
  1546. });
  1547. // Rdbms
  1548. self.rdbmsMode = ko.observable();
  1549. self.rdbmsMode.subscribe(function (val) {
  1550. self.rdbmsTypes(null);
  1551. self.rdbmsType('');
  1552. self.rdbmsDatabaseName('');
  1553. self.rdbmsIsAllTables(false);
  1554. self.rdbmsJdbcDriver('');
  1555. self.rdbmsJdbcDriverName('');
  1556. self.rdbmsHostname('');
  1557. self.rdbmsPort('');
  1558. self.rdbmsUsername('');
  1559. self.rdbmsPassword('');
  1560. self.rdbmsDbIsValid(false);
  1561. if (val === 'configRdbms') {
  1562. $.post("${ url('indexer:get_drivers') }", {}, function (resp) {
  1563. if (resp.data) {
  1564. self.rdbmsTypes(resp.data);
  1565. window.setTimeout(function(){
  1566. self.rdbmsType(self.rdbmsTypes()[0].value);
  1567. }, 0);
  1568. }
  1569. });
  1570. } else if (val === 'customRdbms') {
  1571. self.rdbmsTypes([
  1572. {'value': 'jdbc', 'name': 'JDBC'},
  1573. {'value': 'mysql', 'name': 'MySQL'},
  1574. {'value': 'oracle', 'name': 'Oracle'},
  1575. {'value': 'postgresql', 'name': 'PostgreSQL'}
  1576. ]);
  1577. window.setTimeout(function(){
  1578. self.rdbmsType(self.rdbmsTypes()[0].value);
  1579. }, 0);
  1580. }
  1581. });
  1582. self.rdbmsTypes = ko.observableArray();
  1583. self.rdbmsType = ko.observable(null);
  1584. self.rdbmsType.subscribe(function (val) {
  1585. self.path('');
  1586. self.rdbmsDatabaseNames([]);
  1587. self.rdbmsDatabaseName('');
  1588. resizeElements();
  1589. if(self.rdbmsMode() === 'configRdbms' && val !== 'jdbc') {
  1590. self.isFetchingDatabaseNames(true);
  1591. $.post("${ url('indexer:get_db_component') }", {
  1592. "source": ko.mapping.toJSON(self)
  1593. }, function (resp) {
  1594. if (resp.data) {
  1595. self.rdbmsDatabaseNames(resp.data);
  1596. }
  1597. }).always(function(){
  1598. self.isFetchingDatabaseNames(false);
  1599. });
  1600. } else if(self.rdbmsMode() === 'configRdbms' && val === 'jdbc') {
  1601. self.isFetchingDriverNames(true);
  1602. $.post("${ url('indexer:jdbc_db_list') }", {
  1603. "source": ko.mapping.toJSON(self)
  1604. }, function (resp) {
  1605. if (resp.data) {
  1606. self.rdbmsJdbcDriverNames(resp.data);
  1607. }
  1608. }).always(function() {
  1609. self.isFetchingDriverNames(false);
  1610. });
  1611. }
  1612. });
  1613. self.rdbmsDatabaseName = ko.observable('');
  1614. self.rdbmsDatabaseName.subscribe(function (val) {
  1615. if (val !== '') {
  1616. self.isFetchingTableNames(true);
  1617. $.post("${ url('indexer:get_db_component') }", {
  1618. "source": ko.mapping.toJSON(self)
  1619. }, function (resp) {
  1620. if (resp.data) {
  1621. self.rdbmsTableNames(resp.data.map(function(opt) {
  1622. return ko.mapping.fromJS(opt);
  1623. }));
  1624. }
  1625. }).always(function(){
  1626. self.isFetchingTableNames(false);
  1627. });
  1628. }
  1629. });
  1630. self.rdbmsDatabaseNames = ko.observableArray();
  1631. self.isFetchingDatabaseNames = ko.observable(false);
  1632. self.rdbmsJdbcDriverNames = ko.observableArray();
  1633. self.isFetchingDriverNames = ko.observable(false);
  1634. self.rdbmsJdbcDriverName = ko.observable();
  1635. self.rdbmsJdbcDriverName.subscribe(function () {
  1636. self.rdbmsDatabaseNames([]);
  1637. self.rdbmsDatabaseName('');
  1638. self.isFetchingDatabaseNames(true);
  1639. $.post("${ url('indexer:get_db_component') }", {
  1640. "source": ko.mapping.toJSON(self)
  1641. }, function (resp) {
  1642. if (resp.data) {
  1643. self.rdbmsDatabaseNames(resp.data);
  1644. }
  1645. }).always(function(){
  1646. self.isFetchingDatabaseNames(false);
  1647. });
  1648. });
  1649. self.rdbmsJdbcDriver = ko.observable('');
  1650. self.rdbmsJdbcDriver.subscribe(function () {
  1651. self.rdbmsDatabaseNames([]);
  1652. });
  1653. self.isFetchingTableNames = ko.observable(false);
  1654. self.rdbmsTableNames = ko.observableArray();
  1655. self.rdbmsHostname = ko.observable('');
  1656. self.rdbmsHostname.subscribe(function () {
  1657. self.rdbmsDatabaseNames([]);
  1658. });
  1659. self.rdbmsPort = ko.observable('');
  1660. self.rdbmsPort.subscribe(function () {
  1661. self.rdbmsDatabaseNames([]);
  1662. });
  1663. self.rdbmsUsername = ko.observable('');
  1664. self.rdbmsUsername.subscribe(function () {
  1665. self.rdbmsDatabaseNames([]);
  1666. });
  1667. self.rdbmsPassword = ko.observable('');
  1668. self.rdbmsPassword.subscribe(function () {
  1669. self.rdbmsDatabaseNames([]);
  1670. });
  1671. self.rdbmsIsAllTables = ko.observable(false);
  1672. self.rdbmsIsAllTables.subscribe(function(newVal) {
  1673. if (newVal) {
  1674. wizard.destination.name(self.rdbmsDatabaseName());
  1675. } else {
  1676. wizard.destination.name(self.tables()[0] && self.tables()[0].name() || '');
  1677. }
  1678. });
  1679. self.rdbmsAllTablesSelected = ko.pureComputed(function () {
  1680. return self.rdbmsIsAllTables() || self.tables().length > 1;
  1681. });
  1682. self.rdbmsTablesExclude = ko.pureComputed(function () {
  1683. var rdbmsTables = self.rdbmsTableNames();
  1684. var tables = self.tables();
  1685. if (tables.length <= 1) {
  1686. return [];
  1687. }
  1688. var map = tables.reduce(function (map, table) {
  1689. map[table.name()] = 1;
  1690. return map;
  1691. }, {});
  1692. return rdbmsTables.map(function (table) {
  1693. return table.name();
  1694. }).filter(function (name) {
  1695. return !map[name];
  1696. });
  1697. });
  1698. self.rdbmsDbIsValid = ko.observable(false);
  1699. self.rdbmsCheckConnection = function() {
  1700. self.isFetchingDatabaseNames(true);
  1701. self.rdbmsDatabaseNames([]);
  1702. self.rdbmsDatabaseName(''); // Need to clear or else get_db_component will return list of tables
  1703. $.post("${ url('indexer:get_db_component') }", {
  1704. "source": ko.mapping.toJSON(self)
  1705. }, function (resp) {
  1706. if (resp.status === 0 && resp.data) {
  1707. self.rdbmsDbIsValid(true);
  1708. self.rdbmsDatabaseNames(resp.data);
  1709. } else if (resp.status === 1) {
  1710. $(document).trigger("error", "${ _('Connection Failed: ') }" + resp.message);
  1711. self.rdbmsDbIsValid(false);
  1712. }
  1713. }).always(function(){
  1714. self.isFetchingDatabaseNames(false);
  1715. });
  1716. };
  1717. // Table
  1718. self.tables = ko.observableArray([
  1719. ko.mapping.fromJS({ name: '' })
  1720. ]);
  1721. self.tables.subscribe(function(newVal) {
  1722. if (newVal.length == 0) {
  1723. wizard.destination.name('');
  1724. } else if (newVal.length == 1) {
  1725. wizard.destination.name(self.tables()[0].name() || ''); // TODO: db prefix and check if DB exists
  1726. } else {
  1727. wizard.destination.name(self.rdbmsDatabaseName());
  1728. }
  1729. });
  1730. self.tablesNames = ko.observableArray([]);
  1731. self.selectedTableIndex = ko.observable(0);
  1732. self.table = ko.pureComputed(function() {
  1733. var index = Math.min(self.selectedTableIndex(), self.tables().length - 1);
  1734. return self.tables().length > 0 ? self.tables()[index].name() : ''
  1735. });
  1736. self.tableName = ko.pureComputed(function() {
  1737. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[1] : self.table();
  1738. });
  1739. self.databaseName = ko.pureComputed(function() {
  1740. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[0] : 'default';
  1741. });
  1742. self.table.subscribe(function(val) {
  1743. if (val) {
  1744. wizard.guessFormat();
  1745. wizard.destination.nonDefaultLocation(val);
  1746. }
  1747. resizeElements();
  1748. });
  1749. // Queries
  1750. self.query = ko.observable('');
  1751. self.query.subscribe(function(newValue) {
  1752. if (newValue) {
  1753. viewModel.createWizard.guessFieldTypes();
  1754. }
  1755. });
  1756. self.draggedQuery = ko.observable();
  1757. // Connectors
  1758. self.connectorList = ko.observable([
  1759. {'value': 'sfdc', 'name': 'Salesforce'}
  1760. ]);
  1761. self.connectorSelection = ko.observable(self.connectorList()[0]['value']);
  1762. // Streams, Kafka, Flume
  1763. self.publicStreams = ko.observable([
  1764. {'value': 'kafka', 'name': 'Kafka Topics'},
  1765. {'value': 'flume', 'name': 'Flume Agent'}
  1766. ]);
  1767. self.streamSelection = ko.observable(self.publicStreams()[0]['value']);
  1768. self.streamSelection.subscribe(function(newValue) {
  1769. if (newValue == 'flume') {
  1770. $.post("${ url('metadata:manager_hosts') }", {
  1771. "service": "flume"
  1772. }, function (resp) {
  1773. if (resp.status === 0 && resp.hosts) {
  1774. self.channelSourceHosts(resp.hosts);
  1775. } else {
  1776. $(document).trigger("error", "${ _('Error getting hosts') }" + resp.message);
  1777. }
  1778. });
  1779. }
  1780. });
  1781. self.kafkaTopics = ko.observableArray();
  1782. self.kafkaSelectedTopics = ko.observable(''); // Currently designed just for one
  1783. self.kafkaSelectedTopics.subscribe(function(newValue) {
  1784. if (newValue) {
  1785. viewModel.createWizard.guessFieldTypes();
  1786. self.kafkaFieldNames($.totalStorage('pai' + '_kafka_topics_' + newValue + '_kafkaFieldNames'));
  1787. self.kafkaFieldTypes($.totalStorage('pai' + '_kafka_topics_' + newValue + '_kafkaFieldTypes'));
  1788. }
  1789. });
  1790. self.kafkaSchemaManual = ko.observable('manual');
  1791. self.kafkaFieldType = ko.observable('delimited');
  1792. self.kafkaFieldDelimiter = ko.observable(',');
  1793. self.kafkaFieldNames = ko.observable('');
  1794. self.kafkaFieldNames.subscribe(function(newValue) {
  1795. $.totalStorage('pai' + '_kafka_topics_' + self.kafkaSelectedTopics() + '_kafkaFieldNames', newValue);
  1796. viewModel.createWizard.guessFieldTypes();
  1797. });
  1798. self.kafkaFieldTypes = ko.observable('');
  1799. self.kafkaFieldTypes.subscribe(function(newValue) {
  1800. $.totalStorage('pai' + '_kafka_topics_' + self.kafkaSelectedTopics() + '_kafkaFieldTypes', newValue);
  1801. viewModel.createWizard.guessFieldTypes();
  1802. });
  1803. self.kafkaFieldSchemaPath = ko.observable('');
  1804. self.channelSourceTypes = ko.observableArray([
  1805. {'name': '${ _("Directory or File") }', 'value': 'directory'},
  1806. {'name': '${ _("Program") }', 'value': 'exec'},
  1807. {'name': '${ _("Syslogs") }', 'value': 'syslogs'},
  1808. {'name': '${ _("HTTP") }', 'value': 'http'}
  1809. ]);
  1810. self.channelSourceType = ko.observable();
  1811. self.channelSourceHosts = ko.observableArray();
  1812. self.channelSourceSelectedHosts = ko.observableArray([]);
  1813. self.channelSourceSelectedHosts.subscribe(function(newVal) {
  1814. if (newVal) {
  1815. viewModel.createWizard.guessFieldTypes();
  1816. }
  1817. })
  1818. self.channelSourcePath = ko.observable('/var/log/hue-httpd/access_log');
  1819. self.streamUsername = ko.observable('');
  1820. self.streamPassword = ko.observable('');
  1821. self.streamToken = ko.observable('');
  1822. self.streamToken.subscribe(function(newVal) {
  1823. if (newVal) {
  1824. wizard.guessFormat(); // Todo
  1825. }
  1826. });
  1827. self.streamEndpointUrl = ko.observable('https://login.salesforce.com/services/Soap/u/42.0');
  1828. self.streamObjects = ko.observableArray();
  1829. self.streamObject = ko.observable();
  1830. self.streamObject.subscribe(function(newValue) {
  1831. if (newValue) {
  1832. wizard.guessFieldTypes();
  1833. }
  1834. });
  1835. self.hasStreamSelected = ko.pureComputed(function() {
  1836. return (self.streamSelection() === 'kafka' && self.kafkaSelectedTopics()) ||
  1837. (self.streamSelection() === 'sfdc' && self.streamObject())
  1838. });
  1839. self.hasStreamSelected.subscribe(function(newValue) {
  1840. if (newValue) {
  1841. wizard.guessFormat();
  1842. if (self.streamSelection() === 'kafka') {
  1843. wizard.destination.tableFormat('kudu');
  1844. } else {
  1845. wizard.destination.tableFormat('text');
  1846. }
  1847. }
  1848. });
  1849. self.streamCheckConnection = function() {
  1850. $(".jHueNotify").remove();
  1851. $.post("${ url('indexer:get_db_component') }", {
  1852. "source": ko.mapping.toJSON(self)
  1853. }, function (resp) {
  1854. if (resp.status === 0 && resp.data) {
  1855. huePubSub.publish('notebook.task.submitted', resp.history_uuid);
  1856. } else if (resp.status === 1) {
  1857. $(document).trigger("error", "${ _('Connection Failed: ') }" + resp.message);
  1858. self.rdbmsDbIsValid(false);
  1859. }
  1860. });
  1861. };
  1862. self.format = ko.observable();
  1863. self.format.subscribe(function(newVal) {
  1864. if (typeof newVal.hasHeader !== 'undefined') {
  1865. wizard.destination.hasHeader(newVal.hasHeader());
  1866. newVal.hasHeader.subscribe(function(newVal) {
  1867. wizard.destination.hasHeader(newVal);
  1868. });
  1869. }
  1870. if (typeof newVal.fieldSeparator !== 'undefined') {
  1871. wizard.destination.useCustomDelimiters(newVal.fieldSeparator() !== ',');
  1872. wizard.destination.customFieldDelimiter(newVal.fieldSeparator());
  1873. newVal.fieldSeparator.subscribe(function(newVal) {
  1874. if (newVal !== '') {
  1875. wizard.destination.customFieldDelimiter(newVal);
  1876. }
  1877. });
  1878. }
  1879. });
  1880. self.show = ko.pureComputed(function() {
  1881. if (self.inputFormat() === 'file') {
  1882. return self.path().length > 0;
  1883. }
  1884. if (self.inputFormat() === 'table') {
  1885. return self.tableName().length > 0;
  1886. }
  1887. if (self.inputFormat() === 'query') {
  1888. return self.query();
  1889. }
  1890. if (self.inputFormat() === 'manual') {
  1891. return true;
  1892. }
  1893. if (self.inputFormat() === 'stream') {
  1894. if (self.streamSelection() === 'kafka') {
  1895. return self.kafkaSelectedTopics() && self.kafkaSelectedTopics().length > 0;
  1896. } else if (self.streamSelection() === 'flume') {
  1897. return self.channelSourceSelectedHosts().length > 0;
  1898. }
  1899. }
  1900. if (self.inputFormat() === 'sfdc') {
  1901. return self.streamUsername().length > 0 &&
  1902. self.streamPassword().length > 0 &&
  1903. self.streamToken().length > 0 &&
  1904. self.streamEndpointUrl().length > 0 &&
  1905. self.streamObject();
  1906. }
  1907. if (self.inputFormat() === 'rdbms') {
  1908. return (self.rdbmsDatabaseName().length > 0 && self.tables().length > 0) || self.rdbmsAllTablesSelected();
  1909. }
  1910. });
  1911. };
  1912. var Destination = function (vm, wizard) {
  1913. var self = this;
  1914. self.sourceType = vm.sourceType;
  1915. self.name = ko.observable('').extend({ throttle: 500 });
  1916. self.nameChanged = function(name) {
  1917. var exists = false;
  1918. var checkDbEntryExists = function () {
  1919. wizard.computeSetDeferred.done(function () {
  1920. DataCatalog.getEntry({
  1921. sourceType: self.sourceType,
  1922. compute: wizard.compute(),
  1923. namespace: wizard.namespace(),
  1924. path: self.outputFormat() === 'table' ? [self.databaseName(), self.tableName()] : [self.databaseName()],
  1925. }).done(function (catalogEntry) {
  1926. catalogEntry.getSourceMeta({ silenceErrors: true }).done(function (sourceMeta) {
  1927. self.isTargetExisting(!sourceMeta.notFound);
  1928. self.isTargetChecking(false);
  1929. }).fail(function () {
  1930. self.isTargetExisting(false);
  1931. self.isTargetChecking(false);
  1932. })
  1933. });
  1934. })
  1935. };
  1936. if (name.length === 0) {
  1937. self.isTargetExisting(false);
  1938. self.isTargetChecking(false);
  1939. } else if (self.outputFormat() === 'file') {
  1940. // self.path()
  1941. } else if (self.outputFormat() === 'table' && wizard.isValidDestination()) {
  1942. if (self.tableName() !== '') {
  1943. self.isTargetExisting(false);
  1944. self.isTargetChecking(true);
  1945. checkDbEntryExists();
  1946. } else {
  1947. self.isTargetExisting(false);
  1948. self.isTargetChecking(false);
  1949. }
  1950. } else if (self.outputFormat() === 'hbase') {
  1951. // Todo once autocomplete is implemented for hbase
  1952. } else if (self.outputFormat() === 'database' && wizard.isValidDestination()) {
  1953. if (self.databaseName() !== '') {
  1954. self.isTargetExisting(false);
  1955. self.isTargetChecking(true);
  1956. checkDbEntryExists();
  1957. } else {
  1958. self.isTargetExisting(false);
  1959. self.isTargetChecking(false);
  1960. }
  1961. } else if (self.outputFormat() === 'index') {
  1962. $.post("${ url('indexer:list_index') }", {
  1963. name: self.name()
  1964. }, function (data) {
  1965. self.isTargetExisting(data.status === 0);
  1966. self.isTargetChecking(false);
  1967. }).fail(function (xhr, textStatus, errorThrown) {
  1968. self.isTargetExisting(false);
  1969. self.isTargetChecking(false);
  1970. });
  1971. $.post("/indexer/api/configs/list/", function (data) {
  1972. if (data.status === 0) {
  1973. self.indexerConfigSets(data.configs.map(function (config) {
  1974. return {'name': config, 'value': config};
  1975. }));
  1976. } else {
  1977. $(document).trigger("error", data.message);
  1978. }
  1979. });
  1980. }
  1981. resizeElements();
  1982. };
  1983. self.name.subscribe(self.nameChanged);
  1984. self.description = ko.observable('');
  1985. self.outputFormat = ko.observable(wizard.prefill.target_type() || 'table');
  1986. self.outputFormat.subscribe(function (newValue) {
  1987. if (newValue && newValue !== 'database' && ((['table', 'rdbms', 'index'].indexOf(newValue) >= 0) && wizard.source.table().length > 0)) {
  1988. self.nameChanged(self.name());
  1989. wizard.guessFieldTypes();
  1990. resizeElements();
  1991. }
  1992. });
  1993. self.outputFormatsList = ko.observableArray([
  1994. {'name': '${ _("Table") }', 'value': 'table'},
  1995. % if ENABLE_NEW_INDEXER.get():
  1996. {'name': '${ _("Search index") }', 'value': 'index'},
  1997. % endif
  1998. {'name': '${ _("Database") }', 'value': 'database'},
  1999. % if ENABLE_SQOOP.get() or ENABLE_KAFKA.get():
  2000. {'name': '${ _("Folder") }', 'value': 'file'},
  2001. % endif
  2002. % if ENABLE_KAFKA.get():
  2003. {'name': '${ _("Stream") }', 'value': 'stream'},
  2004. % endif
  2005. % if ENABLE_ALTUS.get():
  2006. {'name': '${ _("Altus SDX") }', 'value': 'altus'},
  2007. % endif
  2008. % if ENABLE_SQOOP.get():
  2009. {'name': '${ _("HBase Table") }', 'value': 'hbase'},
  2010. % endif
  2011. ]);
  2012. self.outputFormats = ko.pureComputed(function() {
  2013. return $.grep(self.outputFormatsList(), function(format) {
  2014. if (format.value === 'database' && wizard.source.inputFormat() !== 'manual' && (wizard.source.inputFormat() !== 'rdbms' || !wizard.source.rdbmsAllTablesSelected())) {
  2015. return false;
  2016. }
  2017. if (format.value === 'file' && ['manual', 'rdbms', 'stream'].indexOf(wizard.source.inputFormat()) === -1) {
  2018. return false;
  2019. }
  2020. if (format.value === 'index' && ['file', 'query', 'stream', 'manual'].indexOf(wizard.source.inputFormat()) === -1) {
  2021. return false;
  2022. }
  2023. if (format.value === 'table' && (wizard.source.inputFormat() === 'table' || (wizard.source.inputFormat() === 'rdbms' && wizard.source.rdbmsAllTablesSelected()))) {
  2024. return false;
  2025. }
  2026. if (format.value === 'altus' && ['table'].indexOf(wizard.source.inputFormat()) === -1) {
  2027. return false;
  2028. }
  2029. if (format.value === 'stream' && ['file', 'stream'].indexOf(wizard.source.inputFormat()) === -1) {
  2030. return false;
  2031. }
  2032. if (format.value === 'hbase' && (wizard.source.inputFormat() !== 'rdbms' || wizard.source.rdbmsAllTablesSelected())) {
  2033. return false;
  2034. }
  2035. return true;
  2036. })
  2037. });
  2038. self.outputFormats.subscribe(function (newValue) {
  2039. if (newValue.length && newValue.map(function (entry) { return entry.value; }).indexOf(self.outputFormat()) < 0) {
  2040. self.outputFormat(newValue && newValue[0].value);
  2041. }
  2042. });
  2043. wizard.prefill.target_type.subscribe(function(newValue) {
  2044. self.outputFormat(newValue || 'table');
  2045. if (newValue === 'database') {
  2046. vm.currentStep(2);
  2047. } else {
  2048. vm.currentStep(1);
  2049. }
  2050. });
  2051. self.defaultName = ko.pureComputed(function() {
  2052. var name = '';
  2053. if (wizard.source.inputFormat() === 'file' || wizard.source.inputFormat() === 'stream') {
  2054. if (self.outputFormat() === 'table') {
  2055. name = wizard.prefill.target_path().length > 0 ? wizard.prefill.target_path() : 'default';
  2056. if (wizard.source.inputFormat() === 'stream') {
  2057. if (wizard.source.streamSelection() === 'kafka') {
  2058. name += '.' + wizard.source.kafkaSelectedTopics();
  2059. } else {
  2060. name += '.' + wizard.source.streamObject();
  2061. }
  2062. } else if (wizard.source.path()) {
  2063. name += '.' + wizard.source.path().split('/').pop().split('.')[0];
  2064. }
  2065. } else { // Index
  2066. name = wizard.prefill.target_path().length > 0 ? wizard.prefill.target_path() : wizard.source.path().split('/').pop().split('.')[0];
  2067. }
  2068. } else if (wizard.source.inputFormat() === 'table') {
  2069. name = wizard.source.databaseName();
  2070. } else if (wizard.source.inputFormat() === 'query') {
  2071. if (wizard.source.query()) {
  2072. name = wizard.source.name();
  2073. }
  2074. if (wizard.prefill.target_path().length > 0) {
  2075. name = wizard.prefill.target_path();
  2076. }
  2077. } else if (wizard.source.inputFormat() === 'manual') {
  2078. name = wizard.prefill.target_path().length > 0 ? wizard.prefill.target_path() + '.' : '';
  2079. }
  2080. return name.replace(/ /g, '_').toLowerCase();
  2081. });
  2082. self.defaultName.subscribe(function(newVal) {
  2083. self.name(newVal);
  2084. });
  2085. self.format = ko.observable();
  2086. self.columns = ko.observableArray();
  2087. self.namespaces = wizard.namespaces;
  2088. self.namespace = wizard.namespace;
  2089. self.compute = wizard.compute;
  2090. self.targetNamespaceId = ko.observable(self.namespace() ? self.namespace().id : undefined);
  2091. self.namespace.subscribe(function (namespace) {
  2092. if (namespace && namespace.id !== self.targetNamespaceId) {
  2093. self.targetNamespaceId(namespace.id);
  2094. }
  2095. })
  2096. self.targetNamespaceId.subscribe(function (namespaceId) {
  2097. if (namespaceId && (!self.namespace() || self.namespace().id !== namespaceId)) {
  2098. self.namespaces().some(function (namespace) {
  2099. if (namespaceId === namespace.id) {
  2100. self.namespace(namespace);
  2101. return true;
  2102. }
  2103. })
  2104. }
  2105. });
  2106. // UI
  2107. self.bulkColumnNames = ko.observable('');
  2108. self.showProperties = ko.observable(false);
  2109. self.columns.subscribe(function (newVal) {
  2110. self.bulkColumnNames(newVal.map(function (item) {
  2111. return item.name()
  2112. }).join(','));
  2113. });
  2114. self.processBulkColumnNames = function () {
  2115. var val = self.bulkColumnNames();
  2116. try {
  2117. if (val.indexOf("\"") === -1 && val.indexOf("'") === -1) {
  2118. val = '"' + val.replace(/,/gi, '","') + '"';
  2119. }
  2120. if (val.indexOf("[") === -1) {
  2121. val = "[" + val + "]";
  2122. }
  2123. var parsed = JSON.parse(val);
  2124. self.columns().forEach(function (item, cnt) {
  2125. if ($.trim(parsed[cnt]) !== '') {
  2126. item.name($.trim(parsed[cnt]));
  2127. }
  2128. });
  2129. }
  2130. catch (err) {
  2131. }
  2132. $('#fieldsBulkEditor').modal('hide');
  2133. };
  2134. self.isTargetExisting = ko.observable();
  2135. self.isTargetChecking = ko.observable(false);
  2136. self.existingTargetUrl = ko.pureComputed(function() { // Should open generic sample popup instead
  2137. if (self.isTargetExisting()) {
  2138. if (self.outputFormat() === 'file') {
  2139. // Todo
  2140. return '';
  2141. }
  2142. if (self.outputFormat() === 'table') {
  2143. return '/metastore/table/' + self.databaseName() + '/' + self.tableName();
  2144. }
  2145. if (self.outputFormat() === 'database') {
  2146. return '/metastore/tables/' + self.databaseName();
  2147. }
  2148. if (self.outputFormat() === 'index') {
  2149. return '/indexer/indexes/' + self.name();
  2150. }
  2151. }
  2152. return '';
  2153. });
  2154. // Table
  2155. self.tableName = ko.pureComputed(function() {
  2156. return self.outputFormat() === 'table' && self.name().indexOf('.') > 0 ? self.name().split('.', 2)[1] : self.name();
  2157. });
  2158. self.databaseName = ko.pureComputed(function() {
  2159. return self.outputFormat() === 'database' ? self.name() : (self.outputFormat() === 'table' && self.name().indexOf('.') > 0 ? self.name().split('.', 2)[0] : 'default');
  2160. });
  2161. self.tableFormat = ko.observable('text');
  2162. self.KUDU_DEFAULT_RANGE_PARTITION_COLUMN = {values: [{value: ''}], name: 'VALUES', lower_val: 0, include_lower_val: '<=', upper_val: 1, include_upper_val: '<='};
  2163. self.KUDU_DEFAULT_PARTITION_COLUMN = {columns: [], range_partitions: [self.KUDU_DEFAULT_RANGE_PARTITION_COLUMN], name: 'HASH', int_val: 16};
  2164. self.tableFormats = ko.pureComputed(function() {
  2165. if (wizard.source.inputFormat() === 'kafka') {
  2166. return [{'value': 'kudu', 'name': 'Kudu'}];
  2167. }
  2168. return [
  2169. {'value': 'text', 'name': 'Text'},
  2170. {'value': 'parquet', 'name': 'Parquet'},
  2171. {'value': 'kudu', 'name': 'Kudu'},
  2172. {'value': 'csv', 'name': 'Csv'},
  2173. {'value': 'avro', 'name': 'Avro'},
  2174. {'value': 'json', 'name': 'Json'},
  2175. {'value': 'regexp', 'name': 'Regexp'},
  2176. {'value': 'orc', 'name': 'ORC'},
  2177. ]
  2178. });
  2179. self.partitionColumns = ko.observableArray();
  2180. self.kuduPartitionColumns = ko.observableArray();
  2181. self.primaryKeys = ko.observableArray();
  2182. self.primaryKeyObjects = ko.observableArray();
  2183. self.useFieldEditor = ko.observable(false);
  2184. // TODO: Figure out the database to use for field editor autocomplete
  2185. self.fieldEditorDatabase = ko.observable('default');
  2186. // TODO: Do something with the editor value
  2187. self.fieldEditorValue = ko.observable();
  2188. self.fieldEditorPlaceHolder = ko.observable();
  2189. self.fieldEditorEnabled = ko.pureComputed(function () {
  2190. return '${ ENABLE_FIELD_EDITOR.get() }' == 'True';
  2191. });
  2192. self.importData = ko.observable(true);
  2193. self.useDefaultLocation = ko.observable(true);
  2194. self.nonDefaultLocation = ko.observable('');
  2195. self.hasHeader = ko.observable(false);
  2196. self.useCustomDelimiters = ko.observable(false);
  2197. self.customFieldDelimiter = ko.observable(',');
  2198. self.customCollectionDelimiter = ko.observable('\\002');
  2199. self.customMapDelimiter = ko.observable('\\003');
  2200. self.customRegexp = ko.observable('');
  2201. // Index
  2202. self.indexerRunJob = ko.observable(false);
  2203. self.indexerJobLibPath = ko.observable('${ CONFIG_INDEXER_LIBS_PATH.get() }');
  2204. self.indexerConfigSet = ko.observable('');
  2205. self.indexerConfigSets = ko.observableArray([]);
  2206. self.indexerNumShards = ko.observable(1);
  2207. self.indexerReplicationFactor = ko.observable(1);
  2208. self.indexerPrimaryKey = ko.observableArray();
  2209. self.indexerPrimaryKeyObject = ko.observableArray();
  2210. self.indexerDefaultField = ko.observableArray();
  2211. self.indexerDefaultFieldObject = ko.observableArray();
  2212. // File, Table, HBase
  2213. self.sqoopJobLibPaths = ko.observableArray([]);
  2214. self.addSqoopJobLibPath = function() {
  2215. var newValue = {
  2216. path: ko.observable('')
  2217. };
  2218. self.sqoopJobLibPaths.push(newValue);
  2219. };
  2220. self.addSqoopJobLibPath();
  2221. self.removeSqoopJobLibPath = function (valueToRemove) {
  2222. self.sqoopJobLibPaths.remove(valueToRemove);
  2223. };
  2224. self.numMappers = ko.observable(1);
  2225. self.customFieldsDelimiter = ko.observable(',');
  2226. self.customLineDelimiter = ko.observable('\\n');
  2227. self.customEnclosedByDelimiter = ko.observable('\'');
  2228. self.rdbmsFileOutputFormat = ko.observable('text');
  2229. self.rdbmsFileOutputFormats = ko.observableArray([
  2230. {'value': 'text', 'name': 'text'},
  2231. {'value': 'sequence', 'name': 'sequence'},
  2232. {'value': 'avro', 'name': 'avro'}
  2233. ]);
  2234. self.rdbmsSplitByColumn = ko.observableArray();
  2235. // Flume
  2236. self.channelSinkTypes = ko.observableArray([
  2237. {'name': '${ _("This topic") }', 'value': 'kafka'},
  2238. {'name': '${ _("Solr") }', 'value': 'solr'},
  2239. {'name': '${ _("HDFS") }', 'value': 'hdfs'}
  2240. ]);
  2241. self.channelSinkType = ko.observable();
  2242. self.channelSinkPath = ko.observable();
  2243. };
  2244. var CreateWizard = function (vm) {
  2245. var self = this;
  2246. var guessFieldTypesXhr;
  2247. self.namespaces = ko.observableArray();
  2248. self.namespace = ko.observable();
  2249. self.compute = ko.observable();
  2250. self.computeSetDeferred = $.Deferred();
  2251. ContextCatalog.getNamespaces({ sourceType: vm.sourceType }).done(function (context) {
  2252. self.namespaces(context.namespaces);
  2253. if (!vm.namespaceId || !context.namespaces.some(function (namespace) {
  2254. if (namespace.id === vm.namespaceId) {
  2255. self.namespace(namespace);
  2256. return true;
  2257. }
  2258. })) {
  2259. self.namespace(context.namespaces[0]);
  2260. }
  2261. if (!vm.computeId || !self.namespace().computes.some(function (compute) {
  2262. if (compute.id === vm.computeId) {
  2263. self.compute(compute);
  2264. return true;
  2265. }
  2266. })) {
  2267. self.compute(self.namespace().computes[0]);
  2268. }
  2269. self.namespace.subscribe(function (namespace) {
  2270. if (!namespace) {
  2271. self.compute(undefined);
  2272. return;
  2273. }
  2274. if (!self.compute() || !namespace.computes.some(function (compute) {
  2275. if (compute.id === self.compute()) {
  2276. return true;
  2277. }
  2278. })) {
  2279. if (namespace.computes.length) {
  2280. self.compute(namespace.computes[0]);
  2281. }
  2282. }
  2283. })
  2284. self.computeSetDeferred.resolve();
  2285. });
  2286. self.fileType = ko.observable();
  2287. self.fileType.subscribe(function (newType) {
  2288. if (self.source.format()) {
  2289. self.source.format().type(newType.name);
  2290. }
  2291. });
  2292. self.fileTypeName = ko.observable();
  2293. self.fileTypeName.subscribe(function (newType) {
  2294. for (var i = 0; i < self.fileTypes.length; i++) {
  2295. if (self.fileTypes[i].name === newType) {
  2296. self.fileType(self.fileTypes[i]);
  2297. break;
  2298. }
  2299. }
  2300. });
  2301. self.operationTypes = ${operators_json | n};
  2302. self.fieldTypes = ${fields_json | n}.solr;
  2303. self.hivePrimitiveFieldTypes = ${fields_json | n}.hivePrimitive;
  2304. self.hiveFieldTypes = ${fields_json | n}.hive;
  2305. self.fileTypes = ${file_types_json | n};
  2306. self.prefill = ko.mapping.fromJS(${prefill | n});
  2307. self.prefill.source_type.subscribe(function(newValue) {
  2308. self.source.inputFormat(newValue ? newValue : 'file');
  2309. });
  2310. self.show = ko.observable(true);
  2311. self.showCreate = ko.observable(false);
  2312. self.source = new Source(vm, self);
  2313. self.destination = new Destination(vm, self);
  2314. self.customDelimiters = ko.observableArray([
  2315. {'value': ',', 'name': '${ _("Comma (,)") }'},
  2316. {'value': '\\t', 'name': '${ _("^Tab (\\t)") }'},
  2317. {'value': '\\n', 'name': '${ _("New line") }'},
  2318. {'value': '|', 'name': '${ _("Pipe") }'},
  2319. {'value': '\"', 'name': '${ _("Double Quote") }'},
  2320. {'value': '\'', 'name': '${ _("Single Quote") }'},
  2321. {'value': '\x00', 'name': '${ _("^0") }'},
  2322. {'value': '\x01', 'name': '${ _("^A (\\001)") }'},
  2323. {'value': '\x02', 'name': '${ _("^B (\\002)") }'},
  2324. {'value': '\x03', 'name': '${ _("^C (\\003)") }'},
  2325. {'value': '\x01', 'name': '${ _("^A (\\x01)") }'}
  2326. ]);
  2327. self.editorId = ko.observable();
  2328. self.jobId = ko.observable();
  2329. self.editorVM = null;
  2330. self.indexingStarted = ko.observable(false);
  2331. self.isValidDestination = ko.pureComputed(function() {
  2332. return self.destination.name().length > 0 && (
  2333. (['table', 'database'].indexOf(self.destination.outputFormat()) === -1 || /^([a-zA-Z0-9_]+\.)?[a-zA-Z0-9_]+$/.test(self.destination.name())) &&
  2334. (['index'].indexOf(self.destination.outputFormat()) === -1 || /^[^\\/:]+$/.test(self.destination.name()))
  2335. );
  2336. });
  2337. self.readyToIndex = ko.pureComputed(function () {
  2338. var validFields = self.destination.columns().length || self.destination.outputFormat() === 'database';
  2339. var validTableColumns = self.destination.outputFormat() !== 'table' || ($.grep(self.destination.columns(), function(column) {
  2340. return column.name().length === 0;
  2341. }).length === 0
  2342. && $.grep(self.destination.partitionColumns(), function(column) {
  2343. return column.name().length === 0 || (self.source.inputFormat() !== 'manual' && column.partitionValue().length === 0);
  2344. }).length === 0
  2345. );
  2346. var isTargetAlreadyExisting;
  2347. if (self.destination.outputFormat() === 'database' && self.source.inputFormat() === 'rdbms') {
  2348. isTargetAlreadyExisting = self.destination.isTargetExisting();
  2349. } else {
  2350. isTargetAlreadyExisting = !self.destination.isTargetExisting()
  2351. || self.destination.outputFormat() === 'index'
  2352. ## Authorize submission but should check if schema is compatible
  2353. || (self.source.inputFormat() === 'stream' && self.destination.outputFormat() === 'table' && self.destination.tableFormat() === 'kudu')
  2354. ;
  2355. }
  2356. var isValidTable = self.destination.outputFormat() !== 'table' || (
  2357. self.destination.tableFormat() !== 'kudu' || (
  2358. $.grep(self.destination.kuduPartitionColumns(), function(partition) {
  2359. return partition.columns().length > 0
  2360. }).length === self.destination.kuduPartitionColumns().length && self.destination.primaryKeys().length > 0
  2361. )
  2362. );
  2363. var validIndexFields = self.destination.outputFormat() !== 'index' || ($.grep(self.destination.columns(), function(column) {
  2364. return ! (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(column.name()) && column.name() !== '_version_');
  2365. }).length === 0
  2366. ) || self.destination.indexerConfigSet();
  2367. return self.isValidDestination() && validFields && validTableColumns && validIndexFields && isTargetAlreadyExisting && isValidTable;
  2368. });
  2369. self.formatTypeSubscribed = false;
  2370. self.source.format.subscribe(function () {
  2371. for (var i = 0; i < self.fileTypes.length; i++) {
  2372. if (self.fileTypes[i].name === self.source.format().type()) {
  2373. self.fileType(self.fileTypes[i]);
  2374. self.fileTypeName(self.fileTypes[i].name);
  2375. break;
  2376. }
  2377. }
  2378. if (self.source.format().type) {
  2379. if (! self.formatTypeSubscribed) {
  2380. self.formatTypeSubscribed = true;
  2381. self.source.format().type.subscribe(function (newType) {
  2382. self.source.format(new FileType(newType));
  2383. self.destination.columns.removeAll();
  2384. self.guessFieldTypes();
  2385. });
  2386. }
  2387. }
  2388. });
  2389. self.isGuessingFormat = ko.observable(false);
  2390. self.guessFormat = function () {
  2391. self.isGuessingFormat(true);
  2392. self.destination.columns.removeAll();
  2393. $.post("${ url('indexer:guess_format') }", {
  2394. "fileFormat": ko.mapping.toJSON(self.source)
  2395. }, function (resp) {
  2396. if (resp.status !== 0) {
  2397. $(document).trigger("error", resp.message);
  2398. } else {
  2399. var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
  2400. self.source.format(newFormat);
  2401. if (self.source.inputFormat() === 'stream') {
  2402. if (self.source.streamSelection() === 'kafka') {
  2403. self.source.kafkaTopics(resp['topics']);
  2404. } else if (self.source.streamSelection() === 'flume') {
  2405. self.source.streamObjects(resp['objects']);
  2406. }
  2407. } else if (self.source.inputFormat() === 'connector') {
  2408. // Assumes selectectedConnector == 'sfdc'
  2409. self.source.streamObjects(resp['objects']);
  2410. }
  2411. if (self.source.inputFormat() !== 'stream' && self.source.inputFormat() !== 'connector') {
  2412. self.guessFieldTypes();
  2413. }
  2414. }
  2415. self.isGuessingFormat(false);
  2416. viewModel.wizardEnabled(true);
  2417. }).fail(function (xhr) {
  2418. $(document).trigger("error", xhr.responseText);
  2419. viewModel.isLoading(false);
  2420. self.isGuessingFormat(false);
  2421. });
  2422. };
  2423. self.isGuessingFieldTypes = ko.observable(false);
  2424. self.guessFieldTypes = function () {
  2425. if (guessFieldTypesXhr) {
  2426. guessFieldTypesXhr.abort();
  2427. }
  2428. self.isGuessingFieldTypes(true);
  2429. guessFieldTypesXhr = $.post("${ url('indexer:guess_field_types') }", {
  2430. "fileFormat": ko.mapping.toJSON(self.source)
  2431. }, function (resp) {
  2432. self.loadSampleData(resp);
  2433. self.isGuessingFieldTypes(false);
  2434. guessFieldTypesXhr = null;
  2435. }).fail(function (xhr) {
  2436. self.loadSampleData({sample_cols: [], columns: [], sample: []});
  2437. $(document).trigger("error", xhr.responseText);
  2438. self.isGuessingFieldTypes(false);
  2439. viewModel.isLoading(false);
  2440. guessFieldTypesXhr = null;
  2441. });
  2442. };
  2443. self.loadSampleData = function(resp) {
  2444. resp.columns.forEach(function (entry, i, arr) {
  2445. if (self.destination.outputFormat() === 'table' && self.source.inputFormat() != 'rdbms') {
  2446. entry.type = MAPPINGS.get(MAPPINGS.SOLR_TO_HIVE, entry.type, 'string');
  2447. } else if (self.destination.outputFormat() === 'index') {
  2448. entry.type = MAPPINGS.get(MAPPINGS.HIVE_TO_SOLR, entry.type, entry.type);
  2449. }
  2450. arr[i] = loadField(entry, self.destination, i);
  2451. });
  2452. self.source.sampleCols(resp.sample_cols ? resp.sample_cols : resp.columns);
  2453. self.source.sample(resp.sample);
  2454. self.destination.columns(resp.columns);
  2455. if (self.destination.tableFormat() == 'kudu' && $.grep(resp.columns, function(column) { return column.name() == 'id' }).length > 0) {
  2456. self.destination.primaryKeys(['id']); // Auto select ID column
  2457. }
  2458. };
  2459. self.isIndexing = ko.observable(false);
  2460. self.indexingError = ko.observable(false);
  2461. self.indexingSuccess = ko.observable(false);
  2462. self.destination.indexerRunJob.subscribe(function () {
  2463. self.destination.columns().forEach(function (column) {
  2464. column.operations.removeAll();
  2465. });
  2466. });
  2467. self.operationTypesFiltered = ko.pureComputed(function () {
  2468. var indexerRunJob = self.destination.indexerRunJob();
  2469. return self.operationTypes.filter(function (operation) {
  2470. return indexerRunJob || operation.name === 'split'; // We only support split for now with CSV API
  2471. }).map(function (o) {
  2472. return o.name;
  2473. });
  2474. });
  2475. self.commands = ko.observable([]);
  2476. self.showCommands = function() {
  2477. self.indexFile({show: true});
  2478. };
  2479. self.indexFile = function (options) {
  2480. if (!self.readyToIndex()) {
  2481. return;
  2482. }
  2483. $(".jHueNotify").remove();
  2484. self.indexingStarted(true);
  2485. %if not is_embeddable:
  2486. viewModel.isLoading(true);
  2487. self.isIndexing(true);
  2488. $.post("${ url('indexer:importer_submit') }", {
  2489. "source": ko.mapping.toJSON(self.source),
  2490. "destination": ko.mapping.toJSON(self.destination)
  2491. }, function (resp) {
  2492. if (resp.status !== 0) {
  2493. $(document).trigger("error", resp.message);
  2494. self.indexingStarted(false);
  2495. self.isIndexing(false);
  2496. } else if (resp.on_success_url) {
  2497. $.jHueNotify.info("${ _('Creation success.') }");
  2498. if (resp.pubSubUrl) {
  2499. huePubSub.publish(notebook.pubSubUrl);
  2500. }
  2501. huePubSub.publish('open.link', resp.on_success_url);
  2502. } else {
  2503. self.showCreate(true);
  2504. self.editorId(resp.history_id);
  2505. self.jobId(resp.handle.id);
  2506. $('#importerNotebook').html($('#importerNotebook-progress').html());
  2507. self.editorVM = new EditorViewModel(resp.history_uuid, '', {
  2508. user: '${ user.username }',
  2509. userId: ${ user.id },
  2510. languages: [{name: "Java", type: "java"}, {name: "Hive SQL", type: "hive"}], // TODO reuse
  2511. snippetViewSettings: {
  2512. java : {
  2513. snippetIcon: 'fa-file-archive-o '
  2514. },
  2515. hive: {
  2516. placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
  2517. aceMode: 'ace/mode/hive',
  2518. snippetImage: '${ static("beeswax/art/icon_beeswax_48.png") }',
  2519. sqlDialect: true
  2520. },
  2521. impala: {
  2522. placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
  2523. aceMode: 'ace/mode/impala',
  2524. snippetImage: '${ static("impala/art/icon_impala_48.png") }',
  2525. sqlDialect: true
  2526. },
  2527. sqoop1: {
  2528. placeHolder: '${ _("Example: import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir hdfs://localhost:8020/user/foo -m 1") }',
  2529. snippetImage: '${ static("sqoop/art/icon_sqoop_48.png") }'
  2530. }
  2531. }
  2532. });
  2533. self.editorVM.editorMode(true);
  2534. self.editorVM.isNotificationManager(true);
  2535. ko.cleanNode($("#importerNotebook")[0]);
  2536. ko.applyBindings(self.editorVM, $("#importerNotebook")[0]);
  2537. self.editorVM.openNotebook(resp.history_uuid, null, true, function(){
  2538. self.editorVM.selectedNotebook().snippets()[0].progress.subscribe(function(val){
  2539. if (val === 100) {
  2540. self.indexingStarted(false);
  2541. self.isIndexing(false);
  2542. self.indexingSuccess(true);
  2543. }
  2544. });
  2545. self.editorVM.selectedNotebook().snippets()[0].status.subscribe(function(val){
  2546. if (val === 'failed') {
  2547. self.isIndexing(false);
  2548. self.indexingStarted(false);
  2549. self.indexingError(true);
  2550. } else if (val === 'available') {
  2551. var snippet = self.editorVM.selectedNotebook().snippets()[0]; // Could be native to editor at some point
  2552. if (!snippet.result.handle().has_more_statements) {
  2553. if (self.editorVM.selectedNotebook().onSuccessUrl()) {
  2554. var match = snippet.statement_raw().match(/CREATE TABLE `([^`]+)`/i);
  2555. if (match) {
  2556. var db = match[1];
  2557. DataCatalog.getEntry({ sourceType: snippet.type(), namespace: self.namespace(), compute: self.compute(), path: [ db ]}).done(function (dbEntry) {
  2558. dbEntry.clearCache({ invalidate: 'invalidate', silenceErrors: true }).done(function () {
  2559. window.location.href = self.editorVM.selectedNotebook().onSuccessUrl();
  2560. })
  2561. });
  2562. } else {
  2563. DataCatalog.getEntry({ sourceType: snippet.type(), namespace: self.namespace(), compute: self.compute(), path: []}).done(function (sourceEntry) {
  2564. sourceEntry.clearCache({ silenceErrors: true }).done(function () {
  2565. window.location.href = self.editorVM.selectedNotebook().onSuccessUrl();
  2566. })
  2567. });
  2568. }
  2569. }
  2570. } else { // Perform last DROP statement execute
  2571. snippet.execute();
  2572. }
  2573. }
  2574. });
  2575. self.editorVM.selectedNotebook().snippets()[0].checkStatus();
  2576. });
  2577. }
  2578. viewModel.isLoading(false);
  2579. }).fail(function (xhr, textStatus, errorThrown) {
  2580. $(document).trigger("error", xhr.responseText);
  2581. viewModel.isLoading(false);
  2582. self.indexingStarted(false);
  2583. self.isIndexing(false);
  2584. });
  2585. % else:
  2586. $.post("${ url('indexer:importer_submit') }", {
  2587. "source": ko.mapping.toJSON(self.source),
  2588. "destination": ko.mapping.toJSON(self.destination),
  2589. "start_time": ko.mapping.toJSON((new Date()).getTime()),
  2590. "options": ko.mapping.toJSON(options || {})
  2591. }, function (resp) {
  2592. self.indexingStarted(false);
  2593. if (resp.status === 0) {
  2594. if (resp.history_uuid) {
  2595. $.jHueNotify.info("${ _('Task submitted') }");
  2596. huePubSub.publish('notebook.task.submitted', resp.history_uuid);
  2597. } else if (resp.on_success_url) {
  2598. if (resp.pub_sub_url) {
  2599. huePubSub.publish(resp.pub_sub_url);
  2600. }
  2601. $.jHueNotify.info("${ _('Creation success') }");
  2602. if (resp.errors && resp.errors.length) {
  2603. $.jHueNotify.warn("${ _('Skipped records: ') }" + resp.errors.join(', '));
  2604. }
  2605. huePubSub.publish('open.link', resp.on_success_url);
  2606. } else if (resp.commands) {
  2607. self.commands(resp.commands);
  2608. $('#showCommandsModal').modal('show');
  2609. }
  2610. } else {
  2611. $(document).trigger("error", resp && resp.message ? resp.message : '${ _("Error importing") }');
  2612. }
  2613. }).fail(function (xhr) {
  2614. self.indexingStarted(false);
  2615. $(document).trigger("error", xhr.responseText);
  2616. });
  2617. % endif
  2618. hueAnalytics.log('importer', 'submit/' + self.source.inputFormat() + '/' + self.destination.outputFormat());
  2619. };
  2620. self.removeOperation = function (operation, operationList) {
  2621. operationList.remove(operation);
  2622. hueAnalytics.log('importer', 'step/removeOperation');
  2623. };
  2624. self.addOperation = function (field) {
  2625. field.operations.push(new Operation("split"));
  2626. hueAnalytics.log('importer', 'step/addOperation');
  2627. };
  2628. self.load = function (state) {
  2629. self.source.name(state.name);
  2630. self.source.show(state.show);
  2631. self.source.path(state.path);
  2632. self.destination.columns.removeAll();
  2633. if (state.format && 'type' in state.format) {
  2634. var koFormat = ko.mapping.fromJS(new FileType(state.format.type, state.format));
  2635. self.source.format(koFormat);
  2636. }
  2637. if (state.columns) {
  2638. state.columns.forEach(function (currCol) {
  2639. self.destination.columns.push(loadField(currCol));
  2640. });
  2641. }
  2642. }
  2643. };
  2644. var loadDefaultField = function (options) {
  2645. if (!options.name) {
  2646. options.name = getNewFieldName();
  2647. }
  2648. return loadField($.extend({}, ${default_field_type | n}, options));
  2649. };
  2650. var loadField = function (currField, parent, idx) {
  2651. var koField = ko.mapping.fromJS(currField);
  2652. if (koField.name && parent) {
  2653. koField.name.subscribe(function (newVal) {
  2654. if (newVal.indexOf(',') > -1) {
  2655. var fields = newVal.split(',');
  2656. fields.forEach(function (val, i) {
  2657. if (i + idx < parent.columns().length) {
  2658. parent.columns()[i + idx].name(val);
  2659. }
  2660. });
  2661. }
  2662. parent.bulkColumnNames(parent.columns().map(function (item) {
  2663. return item.name()
  2664. }).join(','));
  2665. });
  2666. }
  2667. koField.operations.removeAll();
  2668. currField.operations.forEach(function (operationData) {
  2669. var operation = new Operation(operationData.type);
  2670. operation.load(operationData);
  2671. koField.operations.push(operation);
  2672. });
  2673. var autoExpand = function (newVal) {
  2674. if ((newVal === 'array' || newVal === 'map' || newVal === 'struct') && koField.nested().length === 0) {
  2675. koField.nested.push(loadDefaultField({level: koField.level() + 1}));
  2676. }
  2677. };
  2678. koField.type.subscribe(autoExpand);
  2679. koField.keyType.subscribe(autoExpand);
  2680. return koField;
  2681. };
  2682. var IndexerViewModel = function () {
  2683. var self = this;
  2684. self.apiHelper = ApiHelper.getInstance();
  2685. self.sourceType = window.location.getParameter('sourceType', true) || '${ source_type }';
  2686. self.namespaceId = window.location.getParameter('namespace', true);
  2687. self.computeId = window.location.getParameter('compute', true);
  2688. self.assistAvailable = ko.observable(true);
  2689. self.isLeftPanelVisible = ko.observable();
  2690. self.apiHelper.withTotalStorage('assist', 'assist_panel_visible', self.isLeftPanelVisible, true);
  2691. self.loadDefaultField = loadDefaultField;
  2692. self.createWizard = new CreateWizard(self);
  2693. // Wizard related
  2694. self.wizardEnabled = ko.observable(false);
  2695. self.currentStep = ko.observable(self.createWizard.prefill.target_type() === 'database' ? 2 : 1);
  2696. self.currentStep.subscribe(function () {
  2697. %if is_embeddable:
  2698. $('.page-content').scrollTop(0);
  2699. %else:
  2700. $('.content-panel').scrollTop(0);
  2701. %endif
  2702. });
  2703. self.previousStepVisible = ko.pureComputed(function(){
  2704. return self.currentStep() > 1 && (self.createWizard.destination.outputFormat() !== 'database' || self.createWizard.source.inputFormat() === 'rdbms');
  2705. });
  2706. self.nextStepVisible = ko.pureComputed(function(){
  2707. return self.currentStep() < 3 && self.wizardEnabled();
  2708. });
  2709. self.nextStep = function () {
  2710. if (self.nextStepVisible()) {
  2711. self.currentStep(self.currentStep() + 1);
  2712. hueAnalytics.log('importer', 'step/' + self.currentStep());
  2713. }
  2714. };
  2715. self.previousStep = function () {
  2716. if (self.previousStepVisible()) {
  2717. self.currentStep(self.currentStep() - 1);
  2718. hueAnalytics.log('importer', 'step/' + self.currentStep());
  2719. }
  2720. };
  2721. self.isLoading = ko.observable(false);
  2722. };
  2723. var viewModel;
  2724. function resizeElements () {
  2725. var $contentPanel = $('#importerComponents').find('.content-panel-inner');
  2726. $('.step-indicator-fixed').width($contentPanel.width());
  2727. document.styleSheets[0].addRule('.form-actions','margin-left: -11px !important');
  2728. document.styleSheets[0].addRule('.step-indicator li:first-child:before','max-width: ' + ($contentPanel.find('.step-indicator li:first-child .caption').width()) + 'px');
  2729. document.styleSheets[0].addRule('.step-indicator li:first-child:before','left: ' + ($contentPanel.find('.step-indicator li:first-child .caption').width()/2) + 'px');
  2730. document.styleSheets[0].addRule('.step-indicator li:last-child:before','max-width: ' + ($contentPanel.find('.step-indicator li:last-child .caption').width()) + 'px');
  2731. document.styleSheets[0].addRule('.step-indicator li:last-child:before','right: ' + ($contentPanel.find('.step-indicator li:last-child .caption').width()/2) + 'px');
  2732. }
  2733. $(document).ready(function () {
  2734. viewModel = new IndexerViewModel();
  2735. ko.applyBindings(viewModel, $('#importerComponents')[0]);
  2736. var draggableMeta = {};
  2737. huePubSub.subscribe('draggable.text.meta', function (meta) {
  2738. draggableMeta = meta;
  2739. });
  2740. huePubSub.subscribe('importer.show.bulkeditor', function (meta) {
  2741. $('#fieldsBulkEditor').modal('show');
  2742. }, 'importer');
  2743. huePubSub.subscribe('split.panel.resized', resizeElements);
  2744. hueUtils.waitForRendered('.step-indicator li:first-child .caption', function(el){ return el.width() < $('#importerComponents').find('.content-panel-inner').width()/2 }, resizeElements);
  2745. $(window).on('resize', resizeElements);
  2746. $('.importer-droppable').droppable({
  2747. accept: ".draggableText",
  2748. drop: function (e, ui) {
  2749. var text = ui.helper.text();
  2750. var generatedName = 'idx';
  2751. switch (draggableMeta.type){
  2752. case 'sql':
  2753. if (draggableMeta.table !== '') {
  2754. generatedName += draggableMeta.table;
  2755. viewModel.createWizard.source.inputFormat('table');
  2756. viewModel.createWizard.source.table(draggableMeta.table);
  2757. }
  2758. break;
  2759. case 'hdfs':
  2760. generatedName += draggableMeta.definition.name;
  2761. viewModel.createWizard.source.inputFormat('file');
  2762. viewModel.createWizard.source.path(draggableMeta.definition.path);
  2763. break;
  2764. case 'document':
  2765. if (draggableMeta.definition.type === 'query-hive') {
  2766. generatedName += draggableMeta.definition.name;
  2767. viewModel.createWizard.source.inputFormat('query');
  2768. viewModel.createWizard.source.draggedQuery(draggableMeta.definition.uuid);
  2769. }
  2770. break;
  2771. }
  2772. if (generatedName !== 'idx' && viewModel.createWizard.source.name() === '') {
  2773. viewModel.createWizard.source.name(generatedName);
  2774. }
  2775. }
  2776. });
  2777. });
  2778. })();
  2779. </script>
  2780. </span>
  2781. %if not is_embeddable:
  2782. ${ commonfooter(request, messages) | n,unicode }
  2783. %endif