importer.mako 143 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117
  1. ## Licensed to Cloudera, Inc. under one
  2. ## or more contributor license agreements. See the NOTICE file
  3. ## distributed with this work for additional information
  4. ## regarding copyright ownership. Cloudera, Inc. licenses this file
  5. ## to you under the Apache License, Version 2.0 (the
  6. ## "License"); you may not use this file except in compliance
  7. ## with the License. You may obtain a copy of the License at
  8. ##
  9. ## http://www.apache.org/licenses/LICENSE-2.0
  10. ##
  11. ## Unless required by applicable law or agreed to in writing, software
  12. ## distributed under the License is distributed on an "AS IS" BASIS,
  13. ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. ## See the License for the specific language governing permissions and
  15. ## limitations under the License.
  16. <%!
  17. from django.utils.translation import ugettext as _
  18. from desktop import conf
  19. from desktop.views import commonheader, commonfooter, commonshare, commonimportexport, _ko
  20. from filebrowser.conf import SHOW_UPLOAD_BUTTON
  21. from beeswax import hive_site
  22. from impala import impala_flags
  23. from notebook.conf import ENABLE_SQL_INDEXER
  24. from indexer.conf import ENABLE_NEW_INDEXER, ENABLE_SQOOP, ENABLE_KAFKA, CONFIG_INDEXER_LIBS_PATH, ENABLE_SCALABLE_INDEXER, ENABLE_ALTUS, ENABLE_ENVELOPE, ENABLE_FIELD_EDITOR
  25. %>
  26. <%namespace name="actionbar" file="actionbar.mako" />
  27. %if not is_embeddable:
  28. ${ commonheader(_("Importer"), "indexer", user, request, "60px") | n,unicode }
  29. <link rel="stylesheet" href="${ static('notebook/css/notebook.css') }">
  30. <link rel="stylesheet" href="${ static('notebook/css/notebook-layout.css') }">
  31. %endif
  32. <link rel="stylesheet" href="${ static('indexer/css/importer.css') }" type="text/css">
  33. <style type="text/css">
  34. % if conf.CUSTOM.BANNER_TOP_HTML.get():
  35. .show-assist {
  36. top: 110px!important;
  37. }
  38. .main-content {
  39. top: 112px!important;
  40. }
  41. % endif
  42. </style>
  43. <span id="importerComponents" class="notebook importer-main" data-bind="dropzone: { url: '/filebrowser/upload/file?dest=' + DROPZONE_HOME_DIR, params: {dest: DROPZONE_HOME_DIR}, paramName: 'hdfs_file', onComplete: function(path){ createWizard.source.path(path); }, disabled: '${ not hasattr(SHOW_UPLOAD_BUTTON, 'get') or not SHOW_UPLOAD_BUTTON.get() }' === 'True' }">
  44. <div class="dz-message" data-dz-message></div>
  45. <div class="navbar hue-title-bar">
  46. <div class="navbar-inner">
  47. <div class="container-fluid">
  48. <div class="nav-collapse">
  49. <ul class="nav">
  50. <li class="app-header">
  51. <a href="${ url('indexer:importer') }">
  52. <!-- ko if: createWizard.prefill.target_type().length == 0 -->
  53. <i class="fa fa-database app-icon"></i> ${_('Importer')}
  54. <!-- /ko -->
  55. <!-- ko ifnot: createWizard.prefill.target_type().length == 0 -->
  56. <!-- ko if: createWizard.prefill.target_type() === 'index' -->
  57. <i class="fa fa-search app-icon"></i> ${_('Import to index')}
  58. <!-- /ko -->
  59. <!-- ko if: createWizard.prefill.target_type() === 'table' -->
  60. <i class="fa fa-database app-icon"></i> ${_('Import to table')}
  61. <!-- /ko -->
  62. <!-- ko if: createWizard.prefill.target_type() === 'database' -->
  63. <i class="fa fa-database app-icon"></i> ${_('Create a new database')}
  64. <!-- /ko -->
  65. <!-- /ko -->
  66. </a>
  67. </li>
  68. </ul>
  69. </div>
  70. </div>
  71. </div>
  72. </div>
  73. %if not is_embeddable:
  74. <a title="${_('Toggle Assist')}" class="pointer show-assist" data-bind="visible: !$root.isLeftPanelVisible() && $root.assistAvailable(), click: function() { $root.isLeftPanelVisible(true); }">
  75. <i class="fa fa-chevron-right"></i>
  76. </a>
  77. %endif
  78. <div class="main-content">
  79. <div class="vertical-full container-fluid" data-bind="style: { 'padding-left' : $root.isLeftPanelVisible() ? '0' : '20px' }">
  80. <div class="vertical-full">
  81. <div class="vertical-full row-fluid panel-container">
  82. %if not is_embeddable:
  83. <div class="assist-container left-panel" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable()">
  84. <a title="${_('Toggle Assist')}" class="pointer hide-assist" data-bind="click: function() { $root.isLeftPanelVisible(false) }">
  85. <i class="fa fa-chevron-left"></i>
  86. </a>
  87. <div class="assist" data-bind="component: {
  88. name: 'assist-panel',
  89. params: {
  90. user: '${user.username}',
  91. onlySql: false,
  92. sql: {
  93. navigationSettings: {
  94. openItem: false,
  95. showStats: true
  96. }
  97. },
  98. visibleAssistPanels: ['sql', 'hdfs', 'documents']
  99. }
  100. }"></div>
  101. </div>
  102. <div class="resizer" data-bind="visible: $root.isLeftPanelVisible() && $root.assistAvailable(), splitDraggable : { appName: 'notebook', leftPanelVisible: $root.isLeftPanelVisible }"><div class="resize-bar">&nbsp;</div></div>
  103. %endif
  104. <div class="content-panel importer-droppable">
  105. <div class="content-panel-inner">
  106. <!-- ko template: 'create-index-wizard' --><!-- /ko -->
  107. </div>
  108. </div>
  109. </div>
  110. </div>
  111. </div>
  112. </div>
  113. <script type="text/html" id="create-index-wizard">
  114. <div data-bind="visible: createWizard.show">
  115. <div class="step-indicator-fixed">
  116. <ol class="list-inline text-center step-indicator">
  117. <li data-bind="css: { 'active': currentStep() == 1, 'complete': currentStep() > 1, 'pointer': currentStep() > 1 }, click: function() { currentStep(1) }">
  118. <div class="step" title="${ _('Go to Step 1') }">
  119. <!-- ko if: currentStep() == 1 -->
  120. <!-- ko if: createWizard.isGuessingFormat -->
  121. <span class="fa fa-spinner fa-spin"></span>
  122. <!-- /ko -->
  123. <!-- ko ifnot: createWizard.isGuessingFormat -->
  124. 1
  125. <!-- /ko -->
  126. <!-- /ko -->
  127. <!-- ko ifnot: currentStep() == 1 -->
  128. <span class="fa fa-check"></span>
  129. <!-- /ko -->
  130. </div>
  131. <div class="caption">
  132. <!-- ko if: createWizard.source.inputFormat() != 'manual' -->
  133. ${ _('Pick data from ') }<span data-bind="text: createWizard.source.inputFormat"></span>
  134. <!-- ko if: createWizard.source.inputFormat() == 'file' -->
  135. <span data-bind="text: createWizard.source.path"></span>
  136. <!-- /ko -->
  137. <!-- ko if: createWizard.source.inputFormat() == 'stream' -->
  138. <!-- ko if: createWizard.source.streamSelection() == 'kafka' -->
  139. <span data-bind="text: createWizard.source.kafkaSelectedTopics"></span>
  140. <!-- /ko -->
  141. <!-- ko if: createWizard.source.streamSelection() != 'kafka' -->
  142. <span data-bind="text: createWizard.source.streamSelection"></span>
  143. <!-- /ko -->
  144. <!-- /ko -->
  145. <!-- /ko -->
  146. <!-- ko if: createWizard.source.inputFormat() == 'manual' -->
  147. ${ _('No source data') }
  148. <!-- /ko -->
  149. </div>
  150. </li>
  151. <li class="arrow muted"><i class="fa fa-fw fa-angle-double-right"></i></li>
  152. <li data-bind="css: { 'inactive': currentStep() == 1, 'active': currentStep() == 2, 'complete': currentStep() == 3, 'pointer': currentStep() == 1 && !createWizard.isGuessingFormat() && createWizard.source.show() }, click: function() { if (!createWizard.isGuessingFormat() && createWizard.source.show()){ currentStep(2); }}">
  153. <div class="step" title="${ _('Go to Step 2') }">
  154. <!-- ko if: currentStep() < 3 -->
  155. <!-- ko if: createWizard.isGuessingFieldTypes -->
  156. <span class="fa fa-spinner fa-spin"></span>
  157. <!-- /ko -->
  158. <!-- ko ifnot: createWizard.isGuessingFieldTypes -->
  159. 2
  160. <!-- /ko -->
  161. <!-- /ko -->
  162. </div>
  163. <div class="caption">
  164. <!-- ko if: createWizard.source.inputFormat() != 'manual' -->
  165. ${ _('Move it to ') }
  166. <!-- /ko -->
  167. <!-- ko if: createWizard.source.inputFormat() == 'manual' -->
  168. ${ _('Create') }
  169. <!-- /ko -->
  170. <span data-bind="text: createWizard.destination.outputFormat"></span>
  171. <span data-bind="text: createWizard.destination.name"></span>
  172. <!-- ko if: createWizard.destination.outputFormat() == 'altus' -->
  173. ${ _('in') } <span data-bind="text: createWizard.destination.namespace"></span>
  174. <!-- /ko -->
  175. </div>
  176. </li>
  177. </ol>
  178. </div>
  179. <div class="vertical-spacer"></div>
  180. <!-- ko if: currentStep() == 1 -->
  181. <div class="card step">
  182. <h4>${_('Source')}</h4>
  183. <div class="card-body">
  184. <div>
  185. <div class="control-group" data-bind="visible: createWizard.prefill.target_type().length == 0 || createWizard.prefill.source_type() == 'all'">
  186. <label for="sourceType" class="control-label"><div>${ _('Type') }</div>
  187. <select id="sourceType" data-bind="selectize: createWizard.source.inputFormats, value: createWizard.source.inputFormat, optionsText: 'name', optionsValue: 'value'"></select>
  188. </label>
  189. </div>
  190. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'rdbms'">
  191. <label for="rdbmsMode" class="control-label"><div>${ _('Mode') }</div>
  192. <label class="radio inline-block margin-right-20">
  193. <input type="radio" name="rdbmsMode" value="customRdbms" data-bind="checked: createWizard.source.rdbmsMode" /> ${_('Custom')}
  194. </label>
  195. <label class="radio inline-block">
  196. <input type="radio" name="rdbmsMode" value="configRdbms" data-bind="checked: createWizard.source.rdbmsMode" /> ${_('Configured')}
  197. </label>
  198. </label>
  199. </div>
  200. <div class="control-group" data-bind="visible: createWizard.prefill.target_type() == 'database'">
  201. <label for="sourceType" class="control-label">${ _('No source is needed for creating a database.') }</label>
  202. </div>
  203. <div class="control-group input-append" data-bind="visible: createWizard.source.inputFormat() == 'file'">
  204. <label for="path" class="control-label"><div>${ _('Path') }</div>
  205. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: createWizard.source.path, filechooser: createWizard.source.path, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true, openOnFocus: true, selectFolder: false }" placeholder="${ _('Click or drag from the assist') }">
  206. </label>
  207. <!-- ko if: createWizard.source.path().length > 0 -->
  208. <a data-bind="storageContextPopover: { path: createWizard.source.path(), offset: { right: 5 } }" title="${ _('Preview') }" style="font-size: 14px" class="margin-left-10">
  209. <i class="fa fa-fw fa-info"></i>
  210. </a>
  211. <!-- /ko -->
  212. </div>
  213. <!-- ko if: createWizard.source.inputFormat() == 'rdbms' -->
  214. <!-- ko if: createWizard.source.rdbmsMode() -->
  215. <div class="control-group">
  216. <label for="rdbmsType" class="control-label"><div>${ _('Driver') }</div>
  217. <select id="rdbmsType" data-bind="selectize: createWizard.source.rdbmsTypes, value: createWizard.source.rdbmsType, optionsText: 'name', optionsValue: 'value'"></select>
  218. </label>
  219. </div>
  220. <!-- /ko -->
  221. <!-- ko if: createWizard.source.rdbmsMode() == 'customRdbms' -->
  222. <!-- ko if: createWizard.source.rdbmsType() != 'jdbc' -->
  223. <div class="control-group">
  224. <label for="rdbmsHostname" class="control-label"><div>${ _('Hostname') }</div>
  225. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsHostname" placeholder="${ _('Enter host/ip here e.g. mysql.domain.com or 123.123.123.123') }">
  226. </label>
  227. </div>
  228. <!-- /ko -->
  229. <!-- ko if: createWizard.source.rdbmsType() == 'jdbc' -->
  230. <div class="control-group">
  231. <label for="rdbmsHostname" class="control-label"><div>${ _('Url') }</div>
  232. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsHostname" placeholder="${ _('jdbc:mysql://localhost:3306') }">
  233. </label>
  234. </div>
  235. <!-- /ko -->
  236. <!-- ko if: createWizard.source.rdbmsType() == 'jdbc' -->
  237. <div class="control-group">
  238. <label for="rdbmsJdbcDriver" class="control-label"><div>${ _('JDBC Driver') }</div>
  239. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsJdbcDriver">
  240. </label>
  241. </div>
  242. <!-- /ko -->
  243. <!-- ko if: createWizard.source.rdbmsType() != 'jdbc' -->
  244. <div class="control-group">
  245. <label for="rdbmsPort" class="control-label"><div>${ _('Port') }</div>
  246. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsPort" placeholder="${ _('Enter port number here e.g. 3306') }">
  247. </label>
  248. </div>
  249. <!-- /ko -->
  250. <div class="control-group">
  251. <label for="rdbmsUsername" class="control-label"><div>${ _('Username') }</div>
  252. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsUsername" placeholder="${ _('Enter username here') }">
  253. </label>
  254. </div>
  255. <div class="control-group no-margin-bottom">
  256. <label for="rdbmsPassword" class="control-label"><div>${ _('Password') }</div>
  257. <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.rdbmsPassword" placeholder="${ _('Enter password here') }">
  258. </label>
  259. </div>
  260. <div class="control-group">
  261. <button class="btn" data-bind="click: createWizard.source.rdbmsCheckConnection">
  262. ${_('Test Connection')}
  263. </button>
  264. </div>
  265. <!-- /ko -->
  266. <!-- ko if: createWizard.source.rdbmsMode() == 'configRdbms' || (createWizard.source.rdbmsMode() == 'customRdbms' && createWizard.source.rdbmsDbIsValid()) -->
  267. <!-- ko if: createWizard.source.rdbmsMode() == 'configRdbms' && createWizard.source.rdbmsType() == 'jdbc' -->
  268. <div class="control-group">
  269. <label for="rdbmsJdbcDriverName" class="control-label"><div>${ _('Options') }</div>
  270. <select id="rdbmsJdbcDriverName" data-bind="selectize: createWizard.source.rdbmsJdbcDriverNames, value: createWizard.source.rdbmsJdbcDriverName, optionsText: 'name', optionsValue: 'value'"></select>
  271. <!-- ko hueSpinner: { spin: createWizard.source.isFetchingDriverNames, inline: true } --><!-- /ko -->
  272. </label>
  273. </div>
  274. <!-- /ko -->
  275. <!-- ko if: createWizard.source.rdbmsTypes -->
  276. <div class="control-group input-append">
  277. <label for="rdbmsDatabaseName" class="control-label"><div>${ _('Database Name') }</div>
  278. <select id="rdbmsDatabaseName" data-bind="selectize: createWizard.source.rdbmsDatabaseNames, value: createWizard.source.rdbmsDatabaseName, optionsText: 'name', optionsValue: 'value'"></select>
  279. <!-- ko hueSpinner: { spin: createWizard.source.isFetchingDatabaseNames, inline: true } --><!-- /ko -->
  280. </label>
  281. </div>
  282. <!-- /ko -->
  283. <!-- ko if: createWizard.source.rdbmsDatabaseName -->
  284. <div class="control-group">
  285. <div>
  286. <label class="checkbox inline-block">
  287. <input type="checkbox" data-bind="checked: createWizard.source.rdbmsIsAllTables"> ${_('All Tables')}
  288. </label>
  289. </div>
  290. <!-- ko ifnot: createWizard.source.rdbmsIsAllTables() -->
  291. <label for="rdbmsTableName" class="control-label"><div>${ _('Table Name') }</div>
  292. <select id="rdbmsTableName" multiple="multiple" data-bind="selectize: createWizard.source.rdbmsTableNames, selectedObjects: createWizard.source.tables, selectedOptions: createWizard.source.tablesNames, optionsText: 'name', optionsValue: 'value'"></select>
  293. <!-- ko hueSpinner: { spin: createWizard.source.isFetchingTableNames, inline: true } --><!-- /ko -->
  294. </label>
  295. <!-- /ko -->
  296. </div>
  297. <!-- /ko -->
  298. <!-- /ko -->
  299. <!-- /ko -->
  300. <!-- ko if: createWizard.source.inputFormat() == 'stream' -->
  301. <div class="control-group">
  302. <label class="control-label"><div>${ _('List') }</div>
  303. <select data-bind="selectize: createWizard.source.publicStreams, value: createWizard.source.streamSelection, optionsText: 'name', optionsValue: 'value'" placeholder="${ _('The list of streams to consume, e.g. SFDC, Jiras...') }"></select>
  304. </label>
  305. </div>
  306. <!-- ko if: createWizard.source.streamSelection() == 'kafka' -->
  307. <div data-bind="template: { name: 'kafka-topic-template', data: $data }" class="margin-top-10 field inline-block"></div>
  308. <!-- /ko -->
  309. <!-- ko if: createWizard.source.streamSelection() == 'flume' -->
  310. <div class="control-group">
  311. <label class="control-label"><div>${ _('Type') }</div>
  312. <select class="input-medium" data-bind="selectize: createWizard.source.channelSourceTypes, value: createWizard.source.channelSourceType, optionsText: 'name', optionsValue: 'value'"></select>
  313. </label>
  314. <!-- ko if: ['directory', 'exec', 'syslogs'].indexOf(createWizard.source.channelSourceType()) != -1 -->
  315. <label class="control-label"><div>${ _('Hosts') }</div>
  316. <select class="input-xxlarge" data-bind="selectize: createWizard.source.channelSourceHosts, selectedOptions: createWizard.source.channelSourceSelectedHosts" multiple="true"></select>
  317. </label>
  318. <!-- /ko -->
  319. <!-- ko if: createWizard.source.channelSourceType() == 'directory' -->
  320. <label class="control-label"><div>${ _('Path') }</div>
  321. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.channelSourcePath" placeholder="${ _('The path to watch and consume') }">
  322. </label>
  323. <!-- /ko -->
  324. <!-- ko if: createWizard.source.channelSourceType() -->
  325. <input data-bind="click: function() { createWizard.source.channelSourceType(null); }" class="btn" value="${ _('Clear') }"/>
  326. <!-- /ko -->
  327. </div>
  328. <!-- /ko -->
  329. <!-- /ko -->
  330. <!-- ko if: createWizard.source.inputFormat() == 'connector' -->
  331. <div class="control-group">
  332. <label class="control-label"><div>${ _('List') }</div>
  333. <select data-bind="selectize: createWizard.source.connectorList, value: createWizard.source.connectorSelection, optionsText: 'name', optionsValue: 'value'" placeholder="${ _('The connector to use, e.g. SFDC, Jiras...') }"></select>
  334. </label>
  335. </div>
  336. <!-- ko if: createWizard.source.connectorSelection() == 'sfdc' -->
  337. <div class="control-group">
  338. <label class="control-label"><div>${ _('Username') }</div>
  339. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamUsername" placeholder="user@company.com">
  340. </label>
  341. <label class="control-label"><div>${ _('Password') }</div>
  342. <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.streamPassword">
  343. </label>
  344. <label class="control-label"><div>${ _('Token') }</div>
  345. <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.streamToken">
  346. </label>
  347. <label class="control-label"><div>${ _('End point URL') }</div>
  348. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamEndpointUrl">
  349. </label>
  350. <br/>
  351. <!-- ko if: createWizard.source.streamUsername() && createWizard.source.streamPassword() && createWizard.source.streamToken() -->
  352. <label class="control-label"><div>${ _('Object') }</div>
  353. <select class="input-xxlarge" data-bind="options: createWizard.source.streamObjects,
  354. value: createWizard.source.streamObject,
  355. optionsCaption: '${ _("Choose...") }'"
  356. placeholder="${ _('The SFDC object to import, e.g. Account, Opportunity') }"></select>
  357. </label>
  358. <!-- /ko -->
  359. </div>
  360. <!-- /ko -->
  361. <!-- /ko -->
  362. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'table'">
  363. <!-- ko foreach: $root.createWizard.source.tables -->
  364. <div>
  365. <label class="control-label">
  366. <div><!-- ko if: $index() === 0 -->${ _('Table') }<!-- /ko --></div>
  367. <input type="text" class="input-xlarge" data-bind="hiveChooser: name, skipInvalids:true, pathChangeLevel: 'table', skipColumns: true, apiHelperUser: '${ user }', namespace: $root.createWizard.source.namespace, compute: $root.createWizard.source.compute, apiHelperType: $root.createWizard.source.sourceType, mainScrollable: $(MAIN_SCROLLABLE)" placeholder="${ _('Table name or <database>.<table>') }">
  368. <a class="pointer pull-right margin-left-5" style="margin-top: 7px" data-bind="click: function() { $root.createWizard.source.tables.remove(this); }, visible: $root.createWizard.source.tables().length > 1"><i class="fa fa-minus"></i></a>
  369. </label>
  370. </div>
  371. <!-- /ko -->
  372. <div>
  373. <label class="control-label">
  374. <div>&nbsp;</div>
  375. <a class="pointer" data-bind="click: function() { createWizard.source.tables.push(ko.mapping.fromJS({name: ''})); }" title="${_('Add Table')}"><i class="fa fa-plus fa-padding-top"></i> ${_('Add table')}</a>
  376. </label>
  377. </div>
  378. </div>
  379. <div class="control-group" data-bind="visible: createWizard.source.inputFormat() == 'query'">
  380. <label for="path" class="control-label"><div>${ _('Query') }
  381. <!-- ko if: createWizard.source.query() && createWizard.source.query().name -->
  382. <span data-bind="text: createWizard.source.query().name"></span>
  383. <!-- /ko -->
  384. </div>
  385. ## No dropdown for now
  386. ## <select placeholder="${ _('Search your documents...') }" data-bind="documentChooser: { dependentValue: createWizard.source.draggedQuery, mappedDocument: createWizard.source.query }"></select>
  387. </label>
  388. </div>
  389. </div>
  390. </div>
  391. </div>
  392. <!-- ko if: createWizard.source.show() && createWizard.source.inputFormat() != 'manual' -->
  393. <div class="card step" data-bind="visible: createWizard.source.inputFormat() == 'file' || createWizard.source.inputFormat() == 'stream'">
  394. <!-- ko if: createWizard.isGuessingFormat -->
  395. <h4>${_('Guessing format...')} <i class="fa fa-spinner fa-spin"></i></h4>
  396. <!-- /ko -->
  397. <!-- ko ifnot: createWizard.isGuessingFormat -->
  398. <h4>${_('Format')}</h4>
  399. <div class="card-body">
  400. <label data-bind="visible: (createWizard.prefill.source_type().length == 0 || createWizard.prefill.target_type() == 'index') && (createWizard.source.inputFormat() == 'file' || createWizard.source.inputFormat() == 'stream')">
  401. <div>${_('File Type')}</div>
  402. <select data-bind="selectize: $root.createWizard.fileTypes, value: $root.createWizard.fileTypeName, optionsText: 'description', optionsValue: 'name'"></select>
  403. </label>
  404. <span class="inline-labels" data-bind="with: createWizard.source.format, visible: createWizard.source.show">
  405. <span data-bind="foreach: getArguments()">
  406. <!-- ko template: {name: 'arg-' + $data.type, data: {description: $data.description, value: $parent[$data.name]}}--><!-- /ko -->
  407. </span>
  408. </span>
  409. </div>
  410. <!-- /ko -->
  411. </div>
  412. <!-- ko ifnot: createWizard.source.inputFormat() == 'rdbms' && createWizard.source.rdbmsIsAllTables() -->
  413. <div class="card step" style="min-height: 310px;">
  414. <!-- ko if: $root.createWizard.source.inputFormat() === 'table' || ($root.createWizard.source.inputFormat() !== 'table' && !createWizard.isGuessingFormat()) -->
  415. <h4>${_('Preview')}
  416. <!-- ko if: $root.createWizard.source.inputFormat() !== 'table' && createWizard.isGuessingFieldTypes() -->
  417. <i class="fa fa-spinner fa-spin"></i>
  418. <!-- /ko -->
  419. </h4>
  420. <!-- /ko -->
  421. <!-- ko if: createWizard.source.inputFormat() == 'query' && createWizard.source.sample().length == 0 -->
  422. ${ _('Add sample data') } <i class="fa fa-fw fa-play"></i>
  423. <!-- /ko -->
  424. <div class="card-body">
  425. <!-- ko if: ['table', 'rdbms'].indexOf($root.createWizard.source.inputFormat()) >= 0 && $root.createWizard.source.tables().length > 1 -->
  426. <ul class="nav nav-tabs" style="margin-top: -14px">
  427. <!-- ko foreach: $root.createWizard.source.tables -->
  428. <li data-bind="css: { 'active': $root.createWizard.source.selectedTableIndex() === $index() }"><a href="javascript: void(0)" data-bind="text: name, click: function(){ $root.createWizard.source.selectedTableIndex($index()) }"></a></li>
  429. <!-- /ko -->
  430. </ul>
  431. <!-- /ko -->
  432. <!-- ko if: ['table', 'rdbms'].indexOf($root.createWizard.source.inputFormat()) >= 0 -->
  433. <!-- ko hueSpinner: { spin: createWizard.isGuessingFormat() || createWizard.isGuessingFieldTypes() , inline: true } --><!-- /ko -->
  434. <!-- /ko -->
  435. <!-- ko ifnot: createWizard.isGuessingFormat -->
  436. <!-- ko ifnot: createWizard.isGuessingFieldTypes -->
  437. <div data-bind="delayedOverflow">
  438. <table class="table table-condensed table-preview">
  439. <thead>
  440. <tr data-bind="foreach: createWizard.source.sampleCols">
  441. ##<!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  442. <th data-bind="truncatedText: name" style="padding-right: 60px"></th>
  443. ## TODO nested
  444. </tr>
  445. </thead>
  446. <tbody data-bind="foreach: createWizard.source.sample">
  447. <tr data-bind="foreach: $data">
  448. <td data-bind="truncatedText: $data"></td>
  449. </tr>
  450. </tbody>
  451. </table>
  452. </div>
  453. <!-- /ko -->
  454. <!-- /ko -->
  455. </div>
  456. </div>
  457. <!-- /ko -->
  458. <!-- /ko -->
  459. <!-- /ko -->
  460. <!-- ko if: currentStep() == 2 -->
  461. <!-- ko with: createWizard.destination -->
  462. <div class="card step">
  463. <h4>${_('Destination')}</h4>
  464. <div class="card-body">
  465. <div class="control-group">
  466. <label for="destinationType" class="control-label" data-bind="visible: $parent.createWizard.prefill.target_type().length == 0"><div>${ _('Type') }</div>
  467. <select id="destinationType" data-bind="selectize: outputFormats, value: outputFormat, optionsValue: 'value', optionsText: 'name'"></select>
  468. </label>
  469. </div>
  470. <div class="control-group">
  471. <!-- ko if: outputFormat() == 'file' -->
  472. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  473. <input type="text" class="form-control name input-xxlarge" id="collectionName" data-bind="value: name, filechooser: name, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true, openOnFocus: true, selectFolder: true, displayOnlyFolders: true, uploadFile: false}" placeholder="${ _('Name') }" title="${ _('Directory must not exist in the path') }">
  474. <!-- /ko -->
  475. <!-- ko if: outputFormat() == 'index' -->
  476. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  477. <input type="text" class="form-control input-xlarge" id="collectionName" data-bind="value: name, valueUpdate: 'afterkeydown'" placeholder="${ _('Name') }">
  478. <!-- /ko -->
  479. <!-- ko if: ['table', 'database'].indexOf(outputFormat()) != -1 -->
  480. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  481. <input type="text" class="input-xxlarge" data-bind="value: name, hiveChooser: name, namespace: namespace, compute: compute, skipColumns: true, skipTables: outputFormat() == 'database', valueUpdate: 'afterkeydown', apiHelperUser: '${ user }', apiHelperType: sourceType, mainScrollable: $(MAIN_SCROLLABLE), attr: { 'placeholder': outputFormat() == 'table' ? '${ _ko('Table name or <database>.<table>') }' : '${ _ko('Database name') }' }" pattern="^([a-zA-Z0-9_]+\.)?[a-zA-Z0-9_]*$" title="${ _('Only alphanumeric and underscore characters') }">
  482. <!-- /ko -->
  483. <!-- ko if: outputFormat() == 'altus' -->
  484. <label for="collectionName" class="control-label"><div>${ _('Namespace') }</div></label>
  485. <div class="namespace-selection">
  486. <select data-bind="selectize: namespaces, value: targetNamespaceId, optionsValue: 'id', optionsText: 'name'" class="input-medium"></select>
  487. </div>
  488. <label class="checkbox inline-block"><input type="checkbox"> ${_('Copy Sentry privileges')}</label>
  489. <br/>
  490. <label class="control-label "><div>${ _('Database') }</div></label>
  491. <input type="text" class="form-control input-xlarge" data-bind="value: name" placeholder="${ _('Database') }">
  492. <a href="javascript:void(0);" data-bind="sqlContextPopover: { sourceType: $root.createWizard.source.sourceType, namespace: namespace, compute: compute, path: 'default', offset: { top: -3, left: 3 }}">
  493. <i class="fa fa-info"></i>
  494. </a>
  495. <!-- /ko -->
  496. <!-- ko if: outputFormat() == 'stream' -->
  497. <label for="collectionName" class="control-label "><div>${ _('Name') }</div></label>
  498. <!-- ko with: $root -->
  499. <input type="text" data-bind="value: createWizard.source.kafkaSelectedTopics">
  500. ## <div data-bind="template: { name: 'kafka-topic-template' }" class="margin-top-10 field inline-block"></div>
  501. <!-- /ko -->
  502. <!-- /ko -->
  503. <!-- ko if: outputFormat() == 'flume' -->
  504. <h4>${ _('Sink') }</h4>
  505. <div class="row-fluid">
  506. <div>
  507. <label class="control-label"><div>${ _('Type') }</div>
  508. <select class="input-medium" data-bind="selectize: channelSinkTypes, value: channelSinkType, optionsText: 'name', optionsValue: 'value'"></select>
  509. </label>
  510. <!-- ko if: channelSinkType() == 'solr' -->
  511. <label class="control-label"><div>${ _('Collection') }</div>
  512. <select class="input-xxlarge" data-bind="selectize: ['logIndex', 'apacheLogs'], value: channelSinkPath"></select>
  513. </label>
  514. <!-- /ko -->
  515. <!-- ko if: channelSinkType() -->
  516. <input data-bind="click: function() { channelSourceType(null); }" class="btn" value="${ _('Clear') }"/>
  517. <!-- /ko -->
  518. </div>
  519. </div>
  520. <!-- /ko -->
  521. <span class="help-inline muted" data-bind="visible: !isTargetExisting() && isTargetChecking()">
  522. <i class="fa fa-spinner fa-spin"></i>
  523. </span>
  524. <span class="help-inline muted" data-bind="visible: !$parent.createWizard.isValidDestination()">
  525. <i class="fa fa-warning" style="color: #c09853"></i>
  526. <!-- ko if: name() -->
  527. ${ _('Invalid characters') }
  528. <!-- /ko -->
  529. <!-- ko if: !name() -->
  530. ${ _('Empty name') }
  531. <!-- /ko -->
  532. </span>
  533. <span class="help-inline muted" data-bind="visible: isTargetExisting() && outputFormat() != 'altus'">
  534. <i class="fa fa-warning" style="color: #c09853"></i> ${ _('Already existing') } <span data-bind="text: outputFormat"></span>
  535. <a href="javascript:void(0)" data-bind="hueLink: existingTargetUrl(), text: name" title="${ _('Open') }"></a>
  536. </span>
  537. <span class="help-inline muted" data-bind="visible: !isTargetExisting() && $parent.createWizard.source.inputFormat() === 'rdbms' && outputFormat() == 'database' ">
  538. <i class="fa fa-warning" style="color: #c09853"></i> ${ _('Does not exist') } <span data-bind="text: outputFormat"></span>
  539. </span>
  540. </div>
  541. </div>
  542. </div>
  543. <!-- ko if: outputFormat() == 'table' && $root.createWizard.source.inputFormat() != 'rdbms' -->
  544. <div class="card step">
  545. <h4>${_('Properties')}</h4>
  546. <div class="card-body">
  547. <div class="control-group">
  548. <label for="destinationFormat" class="control-label"><div>${ _('Format') }</div>
  549. <select id="destinationFormat" data-bind="selectize: tableFormats, value: tableFormat, optionsValue: 'value', optionsText: 'name'"></select>
  550. </label>
  551. </div>
  552. <div class="control-group">
  553. <label class="control-label"><div>${ _('Extras') }</div>
  554. <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, toggle: showProperties" title="${ _('Show extra properties') }">
  555. <i class="fa fa-sliders fa-padding-top"></i>
  556. </a>
  557. </label>
  558. </div>
  559. <span data-bind="visible: showProperties">
  560. <div class="control-group">
  561. <label class="checkbox inline-block" data-bind="visible: tableFormat() != 'kudu'">
  562. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Store in Default location')}
  563. </label>
  564. </div>
  565. <div class="control-group" data-bind="visible: isTransactionalVisible">
  566. <label class="checkbox inline-block">
  567. <input type="checkbox" data-bind="checked: isTransactional"> ${_('Transactional table')}
  568. </label>
  569. <label class="checkbox inline-block" title="${_('Full transactional support available in Hive with ORC')}">
  570. <input type="checkbox" data-bind="checked: isInsertOnly, enable: isTransactionalUpdateEnabled"> ${_('Insert only')}
  571. </label>
  572. </div>
  573. <div class="control-group" data-bind="visible: !useDefaultLocation()">
  574. <label for="path" class="control-label"><div>${ _('External location') }</div>
  575. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  576. </label>
  577. </div>
  578. <div class="control-group">
  579. <label class="checkbox inline-block" data-bind="visible: $root.createWizard.source.inputFormat() != 'manual'">
  580. <input type="checkbox" data-bind="checked: importData, disable: !useDefaultLocation() && $parent.createWizard.source.path() == nonDefaultLocation();"> ${_('Import data')}
  581. </label>
  582. </div>
  583. <div class="control-group">
  584. <label><div>${ _('Description') }</div>
  585. <input type="text" class="form-control input-xxlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
  586. </label>
  587. </div>
  588. <div class="control-group" data-bind="visible: $root.createWizard.source.inputFormat() == 'file'">
  589. <label class="checkbox inline-block">
  590. <input type="checkbox" data-bind="checked: hasHeader"> ${_('Use first row as header')}
  591. </label>
  592. </div>
  593. <div class="control-group" data-bind="visible: tableFormat() == 'text'">
  594. <label class="checkbox inline-block">
  595. <input type="checkbox" data-bind="checked: useCustomDelimiters"> ${_('Custom char delimiters')}
  596. </label>
  597. </div>
  598. <span class="inline-labels" data-bind="visible: tableFormat() == 'text' && useCustomDelimiters()">
  599. <label for="fieldDelimiter" class="control-label"><div>${ _('Field') }</div>
  600. <select id="fieldDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customFieldDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  601. </label>
  602. <label for="collectionDelimiter" class="control-label"><div>${ _('Array, Map') }</div>
  603. <select id="collectionDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customCollectionDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  604. </label>
  605. <label for="structDelimiter" class="control-label"><div>${ _('Struct') }</div>
  606. <select id="structDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customMapDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  607. </label>
  608. </span>
  609. </span>
  610. <div class="control-group" data-bind="visible: tableFormat() == 'regexp'">
  611. <label for="customRegexp" class="control-label"><div>${ _('Regexp') }</div>
  612. <input id="customRegexp" class="input-xxlarge" type="text" data-bind="value: customRegexp" placeholder='([^]*) ([^]*) ([^]*) (-|\\[^\\]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\".*\") ([^ \"]*|\".*\"))?'>
  613. </label>
  614. </div>
  615. <div class="control-group" data-bind="visible: tableFormat() == 'kudu'">
  616. <label for="kuduPksTable" class="control-label"><div>${ _('Primary keys') }</div>
  617. ## At least one selected
  618. <select id="kuduPksTable" data-bind="selectize: columns, selectedOptions: primaryKeys, selectedObjects: primaryKeyObjects, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="3" multiple="true"></select>
  619. </label>
  620. </div>
  621. <label class="control-label"><div>${ _('Partitions') }</div>
  622. <!-- ko if: tableFormat() != 'kudu' && $root.createWizard.source.inputFormat() != 'rdbms' -->
  623. <div class="inline-table">
  624. <div class="form-inline" data-bind="foreach: partitionColumns">
  625. <a class="pointer pull-right margin-top-20" data-bind="click: function() { $parent.partitionColumns.remove($data); }"><i class="fa fa-minus"></i></a>
  626. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field inline-block"></div>
  627. <div class="clearfix"></div>
  628. </div>
  629. <a data-bind="click: function() { partitionColumns.push($root.loadDefaultField({isPartition: true})); }" class="pointer" title="${_('Add Partition')}"><i class="fa fa-plus fa-padding-top"></i> ${_('Add partition')}</a>
  630. </div>
  631. <!-- /ko -->
  632. <!-- ko if: tableFormat() == 'kudu' -->
  633. <div class="form-inline inline-table">
  634. <ul class="unstyled kudu-partitions" data-bind="foreach: kuduPartitionColumns">
  635. <li>
  636. <a class="pointer pull-right" data-bind="click: function() { $parent.kuduPartitionColumns.remove($data); }"><i class="fa fa-minus"></i></a>
  637. <select data-bind="selectize: $parent.primaryKeyObjects, selectedOptions: columns, optionsValue: 'name', optionsText: 'name', selectizeOptions: { placeholder: '${ _ko('Columns...') }' }" size="3" multiple="true"></select>
  638. <select data-bind="selectize: ['RANGE BY', 'HASH'], value: name"></select>
  639. <!-- ko if: name() == 'HASH' -->
  640. <input class="input-small" type="number" data-bind="value: int_val">
  641. <!-- /ko -->
  642. <!-- ko if: name() == 'RANGE BY' -->
  643. <div class="form-inline" data-bind="foreach: range_partitions">
  644. <div class="range-partition">
  645. <a class="pull-right" data-bind="click: function() { $parent.range_partitions.remove($data); }"><i class="fa fa-minus"></i></a>
  646. <!-- ko if: name() == 'VALUES' -->
  647. <input type="text" class="input-small" data-bind="value: lower_val">
  648. <select data-bind="selectize: ['<', '<='], value: include_lower_val"></select>
  649. <select data-bind="selectize: ['VALUES', 'VALUE'], value: name"></select>
  650. <select data-bind="selectize: ['<', '<='], value: include_upper_val"></select>
  651. <input type="text" class="input-small" data-bind="value: upper_val">
  652. <!-- /ko -->
  653. <!-- ko if: name() == 'VALUE' -->
  654. <select data-bind="selectize: ['VALUES', 'VALUE'], value: name"></select>
  655. <div class="inline-block" data-bind="foreach: values" style="max-width: 370px">
  656. <input class="input-small" type="text" data-bind="textInput: value" style="margin-bottom: 5px">
  657. <a data-bind="click: function() { $parent.values.remove($data); }"><i class="fa fa-minus"></i></a>
  658. </div>
  659. <a class="inline-block" data-bind="click: function() { values.push(ko.mapping.fromJS({value: ''})); }" title="${_('Add value')}"><i class="fa fa-plus"></i></a>
  660. <div class="clearfix"></div>
  661. <!-- /ko -->
  662. </div>
  663. </div>
  664. <a data-bind="click: function() { range_partitions.push(ko.mapping.fromJS($parent.KUDU_DEFAULT_RANGE_PARTITION_COLUMN)); }" title="${_('Add column')}"><i class="fa fa-plus"></i></a>
  665. <!-- /ko -->
  666. <!-- /ko -->
  667. </li>
  668. </ul>
  669. <a data-bind="click: function() { kuduPartitionColumns.push(ko.mapping.fromJS(KUDU_DEFAULT_PARTITION_COLUMN)); }" class="pointer" title="${_('Add partition')}"><i class="fa fa-plus"></i> ${_('Add partition')}</a>
  670. </div>
  671. <!-- /ko -->
  672. </label>
  673. </div>
  674. </div>
  675. <!-- /ko -->
  676. <!-- ko if: outputFormat() == 'index' -->
  677. <div class="card step">
  678. <h4>${_('Properties')}</h4>
  679. <div class="card-body">
  680. % if ENABLE_SCALABLE_INDEXER.get():
  681. <div class="control-group">
  682. <label class="checkbox inline-block" title="${ _('Execute a cluster job to index a large dataset.') }" data-bind="visible: $root.createWizard.source.inputFormat() == 'file'">
  683. <input type="checkbox" data-bind="checked: indexerRunJob"> ${_('Index with a job')}
  684. </label>
  685. <label for="path" class="control-label" data-bind="visible: indexerRunJob"><div>${ _('Libs') }</div>
  686. <input type="text" class="form-control path filechooser-input input-xlarge" data-bind="value: indexerJobLibPath, filechooser: indexerJobLibPath, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  687. </label>
  688. <!-- ko if: indexerRunJob() && indexerJobLibPath().length > 0 -->
  689. <a data-bind="hueLink: '/filebrowser/view=' + indexerJobLibPath()" title="${ _('Open') }" style="font-size: 14px" class="margin-left-10">
  690. <i class="fa fa-external-link-square"></i>
  691. </a>
  692. <!-- /ko -->
  693. </div>
  694. % endif
  695. <div class="control-group">
  696. <label for="kuduPksIndex" class="control-label"><div>${ _('Primary key') }</div>
  697. <select id="kuduPksIndex" data-bind="selectize: columns, selectedOptions: indexerPrimaryKey, selectedObjects: indexerPrimaryKeyObject, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="1" multiple="false"></select>
  698. </label>
  699. </div>
  700. <div class="control-group">
  701. <label for="kuduDefaultField" class="control-label"><div>${ _('Default field') }</div>
  702. <select id="kuduDefaultField" data-bind="selectize: columns, selectedOptions: indexerDefaultField, selectedObjects: indexerDefaultFieldObject, optionsValue: 'name', optionsText: 'name', innerSubscriber: 'name'" size="1" multiple="false"></select>
  703. </label>
  704. </div>
  705. <div class="control-group">
  706. <label class="control-label"><div>${ _('Extras') }</div>
  707. <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, toggle: showProperties" title="${ _('Show extra properties') }">
  708. <i class="fa fa-sliders fa-padding-top"></i>
  709. </a>
  710. </label>
  711. </div>
  712. <span data-bind="visible: showProperties">
  713. <div class="control-group">
  714. <label for="destinationFormatIndex" class="control-label"><div>${ _('Config set') }</div>
  715. <select id="destinationFormatIndex" data-bind="selectize: indexerConfigSets, value: indexerConfigSet, optionsValue: 'value', optionsText: 'name'"></select>
  716. </label>
  717. </div>
  718. <div class="control-group">
  719. <label for="indexerNumShards" class="control-label"><div>${ _('Num shards') }</div>
  720. <input type="number" class="input-small" placeholder="1" data-bind="value: indexerNumShards">
  721. </label>
  722. </div>
  723. <div class="control-group">
  724. <label for="indexerReplicationFactor" class="control-label"><div>${ _('Replication factor') }</div>
  725. <input type="number" class="input-small" placeholder="1" data-bind="value: indexerReplicationFactor">
  726. </label>
  727. </div>
  728. ## Router, maxShardsPer, shards, routerField
  729. <div class="control-group" data-bind="visible: $root.createWizard.source.inputFormat() == 'file'">
  730. <label class="checkbox inline-block">
  731. <input type="checkbox" data-bind="checked: hasHeader"> ${_('Use first row as header')}
  732. </label>
  733. </div>
  734. </span>
  735. </div>
  736. </div>
  737. <!-- /ko -->
  738. <!-- ko if: $root.createWizard.source.inputFormat() == 'rdbms' && ['database', 'file', 'table', 'hbase'].indexOf(outputFormat()) != -1 -->
  739. <div class="card step">
  740. <h4>${_('Properties')}</h4>
  741. <div class="card-body">
  742. <label class="control-label"><div>${ _('Libs') }</div>
  743. <div class="inline-table">
  744. <ul data-bind="sortable: { data: sqoopJobLibPaths, options: { axis: 'y', containment: 'parent', handle: '.move-widget' }}, visible: sqoopJobLibPaths().length" class="unstyled">
  745. <li>
  746. <div class="input-append" style="margin-bottom: 4px">
  747. <input type="text" class="filechooser-input input-xxlarge" data-bind="value: path, valueUpdate:'afterkeydown', filechooser: { value: path, isAddon: true }, filechooserOptions: { skipInitialPathIfEmpty: true }" placeholder="${ _('Path to the file, e.g. hdfs://localhost:8020/user/hue/file.hue') }"/>
  748. <span class="add-on move-widget muted" data-bind="visible: $parent.sqoopJobLibPaths().length > 1"><i class="fa fa-arrows"></i></span>
  749. <a class="add-on muted" href="javascript: void(0);" data-bind="click: function(){ $parent.removeSqoopJobLibPath($data); }"><i class="fa fa-minus"></i></a>
  750. </div>
  751. </li>
  752. </ul>
  753. <div class="config-property-add-value" style="margin-top: 5px;">
  754. <a class="inactive-action pointer" style="padding: 3px 10px 3px 3px;;" data-bind="click: addSqoopJobLibPath">
  755. <i class="fa fa-plus"></i>
  756. </a>
  757. </div>
  758. </div>
  759. </label>
  760. </div>
  761. <div class="control-group">
  762. <label class="control-label"><div>${ _('Extras') }</div>
  763. <a href="javascript:void(0)" data-bind="css: { 'inactive-action': !showProperties() }, toggle: showProperties" title="${ _('Show extra properties') }">
  764. <i class="fa fa-sliders fa-padding-top"></i>
  765. </a>
  766. </label>
  767. </div>
  768. <span data-bind="visible: showProperties">
  769. <div class="control-group">
  770. <label><div>${ _('Mappers') }</div>
  771. <input type="number" class="form-control input-small" data-bind="textInput: numMappers">
  772. </label>
  773. </div>
  774. <div class="control-group">
  775. <!-- ko if: !$root.createWizard.source.rdbmsAllTablesSelected() -->
  776. <label for="rdbmsSplitBy" class="control-label"><div>${ _('Split By') }</div>
  777. <select id="rdbmsSplitBy" data-bind="selectize: columns, value: rdbmsSplitByColumn, optionsValue: 'name', optionsText: 'name'"></select>
  778. </label>
  779. <!-- /ko -->
  780. </div>
  781. <div class="control-group" data-bind="visible: outputFormat() == 'file' && !$root.createWizard.source.rdbmsAllTablesSelected()">
  782. <label for="destinationFormat" class="control-label"><div>${ _('Format') }</div>
  783. <select id="destinationFormat" data-bind="selectize: rdbmsFileOutputFormats, value: rdbmsFileOutputFormat, optionsValue: 'value', optionsText: 'name'"></select>
  784. </label>
  785. </div>
  786. <span class="inline-labels" data-bind="visible: rdbmsFileOutputFormat() == 'text' && outputFormat() == 'file' && !$root.createWizard.source.rdbmsAllTablesSelected()">
  787. <label for="fieldDelimiter" class="control-label"><div>${ _('Fields') }</div>
  788. <select id="fieldDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customFieldsDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  789. </label>
  790. <label for="collectionDelimiter" class="control-label"><div>${ _('Line') }</div>
  791. <select id="collectionDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customLineDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  792. </label>
  793. <label for="structDelimiter" class="control-label"><div>${ _('Optionally Enclosed By') }</div>
  794. <select id="structDelimiter" data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: customEnclosedByDelimiter, optionsValue: 'value', optionsText: 'name'"></select>
  795. </label>
  796. </span>
  797. <!-- ko if: outputFormat() === 'database' -->
  798. <span>
  799. <label class="checkbox">
  800. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Default location')}
  801. </label>
  802. <span data-bind="visible: !useDefaultLocation()">
  803. <label for="path" class="control-label"><div>${ _('External location') }</div>
  804. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  805. </label>
  806. </span>
  807. </span>
  808. <!-- /ko -->
  809. </span>
  810. </div>
  811. <!-- /ko -->
  812. <!-- ko if: $root.createWizard.source.inputFormat() == 'kafka' -->
  813. <div class="card step">
  814. <h4>${_('Properties')}</h4>
  815. <div class="card-body">
  816. <label class="control-label"><div>${ _('Libs') }</div>
  817. <div class="inline-table">
  818. <ul data-bind="sortable: { data: sqoopJobLibPaths, options: { axis: 'y', containment: 'parent', handle: '.move-widget' }}, visible: sqoopJobLibPaths().length" class="unstyled">
  819. <li>
  820. <div class="input-append" style="margin-bottom: 4px">
  821. <input type="text" class="filechooser-input input-xxlarge" data-bind="value: path, valueUpdate:'afterkeydown', filechooser: { value: path, isAddon: true }, filechooserOptions: { skipInitialPathIfEmpty: true }" placeholder="${ _('Path to the file, e.g. hdfs://localhost:8020/user/hue/file.hue') }"/>
  822. <span class="add-on move-widget muted" data-bind="visible: $parent.sqoopJobLibPaths().length > 1"><i class="fa fa-arrows"></i></span>
  823. <a class="add-on muted" href="javascript: void(0);" data-bind="click: function(){ $parent.removeSqoopJobLibPath($data); }"><i class="fa fa-minus"></i></a>
  824. </div>
  825. </li>
  826. </ul>
  827. <div class="config-property-add-value" style="margin-top: 5px;">
  828. <a class="inactive-action pointer" style="padding: 3px 10px 3px 3px;" data-bind="click: addSqoopJobLibPath">
  829. <i class="fa fa-plus"></i>
  830. </a>
  831. </div>
  832. </div>
  833. </label>
  834. </div>
  835. </div>
  836. <!-- /ko -->
  837. <!-- ko if: ['table', 'index', 'hbase'].indexOf(outputFormat()) != -1 -->
  838. <div class="card step">
  839. <h4>
  840. <!-- ko if: fieldEditorEnabled -->
  841. <!-- ko if: useFieldEditor -->
  842. <a class="inactive-action" href="javascript:void(0);" data-bind="toggle: useFieldEditor">${_('Fields')} /</a> ${_('Editor')}
  843. <!-- /ko -->
  844. <!-- ko ifnot: useFieldEditor -->
  845. ${_('Fields')} / <a class="inactive-action" href="javascript:void(0);" data-bind="toggle: useFieldEditor">${_('Editor')}</a>
  846. <!-- /ko -->
  847. <!-- /ko -->
  848. <!-- ko ifnot: fieldEditorEnabled -->
  849. ${_('Fields')} <!-- ko if: $root.createWizard.isGuessingFieldTypes --><i class="fa fa-spinner fa-spin"></i><!-- /ko -->
  850. <a class="inactive-action pointer" data-bind="visible: columns().length > 0, publish: 'importer.show.bulkeditor'" href="javascript:void(0)">
  851. <i class="fa fa-edit"></i>
  852. </a>
  853. <!-- /ko -->
  854. </h4>
  855. <div class="card-body no-margin-top columns-form">
  856. <!-- ko if: useFieldEditor -->
  857. <div class="control-group" style="margin-bottom: 5px;">
  858. <label class="control-label"><div style="width: initial">${ _('Language') }</div>
  859. <select>
  860. <option value="sql">SQL</option>
  861. <option value="morphline">Morphline</option>
  862. </select>
  863. </label>
  864. </div>
  865. <div data-bind="component: { name: 'hue-simple-ace-editor-multi', params: {
  866. value: fieldEditorValue,
  867. placeHolder: fieldEditorPlaceHolder,
  868. autocomplete: { type: sourceType },
  869. lines: 5,
  870. aceOptions: {
  871. minLines: 10,
  872. maxLines: 25
  873. },
  874. database: fieldEditorDatabase,
  875. namespace: namespace,
  876. compute: compute,
  877. temporaryOnly: true,
  878. mode: sourceType
  879. }}"></div>
  880. <!-- /ko -->
  881. <!-- ko ifnot: useFieldEditor -->
  882. <!-- ko if: $root.createWizard.source.inputFormat() === 'manual' -->
  883. <form class="form-inline inline-table" data-bind="foreach: columns">
  884. <!-- ko if: $parent.outputFormat() == 'table' -->
  885. <a class="pointer pull-right margin-top-20" data-bind="click: function() { $parent.columns.remove($data); }"><i class="fa fa-minus"></i></a>
  886. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field inline-block"></div>
  887. <div class="clearfix"></div>
  888. <!-- /ko -->
  889. <!-- ko if: $parent.outputFormat() == 'index' -->
  890. <a class="pointer pull-right margin-top-20" data-bind="click: function() { $parent.columns.remove($data); }"><i class="fa fa-minus"></i></a>
  891. <div data-bind="template: { name: 'index-field-template', data: $data }, css: { 'disabled': !keep() }" class="margin-top-10 field inline-block index-field"></div>
  892. <div class="clearfix"></div>
  893. <!-- /ko -->
  894. </form>
  895. <div class="clearfix"></div>
  896. <!-- ko if: outputFormat() == 'table' || outputFormat() == 'index' -->
  897. <a data-bind="click: function() { columns.push($root.loadDefaultField({})); }" class="pointer" title="${_('Add Field')}"><i class="fa fa-plus"></i> ${_('Add Field')}</a>
  898. <!-- /ko -->
  899. <!-- /ko -->
  900. <!-- ko ifnot: $root.createWizard.source.inputFormat() === 'manual' -->
  901. <form class="form-inline inline-table" data-bind="foreachVisible: { data: columns, minHeight: 54, container: MAIN_SCROLLABLE }">
  902. <!-- ko if: $parent.outputFormat() == 'table' && $root.createWizard.source.inputFormat() != 'rdbms' -->
  903. <div data-bind="template: { name: 'table-field-template', data: $data }" class="margin-top-10 field"></div>
  904. <!-- /ko -->
  905. <!-- ko if: (['file', 'table', 'hbase'].indexOf($parent.outputFormat()) != -1 && $root.createWizard.source.inputFormat() == 'rdbms') || $parent.outputFormat() == 'index' -->
  906. <div data-bind="template: { name: 'index-field-template', data: $data }, css: { 'disabled': !keep() }" class="margin-top-10 field index-field"></div>
  907. <!-- /ko -->
  908. </form>
  909. <div class="clearfix"></div>
  910. <!-- /ko -->
  911. <!-- /ko -->
  912. </div>
  913. </div>
  914. <!-- /ko -->
  915. <!-- ko if: outputFormat() == 'database' && $root.createWizard.source.inputFormat() !== 'rdbms' -->
  916. <div class="card step">
  917. <h4>${_('Properties')}</h4>
  918. <div class="card-body">
  919. <label><div>${ _('Description') }</div>
  920. <input type="text" class="form-control input-xlarge" data-bind="value: description, valueUpdate: 'afterkeydown'" placeholder="${ _('Description') }">
  921. </label>
  922. <label class="checkbox">
  923. <input type="checkbox" data-bind="checked: useDefaultLocation"> ${_('Default location')}
  924. </label>
  925. <span data-bind="visible: !useDefaultLocation()">
  926. <label for="path" class="control-label"><div>${ _('External location') }</div>
  927. <input type="text" class="form-control path filechooser-input input-xxlarge" data-bind="value: nonDefaultLocation, filechooser: nonDefaultLocation, filechooserOptions: { linkMarkup: true, skipInitialPathIfEmpty: true }, valueUpdate: 'afterkeydown'">
  928. </label>
  929. </span>
  930. </div>
  931. </div>
  932. <!-- /ko -->
  933. <!-- /ko -->
  934. <!-- /ko -->
  935. <div class="form-actions">
  936. <!-- ko if: previousStepVisible -->
  937. <button class="btn" data-bind="click: previousStep">${ _('Back') }</button>
  938. <!-- /ko -->
  939. <!-- ko if: currentStep() == 1 -->
  940. <button class="btn" data-bind="enable: !createWizard.isGuessingFormat() && createWizard.source.show(), click: function() { currentStep(2); }">
  941. ${_('Next')}
  942. </button>
  943. <!-- /ko -->
  944. <!-- ko if: currentStep() == 2 -->
  945. <button class="btn btn-primary disable-feedback" data-bind="click: function() { createWizard.indexFile(); }, enable: createWizard.readyToIndex() && !createWizard.indexingStarted()">
  946. ${ _('Submit') } <i class="fa fa-spinner fa-spin" data-bind="visible: createWizard.indexingStarted"></i>
  947. </button>
  948. % if ENABLE_ENVELOPE.get():
  949. <button class="btn disable-feedback" data-bind="click: createWizard.showCommands, enable: createWizard.readyToIndex()">
  950. ${ _('Show Commands') }
  951. </button>
  952. <button class="btn disable-feedback">
  953. ${ _('Save') }
  954. </button>
  955. % endif
  956. <!-- /ko -->
  957. <span data-bind="visible: createWizard.editorId">
  958. <button class="btn btn-success" data-bind="click: function(){ window.open('${ url('notebook:editor') }?editor=' + createWizard.editorId()) }" title="${ _('Open') }">
  959. ${_('Check status')}
  960. </button>
  961. </span>
  962. <div id="importerNotebook"></div>
  963. </div>
  964. </div>
  965. <div id="fieldsBulkEditor" class="modal hide fade">
  966. <div class="modal-header">
  967. <button type="button" class="close" data-dismiss="modal" aria-label="${ _('Close') }"><span aria-hidden="true">&times;</span></button>
  968. <h2 class="modal-title">${ _('Write or paste comma separated field names') }</h2>
  969. </div>
  970. <div class="modal-body">
  971. <input type="text" class="input-xxlarge" placeholder="${ _('e.g. id, name, salary') }" data-bind="textInput: createWizard.destination.bulkColumnNames">
  972. </div>
  973. <div class="modal-footer">
  974. <button class="btn" data-dismiss="modal" aria-hidden="true">${ _('Cancel') }</button>
  975. <button class="btn btn-primary" data-bind="click: createWizard.destination.processBulkColumnNames">${ _('Change field names') }</button>
  976. </div>
  977. </div>
  978. </script>
  979. <script type="text/html" id="table-field-template">
  980. <div>
  981. <label data-bind="visible: level() == 0 || ($parent.type() != 'array' && $parent.type() != 'map')">${ _('Name') }&nbsp;
  982. <input type="text" class="input-large" placeholder="${ _('Field name') }" required data-bind="textInput: name">
  983. </label>
  984. <label class="margin-left-5">${ _('Type') }&nbsp;
  985. <!-- ko if: !(level() > 0 && $parent.type() == 'map') -->
  986. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.hiveFieldTypes, value: type"></select>
  987. <!-- /ko -->
  988. <!-- ko if: level() > 0 && $parent.type() == 'map' -->
  989. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.hivePrimitiveFieldTypes, value: keyType"></select>
  990. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.hiveFieldTypes, value: type"></select>
  991. <!-- /ko -->
  992. <input type="number" class="input-small" placeholder="${ _('Length') }" data-bind="value: length, visible: type() == 'varchar' || type() == 'char'">
  993. <!-- ko if: $parent.type() == 'decimal' -->
  994. <input type="number" class="input-small" placeholder="${ _('Precision') }" data-bind="value: precision">
  995. <input type="number" class="input-small" placeholder="${ _('Scale') }" data-bind="value: scale">
  996. <!-- /ko -->
  997. </label>
  998. <!-- ko if: $root.createWizard.source.inputFormat() != 'manual' && typeof isPartition !== 'undefined' && isPartition() -->
  999. <label class="margin-left-5">${ _('Value') }&nbsp;
  1000. <input type="text" class="input-medium margin-left-5" placeholder="${ _('Partition value') }" data-bind="value: partitionValue">
  1001. </label>
  1002. <!-- /ko -->
  1003. <span data-bind="visible: level() == 0 || ($parent.type() != 'array' && $parent.type() != 'map')">
  1004. <a href="javascript:void(0)" title="${ _('Show field properties') }" data-bind="css: {'inactive-action': !showProperties()}, click: function() {showProperties(!showProperties()) }"><i class="fa fa-sliders"></i></a>
  1005. <span data-bind="visible: showProperties">
  1006. <input type="text" class="input-medium margin-left-5" placeholder="${ _('Field comment') }" data-bind="value: comment">
  1007. <label class="checkbox" data-bind="visible: $root.createWizard.destination.tableFormat() == 'kudu'">
  1008. <input type="checkbox" data-bind="checked: keep"> ${_('Keep')}
  1009. </label>
  1010. </span>
  1011. </span>
  1012. <!-- ko if: level() > 0 && $parent.type() == 'struct' && $parent.nested().length > 1 -->
  1013. <a data-bind="click: function() { $parent.nested.remove($data); }"><i class="fa fa-minus"></i></a>
  1014. <!-- /ko -->
  1015. <div class="inline-block" data-bind="template: { name:'field-column-example' }"></div>
  1016. <!-- ko if: type() == 'array' || type() == 'map' || type() == 'struct' -->
  1017. <div class="operation" data-bind="template: { name: 'table-field-template', foreach: nested }"></div>
  1018. <a data-bind="click: function() { nested.push($root.loadDefaultField({level: level() + 1})); }, visible: type() == 'struct'"><i class="fa fa-plus"></i></a>
  1019. <!-- /ko -->
  1020. </div>
  1021. </script>
  1022. <script type="text/html" id="index-field-template">
  1023. <label>${ _('Name') }&nbsp;
  1024. <input type="text" class="input-large" placeholder="${ _('Field name') }" data-bind="value: name, enable: keep" pattern="^(?!_version_)[a-zA-Z_][a-zA-Z0-9_]*$" title="${ _('Only alphanumeric and underscore characters and not _version_') }">
  1025. </label>
  1026. <!-- ko if: $root.createWizard.source.inputFormat() != 'rdbms' -->
  1027. <label class="margin-left-5">${ _('Type') }&nbsp;
  1028. <select class="input-small" data-bind="browserAwareSelectize: $root.createWizard.fieldTypes, value: type"></select>
  1029. </label>
  1030. <!-- /ko -->
  1031. <!-- ko if: $root.createWizard.source.inputFormat() == 'rdbms' -->
  1032. <label class="margin-left-5">${ _('Type') }&nbsp;
  1033. <input type="text" class="input-small" placeholder="${ _('Field Type') }" data-bind="value: type, enable: false">
  1034. </label>
  1035. <!-- /ko -->
  1036. <a href="javascript:void(0)" title="${ _('Show field properties') }" data-bind="css: {'inactive-action': !showProperties()}, click: function() {showProperties(!showProperties()) }"><i class="fa fa-sliders"></i></a>
  1037. <span data-bind="visible: showProperties" class="field-properties">
  1038. <!-- ko if: $root.createWizard.source.inputFormat() != 'rdbms' -->
  1039. <label class="checkbox">
  1040. <input type="checkbox" data-bind="checked: unique"> ${_('Unique')}
  1041. </label>
  1042. <label class="checkbox">
  1043. <input type="checkbox" data-bind="checked: required"> ${_('Required')}
  1044. </label>
  1045. <!-- /ko -->
  1046. <label class="checkbox">
  1047. <input type="checkbox" data-bind="checked: keep"> ${_('Keep')}
  1048. </label>
  1049. </span>
  1050. <!-- ko if: operations().length == 0 -->
  1051. <a class="pointer margin-left-20" data-bind="click: $root.createWizard.addOperation, visible: $root.createWizard.destination.outputFormat() == 'index'" title="${_('Add Operation')}">
  1052. <i class="fa fa-plus"></i> ${_('Operation')}
  1053. </a>
  1054. <!-- /ko -->
  1055. <div class="inline-block" data-bind="template: { name: 'field-column-example' }"></div>
  1056. <div data-bind="foreach: operations">
  1057. <div data-bind="template: { name:'operation-template',data: { operation: $data, list: $parent.operations } }"></div>
  1058. </div>
  1059. <!-- ko if: operations().length > 0 -->
  1060. <a class="pointer" data-bind="click: $root.createWizard.addOperation" title="${_('Add Operation')}"><i class="fa fa-plus"></i> ${_('Operation to')} <span data-bind="text: name"></span></a>
  1061. <!-- /ko -->
  1062. </script>
  1063. <script type="text/html" id="field-column-example">
  1064. <!-- ko if: $root.createWizard.source.inputFormat() != 'manual' && level() == 0 && (typeof isPartition === 'undefined' || !isPartition()) -->
  1065. <!-- ko if: $root.createWizard.source.sample() && $root.createWizard.source.sample().length > 0 -->
  1066. <div class="inline-block muted field-content-preview" data-bind="truncatedText: $root.createWizard.source.sample()[0][$index()]"></div>
  1067. <!-- ko if: $root.createWizard.source.sample().length > 1 -->
  1068. <div class="inline-block muted field-content-preview" data-bind="truncatedText: $root.createWizard.source.sample()[1][$index()]"></div>
  1069. <!-- /ko -->
  1070. <!-- /ko -->
  1071. <!-- ko if: !$root.createWizard.source.sample() || $root.createWizard.source.sample().length === 0 -->
  1072. <div class="inline-block muted field-content-preview">${ _("No sample to be shown") }</div>
  1073. <!-- /ko -->
  1074. <!-- /ko -->
  1075. </script>
  1076. <script type="text/html" id="operation-template">
  1077. <div class="operation">
  1078. <select data-bind="browserAwareSelectize: $root.createWizard.operationTypesFiltered, value: operation.type"></select>
  1079. <!-- ko template: "args-template" --><!-- /ko -->
  1080. <!-- ko if: operation.settings().outputType() == "custom_fields" -->
  1081. <label class="margin-left-5">${ _('Number of expected fields') }
  1082. <input type="number" class="input-mini" data-bind="value: operation.numExpectedFields">
  1083. </label>
  1084. <!-- /ko -->
  1085. <a class="pointer margin-left-20" data-bind="click: function(){$root.createWizard.removeOperation(operation, list)}" title="${ _('Remove') }"><i class="fa fa-times"></i></a>
  1086. <div class="margin-left-20" data-bind="foreach: operation.fields">
  1087. <div data-bind="template: { name: 'index-field-template', data: $data }, css:{ 'disabled': !keep() }" class="margin-top-10 field index-field"></div>
  1088. </div>
  1089. </div>
  1090. </script>
  1091. <script type="text/html" id="args-template">
  1092. <!-- ko foreach: {data: operation.settings().getArguments(), as: 'argument'} -->
  1093. <!-- ko template: {name: 'arg-' + argument.type, data: {description: argument.description, value: $parent.operation.settings()[argument.name]}}--><!-- /ko -->
  1094. <!-- /ko -->
  1095. </script>
  1096. <script type="text/html" id="arg-text">
  1097. <label>
  1098. <div data-bind="text: description"></div>
  1099. <input type="text" class="input" data-bind="attr: {placeholder: description}, value: value">
  1100. </label>
  1101. </script>
  1102. <script type="text/html" id="arg-text-delimiter">
  1103. <label>
  1104. <div data-bind="text: description"></div>
  1105. <select data-bind="selectize: $root.createWizard.customDelimiters, selectizeOptions: { onOptionAdd: function(value){ $root.createWizard.customDelimiters.push({ 'value': value, 'name': value }) }, create: true, maxLength: 2 }, value: value, optionsValue: 'value', optionsText: 'name', attr: {placeholder: description}"></select>
  1106. </label>
  1107. </script>
  1108. <script type="text/html" id="arg-checkbox">
  1109. <label class="checkbox">
  1110. <input type="checkbox" data-bind="checked: value">
  1111. <span data-bind="text: description" style="vertical-align: middle"></span>
  1112. </label>
  1113. </script>
  1114. <script type="text/html" id="arg-mapping">
  1115. <!-- ko foreach: value-->
  1116. <div>
  1117. <input type="text" data-bind="value: key, attr: {placeholder: 'key'}">
  1118. <input type="text" data-bind="value: value, attr: {placeholder: 'value'}">
  1119. <button class="btn" data-bind="click: function(){$parent.value.remove($data)}">${_('Remove Pair')}</button>
  1120. </div>
  1121. <!-- /ko -->
  1122. <button class="btn" data-bind="click: function(){value.push({key: ko.observable(''), value: ko.observable('')})}">${_('Add Pair')}</button>
  1123. <br>
  1124. </script>
  1125. <script type="text/html" id="field-preview-header-template">
  1126. <th data-bind="visible: keep, text: name" style="padding-right:60px"></th>
  1127. <!-- ko foreach: operations -->
  1128. <!--ko foreach: fields -->
  1129. <!-- ko template: 'field-preview-header-template' --><!-- /ko -->
  1130. <!-- /ko -->
  1131. <!--/ko -->
  1132. </script>
  1133. <script type="text/html" id="output-generated-field-data-template">
  1134. <!-- ko foreach: operations -->
  1135. <!--ko foreach: fields -->
  1136. <td data-bind="visible: keep">[[${_('generated')}]]</td>
  1137. <!-- ko template: 'output-generated-field-data-template' --><!-- /ko -->
  1138. <!-- /ko -->
  1139. <!--/ko -->
  1140. </script>
  1141. <script type="text/html" id="importerNotebook-progress">
  1142. <!-- ko with: selectedNotebook -->
  1143. <!-- ko foreach: snippets -->
  1144. <div class="progress-snippet progress" data-bind="css: {
  1145. 'progress-starting': progress() == 0 && status() == 'running',
  1146. 'progress-warning': progress() > 0 && progress() < 100,
  1147. 'progress-success': progress() == 100,
  1148. 'progress-danger': progress() == 0 && errors().length > 0}" style="background-color: #FFF; width: 100%; height: 4px">
  1149. <div class="bar" data-bind="style: {'width': (errors().length > 0 ? 100 : Math.max(2, progress())) + '%'}"></div>
  1150. </div>
  1151. <div class="snippet-error-container alert alert-error alert-error-gradient" data-bind="visible: errors().length > 0">
  1152. <ul class="unstyled" data-bind="foreach: errors">
  1153. <li data-bind="text: message"></li>
  1154. </ul>
  1155. </div>
  1156. <!-- /ko -->
  1157. <!-- /ko -->
  1158. </script>
  1159. <script type="text/html" id="kafka-topic-template">
  1160. <div class="control-group">
  1161. <label class="control-label"><div>${ _('Topics') }</div>
  1162. <select class="input-xxlarge" data-bind="options: createWizard.source.kafkaTopics,
  1163. value: createWizard.source.kafkaSelectedTopics,
  1164. optionsCaption: '${ _("Choose...") }'"
  1165. placeholder="${ _('The list of topics to consume, e.g. orders,returns') }">
  1166. </select>
  1167. ## <select data-bind="selectize: createWizard.source.kafkaTopics, value: createWizard.source.kafkaSelectedTopics" placeholder="${ _('The list of topics to consume, e.g. orders,returns') }"></select>
  1168. </label>
  1169. <br/>
  1170. <label class="control-group" data-bind="visible: createWizard.source.kafkaSelectedTopics">
  1171. <label class="control-label"><div>${ _('Schema') }</div>
  1172. <label class="radio margin-right-10">
  1173. <input type="radio" name="kafkaSchemaManual" value="manual" data-bind="checked: createWizard.source.kafkaSchemaManual" /> ${_('Manual')}
  1174. </label>
  1175. <label class="radio">
  1176. <input type="radio" name="kafkaSchemaManual" value="detect" data-bind="checked: createWizard.source.kafkaSchemaManual" /> ${_('Guess')}
  1177. </label>
  1178. </label>
  1179. <!-- ko if: createWizard.source.kafkaSchemaManual() == 'manual' -->
  1180. ##<label class="control-label"><div>${ _('Encoding') }</div>
  1181. ## <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldType">
  1182. ##</label>
  1183. <br/>
  1184. <label class="control-label"><div>${ _('Type') }</div>
  1185. <select class="input-medium" data-bind="options: ['delimited', 'bitarray'], value: createWizard.source.kafkaFieldType"></select>
  1186. </label>
  1187. <label class="control-label"><div>${ _('Delimiter') }</div>
  1188. <input type="text" class="input-small" data-bind="value: createWizard.source.kafkaFieldDelimiter">
  1189. </label>
  1190. <br/>
  1191. <label class="control-label"><div>${ _('Field names') }</div>
  1192. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldNames" placeholder="${ _('The list of fields to consume, e.g. orders,returns') }">
  1193. </label>
  1194. <br/>
  1195. <label class="control-label"><div>${ _('Field types') }</div>
  1196. <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.kafkaFieldTypes" placeholder="${ _('The list of field typs, e.g. string,int') }">
  1197. </label>
  1198. <br/>
  1199. <label class="control-label" data-bind="visible: createWizard.source.hasStreamSelected"><div></div>
  1200. <button class="btn" data-bind="click: createWizard.source.streamCheckConnection">${_('Test')}</button>
  1201. </label>
  1202. <!-- /ko -->
  1203. </div>
  1204. </div>
  1205. </script>
  1206. <div id="showCommandsModal" class="modal transparent-modal hide" data-backdrop="true" style="width:980px; margin-left:-510px!important">
  1207. <div class="modal-header">
  1208. <button type="button" class="close" data-dismiss="modal" aria-label="${ _('Close') }"><span aria-hidden="true">&times;</span></button>
  1209. <h2 class="modal-title">${_('Commands')}</h2>
  1210. </div>
  1211. <div class="modal-body">
  1212. <div class="editor-help two-pane">
  1213. <div class="tab-content">
  1214. <div class="tab-pane active" id="help-editor-syntax">
  1215. <ul class="nav nav-tabs">
  1216. <li class="active">
  1217. <a href="#help-editor-syntax-comment" data-bind="click: function(){ $('a[href=\'#help-editor-syntax-comment\']').tab('show'); }">
  1218. ${ _('Commands')}
  1219. </a>
  1220. </li>
  1221. <li>
  1222. <a href="#help-editor-syntax-click" data-bind="click: function(){ $('a[href=\'#help-editor-syntax-click\']').tab('show'); }">
  1223. ${ _('Shell')}
  1224. </a>
  1225. </li>
  1226. <li>
  1227. <a href="#help-editor-syntax-multiquery" data-bind="click: function(){ $('a[href=\'#help-editor-syntax-multiquery\']').tab('show'); }">
  1228. ${ _('REST')}
  1229. </a>
  1230. </li>
  1231. </ul>
  1232. <div class="tab-content">
  1233. <div class="tab-pane active" id="help-editor-syntax-comment">
  1234. <div data-bind="text: createWizard.commands" style="white-space: pre-wrap;">
  1235. </div>
  1236. </div>
  1237. <div class="tab-pane" id="help-editor-syntax-click">
  1238. <ul class="nav help-list-spacing">
  1239. </ul>
  1240. </div>
  1241. <div class="tab-pane" id="help-editor-syntax-multiquery">
  1242. </div>
  1243. </div>
  1244. </div>
  1245. </div>
  1246. </div>
  1247. </div>
  1248. <div class="modal-footer">
  1249. <a href="javascript: void(0)" class="btn" data-dismiss="modal">${_('Close')}</a>
  1250. </div>
  1251. </div>
  1252. <script type="text/javascript">
  1253. % if is_embeddable:
  1254. var MAIN_SCROLLABLE = '.page-content';
  1255. % else:
  1256. var MAIN_SCROLLABLE = '.content-panel';
  1257. % endif
  1258. (function () {
  1259. if (ko.options) {
  1260. ko.options.deferUpdates = true;
  1261. }
  1262. var MAPPINGS = {
  1263. SOLR_TO_HIVE: {
  1264. "string": "string",
  1265. "plong": "bigint",
  1266. "pdouble": "double",
  1267. "pdate": "timestamp",
  1268. "long": "bigint",
  1269. "double": "double",
  1270. "date": "timestamp",
  1271. "boolean": "boolean",
  1272. },
  1273. HIVE_TO_SOLR: {
  1274. "bigint": "plong"
  1275. },
  1276. get: function(type, key, defaultValue) {
  1277. return type[key] || defaultValue
  1278. }
  1279. };
  1280. var fieldNum = 0;
  1281. var getNewFieldName = function () {
  1282. fieldNum++;
  1283. return "new_field_" + fieldNum
  1284. };
  1285. var createDefaultField = function () {
  1286. var defaultField = ko.mapping.fromJS(${default_field_type | n});
  1287. defaultField.name = ko.observable(getNewFieldName());
  1288. return defaultField;
  1289. };
  1290. var Operation = function (type) {
  1291. var self = this;
  1292. var createArgumentValue = function (arg) {
  1293. if (arg.type === "mapping") {
  1294. return ko.observableArray([]);
  1295. }
  1296. if (arg.type === "checkbox") {
  1297. return ko.observable(false);
  1298. }
  1299. return ko.observable("");
  1300. };
  1301. var constructSettings = function (type) {
  1302. var settings = {};
  1303. var operation = viewModel.createWizard.operationTypes.find(function (currOperation) {
  1304. return currOperation.name === type;
  1305. });
  1306. for (var i = 0; i < operation.args.length; i++) {
  1307. var argVal = createArgumentValue(operation.args[i]);
  1308. if (operation.args[i].type === "checkbox" && operation.outputType === "checkbox_fields") {
  1309. argVal.subscribe(function (newVal) {
  1310. if (newVal) {
  1311. self.fields.push(createDefaultField());
  1312. } else {
  1313. self.fields.pop();
  1314. }
  1315. });
  1316. }
  1317. settings[operation.args[i].name] = argVal;
  1318. }
  1319. settings.getArguments = function () {
  1320. return operation.args
  1321. };
  1322. settings.outputType = function () {
  1323. return operation.outputType;
  1324. };
  1325. return settings;
  1326. };
  1327. var init = function () {
  1328. self.fields([]);
  1329. self.numExpectedFields(0);
  1330. self.numExpectedFields.subscribe(function (numExpectedFields) {
  1331. if (numExpectedFields < self.fields().length) {
  1332. self.fields(self.fields().slice(0, numExpectedFields));
  1333. } else if (numExpectedFields > self.fields().length) {
  1334. var difference = numExpectedFields - self.fields().length;
  1335. for (var i = 0; i < difference; i++) {
  1336. self.fields.push(createDefaultField());
  1337. }
  1338. }
  1339. });
  1340. self.settings(constructSettings(self.type()));
  1341. };
  1342. self.load = function (data) {
  1343. self.numExpectedFields(data.numExpectedFields);
  1344. var newSettings = constructSettings(data.type);
  1345. for (var key in data.settings) {
  1346. newSettings[key] = ko.mapping.fromJS(data.settings[key]);
  1347. }
  1348. self.settings(newSettings);
  1349. data.fields.forEach(function (field) {
  1350. self.fields.push(loadField(field));
  1351. });
  1352. };
  1353. self.type = ko.observable(type);
  1354. self.fields = ko.observableArray();
  1355. self.numExpectedFields = ko.observable();
  1356. self.settings = ko.observable();
  1357. init();
  1358. self.type.subscribe(function () {
  1359. init();
  1360. });
  1361. };
  1362. var FileType = function (typeName, args) {
  1363. var self = this;
  1364. var type;
  1365. var init = function () {
  1366. self.type = ko.observable(typeName);
  1367. var types = viewModel.createWizard.fileTypes;
  1368. for (var i = 0; i < types.length; i++) {
  1369. if (types[i].name === typeName) {
  1370. type = types[i];
  1371. break;
  1372. }
  1373. }
  1374. if (type) {
  1375. for (var i = 0; i < type.args.length; i++) {
  1376. self[type.args[i].name] = ko.observable();
  1377. }
  1378. if (args) {
  1379. loadFromObj(args);
  1380. }
  1381. setTimeout(function () {
  1382. var types = type.args.filter(function (x){
  1383. return x && (x.type !== 'checkbox' || x.name === 'hasHeader');
  1384. });
  1385. for (var i = 0; i < types.length; i++) {
  1386. self[types[i].name].subscribe(function() {
  1387. // Update the data preview when tweaking Format options on step 1
  1388. viewModel.createWizard.guessFieldTypes();
  1389. });
  1390. }
  1391. }); // Prevent notification on start
  1392. }
  1393. };
  1394. var loadFromObj = function (args) {
  1395. for (var attr in args) {
  1396. self[attr] = ko.mapping.fromJS(args[attr]);
  1397. }
  1398. };
  1399. self.getArguments = function () {
  1400. return type ? type.args : [];
  1401. };
  1402. self.isCustomizable = function () {
  1403. return type.isCustomizable;
  1404. };
  1405. init();
  1406. };
  1407. var Source = function (vm, wizard) {
  1408. var self = this;
  1409. self.sourceType = vm.sourceType;
  1410. self.name = ko.observable('');
  1411. self.sample = ko.observableArray();
  1412. self.sampleCols = ko.observableArray();
  1413. self.namespace = wizard.namespace;
  1414. self.compute = wizard.compute;
  1415. var refreshThrottle = -1;
  1416. var sampleColSubDisposals = [];
  1417. var lastStatement = '';
  1418. var refreshTemporaryTable = function (sampleCols) {
  1419. window.clearTimeout(refreshThrottle);
  1420. window.setTimeout(function () {
  1421. while (sampleColSubDisposals.length) {
  1422. sampleColSubDisposals.pop()();
  1423. }
  1424. if (!self.sampleCols().length) {
  1425. return;
  1426. }
  1427. var tableName = 'input';
  1428. switch (self.inputFormat()) {
  1429. case 'stream':
  1430. if (self.streamSelection() === 'kafka' && self.kafkaSelectedTopics()) {
  1431. tableName = self.kafkaSelectedTopics();
  1432. }
  1433. break;
  1434. case 'file' :
  1435. tableName = self.path().split('/').pop();
  1436. break;
  1437. case 'table':
  1438. case 'rdbms':
  1439. tableName = self.tableName();
  1440. }
  1441. var statementCols = [];
  1442. var temporaryColumns = [];
  1443. sampleCols.forEach(function (sampleCol) {
  1444. statementCols.push(sqlUtils.backTickIfNeeded(self.sourceType, sampleCol.name()));
  1445. var col = {
  1446. name: sampleCol.name(),
  1447. type: sampleCol.type()
  1448. };
  1449. temporaryColumns.push(col);
  1450. var colNameSub = sampleCol.name.subscribe(function () {
  1451. refreshTemporaryTable(self.sampleCols())
  1452. });
  1453. var colTypeSub = sampleCol.type.subscribe(function () {
  1454. refreshTemporaryTable(self.sampleCols())
  1455. });
  1456. sampleColSubDisposals.push(function () {
  1457. colNameSub.dispose();
  1458. colTypeSub.dispose();
  1459. })
  1460. });
  1461. var statement = 'SELECT ';
  1462. statement += statementCols.join(',\n ');
  1463. statement += '\n FROM ' + sqlUtils.backTickIfNeeded(self.sourceType, tableName) + ';';
  1464. if (!wizard.destination.fieldEditorValue() || wizard.destination.fieldEditorValue() === lastStatement) {
  1465. wizard.destination.fieldEditorValue(statement);
  1466. }
  1467. lastStatement = statement;
  1468. wizard.destination.fieldEditorPlaceHolder('${ _('Example: SELECT') }' + ' * FROM ' + sqlUtils.backTickIfNeeded(self.sourceType, tableName));
  1469. var handle = dataCatalog.addTemporaryTable({
  1470. namespace: self.namespace(),
  1471. compute: self.compute(),
  1472. connector: { id: self.sourceType }, // TODO: Migrate importer to connectors
  1473. name: tableName,
  1474. columns: temporaryColumns,
  1475. sample: self.sample()
  1476. });
  1477. sampleColSubDisposals.push(function () {
  1478. handle.delete();
  1479. })
  1480. }, 500)
  1481. };
  1482. self.sampleCols.subscribe(refreshTemporaryTable);
  1483. self.inputFormat = ko.observable(wizard.prefill.source_type() ? wizard.prefill.source_type() : 'file');
  1484. self.inputFormat.subscribe(function(val) {
  1485. wizard.destination.columns.removeAll();
  1486. if (self.sample()) {
  1487. self.sample.removeAll();
  1488. }
  1489. self.path('');
  1490. resizeElements();
  1491. if (val === 'stream') {
  1492. if (self.streamSelection() === 'kafka') {
  1493. wizard.guessFormat();
  1494. wizard.destination.tableFormat('kudu');
  1495. } else {
  1496. wizard.destination.tableFormat('text');
  1497. }
  1498. } else if (val === 'table') {
  1499. wizard.destination.outputFormat('altus');
  1500. } else if (val === 'rdbms') {
  1501. self.rdbmsMode('customRdbms');
  1502. }
  1503. });
  1504. self.inputFormatsAll = ko.observableArray([
  1505. % if request.fs:
  1506. {'value': 'file', 'name': 'File'},
  1507. % endif
  1508. % if ENABLE_SQOOP.get():
  1509. {'value': 'rdbms', 'name': 'External Database'},
  1510. % endif
  1511. % if ENABLE_KAFKA.get():
  1512. {'value': 'stream', 'name': 'Stream'},
  1513. % endif
  1514. % if ENABLE_ALTUS.get():
  1515. {'value': 'table', 'name': 'Table'},
  1516. % endif
  1517. % if ENABLE_SQL_INDEXER.get():
  1518. {'value': 'query', 'name': 'SQL Query'},
  1519. % endif
  1520. ## TODO: Rename to 'Connectors'
  1521. % if ENABLE_ENVELOPE.get():
  1522. {'value': 'connector', 'name': 'Connectors'},
  1523. % endif
  1524. {'value': 'manual', 'name': 'Manually'}
  1525. ##{'value': 'text', 'name': 'Paste Text'},
  1526. ]);
  1527. self.inputFormatsManual = ko.observableArray([
  1528. {'value': 'manual', 'name': 'Manually'}
  1529. ]);
  1530. self.inputFormats = ko.pureComputed(function() {
  1531. if (wizard.prefill.source_type() === 'manual') {
  1532. return self.inputFormatsManual();
  1533. }
  1534. return self.inputFormatsAll();
  1535. });
  1536. // File
  1537. self.path = ko.observable('');
  1538. self.path.subscribe(function(val) {
  1539. if (val) {
  1540. wizard.guessFormat();
  1541. wizard.destination.nonDefaultLocation(val);
  1542. }
  1543. resizeElements();
  1544. });
  1545. self.isObjectStore = ko.pureComputed(function() {
  1546. return self.inputFormat() === 'file' && /^(s3a|adl|abfs):\/.*$/.test(self.path());
  1547. });
  1548. self.isObjectStore.subscribe(function(newVal) {
  1549. wizard.destination.useDefaultLocation(!newVal);
  1550. });
  1551. // Rdbms
  1552. self.rdbmsMode = ko.observable();
  1553. self.rdbmsMode.subscribe(function (val) {
  1554. self.rdbmsTypes(null);
  1555. self.rdbmsType('');
  1556. self.rdbmsDatabaseName('');
  1557. self.rdbmsIsAllTables(false);
  1558. self.rdbmsJdbcDriver('');
  1559. self.rdbmsJdbcDriverName('');
  1560. self.rdbmsHostname('');
  1561. self.rdbmsPort('');
  1562. self.rdbmsUsername('');
  1563. self.rdbmsPassword('');
  1564. self.rdbmsDbIsValid(false);
  1565. if (val === 'configRdbms') {
  1566. $.post("${ url('indexer:get_drivers') }", {}, function (resp) {
  1567. if (resp.data) {
  1568. self.rdbmsTypes(resp.data);
  1569. window.setTimeout(function(){
  1570. self.rdbmsType(self.rdbmsTypes()[0].value);
  1571. }, 0);
  1572. }
  1573. });
  1574. } else if (val === 'customRdbms') {
  1575. self.rdbmsTypes([
  1576. {'value': 'jdbc', 'name': 'JDBC'},
  1577. {'value': 'mysql', 'name': 'MySQL'},
  1578. {'value': 'oracle', 'name': 'Oracle'},
  1579. {'value': 'postgresql', 'name': 'PostgreSQL'}
  1580. ]);
  1581. window.setTimeout(function(){
  1582. self.rdbmsType(self.rdbmsTypes()[0].value);
  1583. }, 0);
  1584. }
  1585. });
  1586. self.rdbmsTypes = ko.observableArray();
  1587. self.rdbmsType = ko.observable(null);
  1588. self.rdbmsType.subscribe(function (val) {
  1589. self.path('');
  1590. self.rdbmsDatabaseNames([]);
  1591. self.rdbmsDatabaseName('');
  1592. resizeElements();
  1593. if(self.rdbmsMode() === 'configRdbms' && val !== 'jdbc') {
  1594. self.isFetchingDatabaseNames(true);
  1595. $.post("${ url('indexer:get_db_component') }", {
  1596. "source": ko.mapping.toJSON(self)
  1597. }, function (resp) {
  1598. if (resp.data) {
  1599. self.rdbmsDatabaseNames(resp.data);
  1600. }
  1601. }).always(function(){
  1602. self.isFetchingDatabaseNames(false);
  1603. });
  1604. } else if(self.rdbmsMode() === 'configRdbms' && val === 'jdbc') {
  1605. self.isFetchingDriverNames(true);
  1606. $.post("${ url('indexer:jdbc_db_list') }", {
  1607. "source": ko.mapping.toJSON(self)
  1608. }, function (resp) {
  1609. if (resp.data) {
  1610. self.rdbmsJdbcDriverNames(resp.data);
  1611. }
  1612. }).always(function() {
  1613. self.isFetchingDriverNames(false);
  1614. });
  1615. }
  1616. });
  1617. self.rdbmsDatabaseName = ko.observable('');
  1618. self.rdbmsDatabaseName.subscribe(function (val) {
  1619. if (val !== '') {
  1620. self.isFetchingTableNames(true);
  1621. $.post("${ url('indexer:get_db_component') }", {
  1622. "source": ko.mapping.toJSON(self)
  1623. }, function (resp) {
  1624. if (resp.data) {
  1625. self.rdbmsTableNames(resp.data.map(function(opt) {
  1626. return ko.mapping.fromJS(opt);
  1627. }));
  1628. }
  1629. }).always(function(){
  1630. self.isFetchingTableNames(false);
  1631. });
  1632. }
  1633. });
  1634. self.rdbmsDatabaseNames = ko.observableArray();
  1635. self.isFetchingDatabaseNames = ko.observable(false);
  1636. self.rdbmsJdbcDriverNames = ko.observableArray();
  1637. self.isFetchingDriverNames = ko.observable(false);
  1638. self.rdbmsJdbcDriverName = ko.observable();
  1639. self.rdbmsJdbcDriverName.subscribe(function () {
  1640. self.rdbmsDatabaseNames([]);
  1641. self.rdbmsDatabaseName('');
  1642. self.isFetchingDatabaseNames(true);
  1643. $.post("${ url('indexer:get_db_component') }", {
  1644. "source": ko.mapping.toJSON(self)
  1645. }, function (resp) {
  1646. if (resp.data) {
  1647. self.rdbmsDatabaseNames(resp.data);
  1648. }
  1649. }).always(function(){
  1650. self.isFetchingDatabaseNames(false);
  1651. });
  1652. });
  1653. self.rdbmsJdbcDriver = ko.observable('');
  1654. self.rdbmsJdbcDriver.subscribe(function () {
  1655. self.rdbmsDatabaseNames([]);
  1656. });
  1657. self.isFetchingTableNames = ko.observable(false);
  1658. self.rdbmsTableNames = ko.observableArray();
  1659. self.rdbmsHostname = ko.observable('');
  1660. self.rdbmsHostname.subscribe(function () {
  1661. self.rdbmsDatabaseNames([]);
  1662. });
  1663. self.rdbmsPort = ko.observable('');
  1664. self.rdbmsPort.subscribe(function () {
  1665. self.rdbmsDatabaseNames([]);
  1666. });
  1667. self.rdbmsUsername = ko.observable('');
  1668. self.rdbmsUsername.subscribe(function () {
  1669. self.rdbmsDatabaseNames([]);
  1670. });
  1671. self.rdbmsPassword = ko.observable('');
  1672. self.rdbmsPassword.subscribe(function () {
  1673. self.rdbmsDatabaseNames([]);
  1674. });
  1675. self.rdbmsIsAllTables = ko.observable(false);
  1676. self.rdbmsIsAllTables.subscribe(function(newVal) {
  1677. if (newVal) {
  1678. wizard.destination.name(self.rdbmsDatabaseName());
  1679. } else {
  1680. wizard.destination.name(self.tables()[0] && self.tables()[0].name() || '');
  1681. }
  1682. });
  1683. self.rdbmsAllTablesSelected = ko.pureComputed(function () {
  1684. return self.rdbmsIsAllTables() || self.tables().length > 1;
  1685. });
  1686. self.rdbmsTablesExclude = ko.pureComputed(function () {
  1687. var rdbmsTables = self.rdbmsTableNames();
  1688. var tables = self.tables();
  1689. if (tables.length <= 1) {
  1690. return [];
  1691. }
  1692. var map = tables.reduce(function (map, table) {
  1693. map[table.name()] = 1;
  1694. return map;
  1695. }, {});
  1696. return rdbmsTables.map(function (table) {
  1697. return table.name();
  1698. }).filter(function (name) {
  1699. return !map[name];
  1700. });
  1701. });
  1702. self.rdbmsDbIsValid = ko.observable(false);
  1703. self.rdbmsCheckConnection = function() {
  1704. self.isFetchingDatabaseNames(true);
  1705. self.rdbmsDatabaseNames([]);
  1706. self.rdbmsDatabaseName(''); // Need to clear or else get_db_component will return list of tables
  1707. $.post("${ url('indexer:get_db_component') }", {
  1708. "source": ko.mapping.toJSON(self)
  1709. }, function (resp) {
  1710. if (resp.status === 0 && resp.data) {
  1711. self.rdbmsDbIsValid(true);
  1712. self.rdbmsDatabaseNames(resp.data);
  1713. } else if (resp.status === 1) {
  1714. $(document).trigger("error", "${ _('Connection Failed: ') }" + resp.message);
  1715. self.rdbmsDbIsValid(false);
  1716. }
  1717. }).always(function(){
  1718. self.isFetchingDatabaseNames(false);
  1719. });
  1720. };
  1721. // Table
  1722. self.tables = ko.observableArray([
  1723. ko.mapping.fromJS({ name: '' })
  1724. ]);
  1725. self.tables.subscribe(function(newVal) {
  1726. if (newVal.length == 0) {
  1727. wizard.destination.name('');
  1728. } else if (newVal.length == 1) {
  1729. wizard.destination.name(self.tables()[0].name() || ''); // TODO: db prefix and check if DB exists
  1730. } else {
  1731. wizard.destination.name(self.rdbmsDatabaseName());
  1732. }
  1733. });
  1734. self.tablesNames = ko.observableArray([]);
  1735. self.selectedTableIndex = ko.observable(0);
  1736. self.table = ko.pureComputed(function() {
  1737. var index = Math.min(self.selectedTableIndex(), self.tables().length - 1);
  1738. return self.tables().length > 0 ? self.tables()[index].name() : ''
  1739. });
  1740. self.tableName = ko.pureComputed(function() {
  1741. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[1] : self.table();
  1742. });
  1743. self.databaseName = ko.pureComputed(function() {
  1744. return self.table().indexOf('.') > 0 ? self.table().split('.', 2)[0] : 'default';
  1745. });
  1746. self.table.subscribe(function(val) {
  1747. if (val) {
  1748. wizard.guessFormat();
  1749. wizard.destination.nonDefaultLocation(val);
  1750. }
  1751. resizeElements();
  1752. });
  1753. // Queries
  1754. self.query = ko.observable('');
  1755. self.query.subscribe(function(newValue) {
  1756. if (newValue) {
  1757. viewModel.createWizard.guessFieldTypes();
  1758. }
  1759. });
  1760. self.draggedQuery = ko.observable();
  1761. // Connectors
  1762. self.connectorList = ko.observable([
  1763. {'value': 'sfdc', 'name': 'Salesforce'}
  1764. ]);
  1765. self.connectorSelection = ko.observable(self.connectorList()[0]['value']);
  1766. // Streams, Kafka, Flume
  1767. self.publicStreams = ko.observable([
  1768. {'value': 'kafka', 'name': 'Kafka Topics'},
  1769. {'value': 'flume', 'name': 'Flume Agent'}
  1770. ]);
  1771. self.streamSelection = ko.observable(self.publicStreams()[0]['value']);
  1772. self.streamSelection.subscribe(function(newValue) {
  1773. if (newValue == 'flume') {
  1774. $.post("${ url('metadata:manager_hosts') }", {
  1775. "service": "flume"
  1776. }, function (resp) {
  1777. if (resp.status === 0 && resp.hosts) {
  1778. self.channelSourceHosts(resp.hosts);
  1779. } else {
  1780. $(document).trigger("error", "${ _('Error getting hosts') }" + resp.message);
  1781. }
  1782. });
  1783. }
  1784. });
  1785. self.kafkaTopics = ko.observableArray();
  1786. self.kafkaSelectedTopics = ko.observable(''); // Currently designed just for one
  1787. self.kafkaSelectedTopics.subscribe(function(newValue) {
  1788. if (newValue) {
  1789. viewModel.createWizard.guessFieldTypes();
  1790. self.kafkaFieldNames($.totalStorage('pai' + '_kafka_topics_' + newValue + '_kafkaFieldNames'));
  1791. self.kafkaFieldTypes($.totalStorage('pai' + '_kafka_topics_' + newValue + '_kafkaFieldTypes'));
  1792. }
  1793. });
  1794. self.kafkaSchemaManual = ko.observable('manual');
  1795. self.kafkaFieldType = ko.observable('delimited');
  1796. self.kafkaFieldDelimiter = ko.observable(',');
  1797. self.kafkaFieldNames = ko.observable('');
  1798. self.kafkaFieldNames.subscribe(function(newValue) {
  1799. $.totalStorage('pai' + '_kafka_topics_' + self.kafkaSelectedTopics() + '_kafkaFieldNames', newValue);
  1800. viewModel.createWizard.guessFieldTypes();
  1801. });
  1802. self.kafkaFieldTypes = ko.observable('');
  1803. self.kafkaFieldTypes.subscribe(function(newValue) {
  1804. $.totalStorage('pai' + '_kafka_topics_' + self.kafkaSelectedTopics() + '_kafkaFieldTypes', newValue);
  1805. viewModel.createWizard.guessFieldTypes();
  1806. });
  1807. self.kafkaFieldSchemaPath = ko.observable('');
  1808. self.channelSourceTypes = ko.observableArray([
  1809. {'name': '${ _("Directory or File") }', 'value': 'directory'},
  1810. {'name': '${ _("Program") }', 'value': 'exec'},
  1811. {'name': '${ _("Syslogs") }', 'value': 'syslogs'},
  1812. {'name': '${ _("HTTP") }', 'value': 'http'}
  1813. ]);
  1814. self.channelSourceType = ko.observable();
  1815. self.channelSourceHosts = ko.observableArray();
  1816. self.channelSourceSelectedHosts = ko.observableArray([]);
  1817. self.channelSourceSelectedHosts.subscribe(function(newVal) {
  1818. if (newVal) {
  1819. viewModel.createWizard.guessFieldTypes();
  1820. }
  1821. })
  1822. self.channelSourcePath = ko.observable('/var/log/hue-httpd/access_log');
  1823. self.streamUsername = ko.observable('');
  1824. self.streamPassword = ko.observable('');
  1825. self.streamToken = ko.observable('');
  1826. self.streamToken.subscribe(function(newVal) {
  1827. if (newVal) {
  1828. wizard.guessFormat(); // Todo
  1829. }
  1830. });
  1831. self.streamEndpointUrl = ko.observable('https://login.salesforce.com/services/Soap/u/42.0');
  1832. self.streamObjects = ko.observableArray();
  1833. self.streamObject = ko.observable();
  1834. self.streamObject.subscribe(function(newValue) {
  1835. if (newValue) {
  1836. wizard.guessFieldTypes();
  1837. }
  1838. });
  1839. self.hasStreamSelected = ko.pureComputed(function() {
  1840. return (self.streamSelection() === 'kafka' && self.kafkaSelectedTopics()) ||
  1841. (self.streamSelection() === 'sfdc' && self.streamObject())
  1842. });
  1843. self.hasStreamSelected.subscribe(function(newValue) {
  1844. if (newValue) {
  1845. wizard.guessFormat();
  1846. if (self.streamSelection() === 'kafka') {
  1847. wizard.destination.tableFormat('kudu');
  1848. } else {
  1849. wizard.destination.tableFormat('text');
  1850. }
  1851. }
  1852. });
  1853. self.streamCheckConnection = function() {
  1854. $(".jHueNotify").remove();
  1855. $.post("${ url('indexer:get_db_component') }", {
  1856. "source": ko.mapping.toJSON(self)
  1857. }, function (resp) {
  1858. if (resp.status === 0 && resp.data) {
  1859. huePubSub.publish('notebook.task.submitted', resp);
  1860. } else if (resp.status === 1) {
  1861. $(document).trigger("error", "${ _('Connection Failed: ') }" + resp.message);
  1862. self.rdbmsDbIsValid(false);
  1863. }
  1864. });
  1865. };
  1866. self.format = ko.observable();
  1867. self.format.subscribe(function(newVal) {
  1868. if (typeof newVal.hasHeader !== 'undefined') {
  1869. wizard.destination.hasHeader(newVal.hasHeader());
  1870. newVal.hasHeader.subscribe(function(newVal) {
  1871. wizard.destination.hasHeader(newVal);
  1872. });
  1873. }
  1874. if (typeof newVal.fieldSeparator !== 'undefined') {
  1875. wizard.destination.useCustomDelimiters(newVal.fieldSeparator() !== ',');
  1876. wizard.destination.customFieldDelimiter(newVal.fieldSeparator());
  1877. newVal.fieldSeparator.subscribe(function(newVal) {
  1878. if (newVal !== '') {
  1879. wizard.destination.customFieldDelimiter(newVal);
  1880. }
  1881. });
  1882. }
  1883. });
  1884. self.show = ko.pureComputed(function() {
  1885. if (self.inputFormat() === 'file') {
  1886. return self.path().length > 0;
  1887. }
  1888. if (self.inputFormat() === 'table') {
  1889. return self.tableName().length > 0;
  1890. }
  1891. if (self.inputFormat() === 'query') {
  1892. return self.query();
  1893. }
  1894. if (self.inputFormat() === 'manual') {
  1895. return true;
  1896. }
  1897. if (self.inputFormat() === 'stream') {
  1898. if (self.streamSelection() === 'kafka') {
  1899. return self.kafkaSelectedTopics() && self.kafkaSelectedTopics().length > 0;
  1900. } else if (self.streamSelection() === 'flume') {
  1901. return self.channelSourceSelectedHosts().length > 0;
  1902. }
  1903. }
  1904. if (self.inputFormat() === 'sfdc') {
  1905. return self.streamUsername().length > 0 &&
  1906. self.streamPassword().length > 0 &&
  1907. self.streamToken().length > 0 &&
  1908. self.streamEndpointUrl().length > 0 &&
  1909. self.streamObject();
  1910. }
  1911. if (self.inputFormat() === 'rdbms') {
  1912. return (self.rdbmsDatabaseName().length > 0 && self.tables().length > 0) || self.rdbmsAllTablesSelected();
  1913. }
  1914. });
  1915. };
  1916. var Destination = function (vm, wizard) {
  1917. var self = this;
  1918. self.apiHelperType = self.sourceType = vm.sourceType;
  1919. self.name = ko.observable('').extend({ throttle: 500 });
  1920. self.nameChanged = function(name) {
  1921. var exists = false;
  1922. var checkDbEntryExists = function () {
  1923. wizard.computeSetDeferred.done(function () {
  1924. dataCatalog.getEntry({
  1925. compute: wizard.compute(),
  1926. connector: { id: self.sourceType }, // TODO: Use connectors in the importer
  1927. namespace: wizard.namespace(),
  1928. path: self.outputFormat() === 'table' ? [self.databaseName(), self.tableName()] : [],
  1929. }).done(function (catalogEntry) {
  1930. catalogEntry.getSourceMeta({ silenceErrors: true }).done(function (sourceMeta) {
  1931. self.isTargetExisting(self.outputFormat() === 'table' ? !sourceMeta.notFound : (sourceMeta.databases || []).indexOf(self.databaseName()) >= 0);
  1932. self.isTargetChecking(false);
  1933. }).fail(function () {
  1934. self.isTargetExisting(false);
  1935. self.isTargetChecking(false);
  1936. })
  1937. });
  1938. })
  1939. };
  1940. if (name.length === 0) {
  1941. self.isTargetExisting(false);
  1942. self.isTargetChecking(false);
  1943. } else if (self.outputFormat() === 'file') {
  1944. // self.path()
  1945. } else if (self.outputFormat() === 'table' && wizard.isValidDestination()) {
  1946. if (self.tableName() !== '') {
  1947. self.isTargetExisting(false);
  1948. self.isTargetChecking(true);
  1949. checkDbEntryExists();
  1950. } else {
  1951. self.isTargetExisting(false);
  1952. self.isTargetChecking(false);
  1953. }
  1954. } else if (self.outputFormat() === 'hbase') {
  1955. // Todo once autocomplete is implemented for hbase
  1956. } else if (self.outputFormat() === 'database' && wizard.isValidDestination()) {
  1957. if (self.databaseName() !== '') {
  1958. self.isTargetExisting(false);
  1959. self.isTargetChecking(true);
  1960. checkDbEntryExists();
  1961. } else {
  1962. self.isTargetExisting(false);
  1963. self.isTargetChecking(false);
  1964. }
  1965. } else if (self.outputFormat() === 'index') {
  1966. $.post("${ url('indexer:list_index') }", {
  1967. name: self.name()
  1968. }, function (data) {
  1969. self.isTargetExisting(data.status === 0);
  1970. self.isTargetChecking(false);
  1971. }).fail(function (xhr, textStatus, errorThrown) {
  1972. self.isTargetExisting(false);
  1973. self.isTargetChecking(false);
  1974. });
  1975. $.post("/indexer/api/configs/list/", function (data) {
  1976. if (data.status === 0) {
  1977. self.indexerConfigSets(data.configs.map(function (config) {
  1978. return {'name': config, 'value': config};
  1979. }));
  1980. } else {
  1981. $(document).trigger("error", data.message);
  1982. }
  1983. });
  1984. }
  1985. resizeElements();
  1986. };
  1987. self.name.subscribe(self.nameChanged);
  1988. self.description = ko.observable('');
  1989. self.outputFormat = ko.observable(wizard.prefill.target_type() || 'table');
  1990. self.outputFormat.subscribe(function (newValue) {
  1991. if (newValue && newValue !== 'database' && ((['table', 'rdbms', 'index'].indexOf(newValue) >= 0) && wizard.source.table().length > 0)) {
  1992. self.nameChanged(self.name());
  1993. wizard.guessFieldTypes();
  1994. resizeElements();
  1995. }
  1996. });
  1997. self.outputFormatsList = ko.observableArray([
  1998. {'name': '${ _("Table") }', 'value': 'table'},
  1999. % if ENABLE_NEW_INDEXER.get():
  2000. {'name': '${ _("Search index") }', 'value': 'index'},
  2001. % endif
  2002. {'name': '${ _("Database") }', 'value': 'database'},
  2003. % if ENABLE_SQOOP.get() or ENABLE_KAFKA.get():
  2004. {'name': '${ _("Folder") }', 'value': 'file'},
  2005. % endif
  2006. % if ENABLE_KAFKA.get():
  2007. {'name': '${ _("Stream") }', 'value': 'stream'},
  2008. % endif
  2009. % if ENABLE_ALTUS.get():
  2010. {'name': '${ _("Altus SDX") }', 'value': 'altus'},
  2011. % endif
  2012. % if ENABLE_SQOOP.get():
  2013. {'name': '${ _("HBase Table") }', 'value': 'hbase'},
  2014. % endif
  2015. ]);
  2016. self.outputFormats = ko.pureComputed(function() {
  2017. return $.grep(self.outputFormatsList(), function(format) {
  2018. if (format.value === 'database' && wizard.source.inputFormat() !== 'manual' && (wizard.source.inputFormat() !== 'rdbms' || !wizard.source.rdbmsAllTablesSelected())) {
  2019. return false;
  2020. }
  2021. if (format.value === 'file' && ['manual', 'rdbms', 'stream'].indexOf(wizard.source.inputFormat()) === -1) {
  2022. return false;
  2023. }
  2024. if (format.value === 'index' && ['file', 'query', 'stream', 'manual'].indexOf(wizard.source.inputFormat()) === -1) {
  2025. return false;
  2026. }
  2027. if (format.value === 'table' && (wizard.source.inputFormat() === 'table' || (wizard.source.inputFormat() === 'rdbms' && wizard.source.rdbmsAllTablesSelected()))) {
  2028. return false;
  2029. }
  2030. if (format.value === 'altus' && ['table'].indexOf(wizard.source.inputFormat()) === -1) {
  2031. return false;
  2032. }
  2033. if (format.value === 'stream' && ['file', 'stream'].indexOf(wizard.source.inputFormat()) === -1) {
  2034. return false;
  2035. }
  2036. if (format.value === 'hbase' && (wizard.source.inputFormat() !== 'rdbms' || wizard.source.rdbmsAllTablesSelected())) {
  2037. return false;
  2038. }
  2039. return true;
  2040. })
  2041. });
  2042. self.outputFormats.subscribe(function (newValue) {
  2043. if (newValue.length && newValue.map(function (entry) { return entry.value; }).indexOf(self.outputFormat()) < 0) {
  2044. self.outputFormat(newValue && newValue[0].value);
  2045. }
  2046. });
  2047. wizard.prefill.target_type.subscribe(function(newValue) {
  2048. setTimeout(function() { // target_type gets updated by the router (onePageViewModelModel.js) and delaying allow the notification to go through
  2049. self.outputFormat(newValue || 'table');
  2050. },0);
  2051. if (newValue === 'database') {
  2052. vm.currentStep(2);
  2053. } else {
  2054. vm.currentStep(1);
  2055. }
  2056. });
  2057. self.defaultName = ko.pureComputed(function() {
  2058. var name = '';
  2059. if (wizard.source.inputFormat() === 'file' || wizard.source.inputFormat() === 'stream') {
  2060. if (self.outputFormat() === 'table') {
  2061. name = wizard.prefill.target_path().length > 0 ? wizard.prefill.target_path() : 'default';
  2062. if (wizard.source.inputFormat() === 'stream') {
  2063. if (wizard.source.streamSelection() === 'kafka') {
  2064. name += '.' + wizard.source.kafkaSelectedTopics();
  2065. } else {
  2066. name += '.' + wizard.source.streamObject();
  2067. }
  2068. } else if (wizard.source.path()) {
  2069. name += '.' + wizard.source.path().split('/').pop().split('.')[0];
  2070. }
  2071. } else { // Index
  2072. name = wizard.prefill.target_path().length > 0 ? wizard.prefill.target_path() : wizard.source.path().split('/').pop().split('.')[0];
  2073. }
  2074. } else if (wizard.source.inputFormat() === 'table') {
  2075. name = wizard.source.databaseName();
  2076. } else if (wizard.source.inputFormat() === 'query') {
  2077. if (wizard.source.query()) {
  2078. name = wizard.source.name();
  2079. }
  2080. if (wizard.prefill.target_path().length > 0) {
  2081. name = wizard.prefill.target_path();
  2082. }
  2083. } else if (wizard.source.inputFormat() === 'manual') {
  2084. name = wizard.prefill.target_path().length > 0 ? wizard.prefill.target_path() + '.' : '';
  2085. }
  2086. return name.replace(/ /g, '_').toLowerCase();
  2087. });
  2088. self.defaultName.subscribe(function(newVal) {
  2089. self.name(newVal);
  2090. });
  2091. self.format = ko.observable();
  2092. self.columns = ko.observableArray();
  2093. self.namespaces = wizard.namespaces;
  2094. self.namespace = wizard.namespace;
  2095. self.compute = wizard.compute;
  2096. self.targetNamespaceId = ko.observable(self.namespace() ? self.namespace().id : undefined);
  2097. self.namespace.subscribe(function (namespace) {
  2098. if (namespace && namespace.id !== self.targetNamespaceId) {
  2099. self.targetNamespaceId(namespace.id);
  2100. }
  2101. })
  2102. self.targetNamespaceId.subscribe(function (namespaceId) {
  2103. if (namespaceId && (!self.namespace() || self.namespace().id !== namespaceId)) {
  2104. self.namespaces().some(function (namespace) {
  2105. if (namespaceId === namespace.id) {
  2106. self.namespace(namespace);
  2107. return true;
  2108. }
  2109. })
  2110. }
  2111. });
  2112. // UI
  2113. self.bulkColumnNames = ko.observable('');
  2114. self.showProperties = ko.observable(false);
  2115. self.columns.subscribe(function (newVal) {
  2116. self.bulkColumnNames(newVal.map(function (item) {
  2117. return item.name()
  2118. }).join(','));
  2119. });
  2120. self.processBulkColumnNames = function () {
  2121. var val = self.bulkColumnNames();
  2122. try {
  2123. if (val.indexOf("\"") === -1 && val.indexOf("'") === -1) {
  2124. val = '"' + val.replace(/,/gi, '","') + '"';
  2125. }
  2126. if (val.indexOf("[") === -1) {
  2127. val = "[" + val + "]";
  2128. }
  2129. var parsed = JSON.parse(val);
  2130. self.columns().forEach(function (item, cnt) {
  2131. if ($.trim(parsed[cnt]) !== '') {
  2132. item.name($.trim(parsed[cnt]));
  2133. }
  2134. });
  2135. }
  2136. catch (err) {
  2137. }
  2138. $('#fieldsBulkEditor').modal('hide');
  2139. };
  2140. self.isTargetExisting = ko.observable();
  2141. self.isTargetChecking = ko.observable(false);
  2142. self.existingTargetUrl = ko.pureComputed(function() { // Should open generic sample popup instead
  2143. if (self.isTargetExisting()) {
  2144. if (self.outputFormat() === 'file') {
  2145. // Todo
  2146. return '';
  2147. }
  2148. if (self.outputFormat() === 'table') {
  2149. return '/metastore/table/' + self.databaseName() + '/' + self.tableName();
  2150. }
  2151. if (self.outputFormat() === 'database') {
  2152. return '/metastore/tables/' + self.databaseName();
  2153. }
  2154. if (self.outputFormat() === 'index') {
  2155. return '/indexer/indexes/' + self.name();
  2156. }
  2157. }
  2158. return '';
  2159. });
  2160. // Table
  2161. self.tableName = ko.pureComputed(function() {
  2162. return self.outputFormat() === 'table' && self.name().indexOf('.') > 0 ? self.name().split('.', 2)[1] : self.name();
  2163. });
  2164. self.databaseName = ko.pureComputed(function() {
  2165. return self.outputFormat() === 'database' ? self.name() : (self.outputFormat() === 'table' && self.name().indexOf('.') > 0 ? self.name().split('.', 2)[0] : 'default');
  2166. });
  2167. self.tableFormat = ko.observable('text');
  2168. self.KUDU_DEFAULT_RANGE_PARTITION_COLUMN = {values: [{value: ''}], name: 'VALUES', lower_val: 0, include_lower_val: '<=', upper_val: 1, include_upper_val: '<='};
  2169. self.KUDU_DEFAULT_PARTITION_COLUMN = {columns: [], range_partitions: [self.KUDU_DEFAULT_RANGE_PARTITION_COLUMN], name: 'HASH', int_val: 16};
  2170. self.tableFormats = ko.pureComputed(function() {
  2171. if (wizard.source.inputFormat() === 'kafka') {
  2172. return [{'value': 'kudu', 'name': 'Kudu'}];
  2173. } else if (vm.sourceType == 'impala') { // Impala supports Kudu
  2174. return [
  2175. {'value': 'text', 'name': 'Text'},
  2176. {'value': 'parquet', 'name': 'Parquet'},
  2177. {'value': 'kudu', 'name': 'Kudu'},
  2178. {'value': 'csv', 'name': 'Csv'},
  2179. {'value': 'avro', 'name': 'Avro'},
  2180. {'value': 'json', 'name': 'Json'},
  2181. {'value': 'regexp', 'name': 'Regexp'},
  2182. {'value': 'orc', 'name': 'ORC'},
  2183. ];
  2184. }
  2185. return [
  2186. {'value': 'text', 'name': 'Text'},
  2187. {'value': 'parquet', 'name': 'Parquet'},
  2188. {'value': 'csv', 'name': 'Csv'},
  2189. {'value': 'avro', 'name': 'Avro'},
  2190. {'value': 'json', 'name': 'Json'},
  2191. {'value': 'regexp', 'name': 'Regexp'},
  2192. {'value': 'orc', 'name': 'ORC'},
  2193. ]
  2194. });
  2195. self.partitionColumns = ko.observableArray();
  2196. self.kuduPartitionColumns = ko.observableArray();
  2197. self.primaryKeys = ko.observableArray();
  2198. self.primaryKeyObjects = ko.observableArray();
  2199. self.useFieldEditor = ko.observable(false);
  2200. // TODO: Figure out the database to use for field editor autocomplete
  2201. self.fieldEditorDatabase = ko.observable('default');
  2202. // TODO: Do something with the editor value
  2203. self.fieldEditorValue = ko.observable();
  2204. self.fieldEditorPlaceHolder = ko.observable();
  2205. self.fieldEditorEnabled = ko.pureComputed(function () {
  2206. return '${ ENABLE_FIELD_EDITOR.get() }' == 'True';
  2207. });
  2208. self.importData = ko.observable(true);
  2209. self.useDefaultLocation = ko.observable(true);
  2210. self.nonDefaultLocation = ko.observable('');
  2211. var isTransactionalVisibleImpala = '${ impala_flags.is_transactional() }'.toLowerCase() == 'true';
  2212. var isTransactionalVisibleHive = '${ hive_site.has_concurrency_support() }'.toLowerCase() == 'true';
  2213. var transactionalDefaultType = '${ impala_flags.default_transactional_type() }'.toLowerCase();
  2214. self.isTransactionalVisible = ko.observable((vm.sourceType == 'impala' && isTransactionalVisibleImpala) || (vm.sourceType == 'hive' && isTransactionalVisibleHive));
  2215. self.isTransactional = ko.observable(self.isTransactionalVisible());
  2216. self.isInsertOnly = ko.observable(true); // Impala doesn't have yet full support.
  2217. self.isTransactionalUpdateEnabled = ko.pureComputed(function() {
  2218. var enabled = self.tableFormat() == 'orc' && (vm.sourceType == 'hive' || (vm.sourceType == 'impala' && transactionalDefaultType.length && transactionalDefaultType != 'insert_only'));
  2219. if (!enabled) {
  2220. self.isInsertOnly(true);
  2221. }
  2222. return enabled;
  2223. });
  2224. self.hasHeader = ko.observable(false);
  2225. self.useCustomDelimiters = ko.observable(false);
  2226. self.customFieldDelimiter = ko.observable(',');
  2227. self.customCollectionDelimiter = ko.observable('\\002');
  2228. self.customMapDelimiter = ko.observable('\\003');
  2229. self.customRegexp = ko.observable('');
  2230. // Index
  2231. self.indexerRunJob = ko.observable(false);
  2232. self.indexerJobLibPath = ko.observable('${ CONFIG_INDEXER_LIBS_PATH.get() }');
  2233. self.indexerConfigSet = ko.observable('');
  2234. self.indexerConfigSets = ko.observableArray([]);
  2235. self.indexerNumShards = ko.observable(1);
  2236. self.indexerReplicationFactor = ko.observable(1);
  2237. self.indexerPrimaryKey = ko.observableArray();
  2238. self.indexerPrimaryKeyObject = ko.observableArray();
  2239. self.indexerDefaultField = ko.observableArray();
  2240. self.indexerDefaultFieldObject = ko.observableArray();
  2241. // File, Table, HBase
  2242. self.sqoopJobLibPaths = ko.observableArray([]);
  2243. self.addSqoopJobLibPath = function() {
  2244. var newValue = {
  2245. path: ko.observable('')
  2246. };
  2247. self.sqoopJobLibPaths.push(newValue);
  2248. };
  2249. self.addSqoopJobLibPath();
  2250. self.removeSqoopJobLibPath = function (valueToRemove) {
  2251. self.sqoopJobLibPaths.remove(valueToRemove);
  2252. };
  2253. self.numMappers = ko.observable(1);
  2254. self.customFieldsDelimiter = ko.observable(',');
  2255. self.customLineDelimiter = ko.observable('\\n');
  2256. self.customEnclosedByDelimiter = ko.observable('\'');
  2257. self.rdbmsFileOutputFormat = ko.observable('text');
  2258. self.rdbmsFileOutputFormats = ko.observableArray([
  2259. {'value': 'text', 'name': 'text'},
  2260. {'value': 'sequence', 'name': 'sequence'},
  2261. {'value': 'avro', 'name': 'avro'}
  2262. ]);
  2263. self.rdbmsSplitByColumn = ko.observableArray();
  2264. // Flume
  2265. self.channelSinkTypes = ko.observableArray([
  2266. {'name': '${ _("This topic") }', 'value': 'kafka'},
  2267. {'name': '${ _("Solr") }', 'value': 'solr'},
  2268. {'name': '${ _("HDFS") }', 'value': 'hdfs'}
  2269. ]);
  2270. self.channelSinkType = ko.observable();
  2271. self.channelSinkPath = ko.observable();
  2272. };
  2273. var CreateWizard = function (vm) {
  2274. var self = this;
  2275. var guessFieldTypesXhr;
  2276. self.namespaces = ko.observableArray();
  2277. self.namespace = ko.observable();
  2278. self.compute = ko.observable();
  2279. self.computeSetDeferred = $.Deferred();
  2280. // TODO: Use connectors in the importer
  2281. contextCatalog.getNamespaces({ connector: { id: vm.sourceType } }).done(function (context) {
  2282. self.namespaces(context.namespaces);
  2283. if (!vm.namespaceId || !context.namespaces.some(function (namespace) {
  2284. if (namespace.id === vm.namespaceId) {
  2285. self.namespace(namespace);
  2286. return true;
  2287. }
  2288. })) {
  2289. self.namespace(context.namespaces[0]);
  2290. }
  2291. if (!vm.computeId || !self.namespace().computes.some(function (compute) {
  2292. if (compute.id === vm.computeId) {
  2293. self.compute(compute);
  2294. return true;
  2295. }
  2296. })) {
  2297. self.compute(self.namespace().computes[0]);
  2298. }
  2299. self.namespace.subscribe(function (namespace) {
  2300. if (!namespace) {
  2301. self.compute(undefined);
  2302. return;
  2303. }
  2304. if (!self.compute() || !namespace.computes.some(function (compute) {
  2305. if (compute.id === self.compute()) {
  2306. return true;
  2307. }
  2308. })) {
  2309. if (namespace.computes.length) {
  2310. self.compute(namespace.computes[0]);
  2311. }
  2312. }
  2313. })
  2314. self.computeSetDeferred.resolve();
  2315. });
  2316. self.fileType = ko.observable();
  2317. self.fileType.subscribe(function (newType) {
  2318. if (self.source.format()) {
  2319. self.source.format().type(newType.name);
  2320. }
  2321. });
  2322. self.fileTypeName = ko.observable();
  2323. self.fileTypeName.subscribe(function (newType) {
  2324. for (var i = 0; i < self.fileTypes.length; i++) {
  2325. if (self.fileTypes[i].name === newType) {
  2326. self.fileType(self.fileTypes[i]);
  2327. break;
  2328. }
  2329. }
  2330. });
  2331. self.operationTypes = ${operators_json | n};
  2332. self.fieldTypes = ${fields_json | n}.solr;
  2333. self.hivePrimitiveFieldTypes = ${fields_json | n}.hivePrimitive;
  2334. self.hiveFieldTypes = ${fields_json | n}.hive;
  2335. self.fileTypes = ${file_types_json | n};
  2336. self.prefill = ko.mapping.fromJS(${prefill | n});
  2337. self.prefill.source_type.subscribe(function(newValue) {
  2338. self.source.inputFormat(newValue ? newValue : 'file');
  2339. });
  2340. self.show = ko.observable(true);
  2341. self.showCreate = ko.observable(false);
  2342. self.source = new Source(vm, self);
  2343. self.destination = new Destination(vm, self);
  2344. self.customDelimiters = ko.observableArray([
  2345. {'value': ',', 'name': '${ _("Comma (,)") }'},
  2346. {'value': '\\t', 'name': '${ _("^Tab (\\t)") }'},
  2347. {'value': '\\n', 'name': '${ _("New line") }'},
  2348. {'value': '|', 'name': '${ _("Pipe") }'},
  2349. {'value': '\"', 'name': '${ _("Double Quote") }'},
  2350. {'value': '\'', 'name': '${ _("Single Quote") }'},
  2351. {'value': '\x00', 'name': '${ _("^0") }'},
  2352. {'value': '\x01', 'name': '${ _("^A (\\001)") }'},
  2353. {'value': '\x02', 'name': '${ _("^B (\\002)") }'},
  2354. {'value': '\x03', 'name': '${ _("^C (\\003)") }'},
  2355. {'value': '\x01', 'name': '${ _("^A (\\x01)") }'}
  2356. ]);
  2357. self.editorId = ko.observable();
  2358. self.jobId = ko.observable();
  2359. self.editorVM = null;
  2360. self.indexingStarted = ko.observable(false);
  2361. self.isValidDestination = ko.pureComputed(function() {
  2362. return self.destination.name().length > 0 && (
  2363. (['table', 'database'].indexOf(self.destination.outputFormat()) === -1 || /^([a-zA-Z0-9_]+\.)?[a-zA-Z0-9_]+$/.test(self.destination.name())) &&
  2364. (['index'].indexOf(self.destination.outputFormat()) === -1 || /^[^\\/:]+$/.test(self.destination.name()))
  2365. );
  2366. });
  2367. self.readyToIndex = ko.pureComputed(function () {
  2368. var validFields = self.destination.columns().length || self.destination.outputFormat() === 'database';
  2369. var validTableColumns = self.destination.outputFormat() !== 'table' || ($.grep(self.destination.columns(), function(column) {
  2370. return column.name().length === 0;
  2371. }).length === 0
  2372. && $.grep(self.destination.partitionColumns(), function(column) {
  2373. return column.name().length === 0 || (self.source.inputFormat() !== 'manual' && column.partitionValue().length === 0);
  2374. }).length === 0
  2375. );
  2376. var isTargetAlreadyExisting;
  2377. if (self.destination.outputFormat() === 'database' && self.source.inputFormat() === 'rdbms') {
  2378. isTargetAlreadyExisting = self.destination.isTargetExisting();
  2379. } else {
  2380. isTargetAlreadyExisting = !self.destination.isTargetExisting()
  2381. || self.destination.outputFormat() === 'index'
  2382. ## Authorize submission but should check if schema is compatible
  2383. || (self.source.inputFormat() === 'stream' && self.destination.outputFormat() === 'table' && self.destination.tableFormat() === 'kudu')
  2384. ;
  2385. }
  2386. var isValidTable = self.destination.outputFormat() !== 'table' || (
  2387. self.destination.tableFormat() !== 'kudu' || (
  2388. $.grep(self.destination.kuduPartitionColumns(), function(partition) {
  2389. return partition.columns().length > 0
  2390. }).length === self.destination.kuduPartitionColumns().length && self.destination.primaryKeys().length > 0
  2391. )
  2392. );
  2393. var validIndexFields = self.destination.outputFormat() !== 'index' || ($.grep(self.destination.columns(), function(column) {
  2394. return ! (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(column.name()) && column.name() !== '_version_');
  2395. }).length === 0
  2396. ) || self.destination.indexerConfigSet();
  2397. return self.isValidDestination() && validFields && validTableColumns && validIndexFields && isTargetAlreadyExisting && isValidTable;
  2398. });
  2399. self.formatTypeSubscribed = false;
  2400. self.source.format.subscribe(function () {
  2401. for (var i = 0; i < self.fileTypes.length; i++) {
  2402. if (self.fileTypes[i].name === self.source.format().type()) {
  2403. self.fileType(self.fileTypes[i]);
  2404. self.fileTypeName(self.fileTypes[i].name);
  2405. break;
  2406. }
  2407. }
  2408. if (self.source.format().type) {
  2409. if (! self.formatTypeSubscribed) {
  2410. self.formatTypeSubscribed = true;
  2411. self.source.format().type.subscribe(function (newType) {
  2412. self.source.format(new FileType(newType));
  2413. self.destination.columns.removeAll();
  2414. self.guessFieldTypes();
  2415. });
  2416. }
  2417. }
  2418. });
  2419. self.isGuessingFormat = ko.observable(false);
  2420. self.guessFormat = function () {
  2421. self.isGuessingFormat(true);
  2422. self.destination.columns.removeAll();
  2423. $.post("${ url('indexer:guess_format') }", {
  2424. "fileFormat": ko.mapping.toJSON(self.source)
  2425. }, function (resp) {
  2426. if (resp.status !== 0) {
  2427. $(document).trigger("error", resp.message);
  2428. } else {
  2429. var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
  2430. self.source.format(newFormat);
  2431. if (self.source.inputFormat() === 'stream') {
  2432. if (self.source.streamSelection() === 'kafka') {
  2433. self.source.kafkaTopics(resp['topics']);
  2434. } else if (self.source.streamSelection() === 'flume') {
  2435. self.source.streamObjects(resp['objects']);
  2436. }
  2437. } else if (self.source.inputFormat() === 'connector') {
  2438. // Assumes selectectedConnector == 'sfdc'
  2439. self.source.streamObjects(resp['objects']);
  2440. }
  2441. if (self.source.inputFormat() !== 'stream' && self.source.inputFormat() !== 'connector') {
  2442. self.guessFieldTypes();
  2443. }
  2444. }
  2445. self.isGuessingFormat(false);
  2446. viewModel.wizardEnabled(true);
  2447. }).fail(function (xhr) {
  2448. $(document).trigger("error", xhr.responseText);
  2449. viewModel.isLoading(false);
  2450. self.isGuessingFormat(false);
  2451. });
  2452. };
  2453. self.isGuessingFieldTypes = ko.observable(false);
  2454. self.guessFieldTypes = function () {
  2455. if (guessFieldTypesXhr) {
  2456. guessFieldTypesXhr.abort();
  2457. }
  2458. self.isGuessingFieldTypes(true);
  2459. guessFieldTypesXhr = $.post("${ url('indexer:guess_field_types') }", {
  2460. "fileFormat": ko.mapping.toJSON(self.source)
  2461. }, function (resp) {
  2462. self.loadSampleData(resp);
  2463. self.isGuessingFieldTypes(false);
  2464. guessFieldTypesXhr = null;
  2465. }).fail(function (xhr) {
  2466. self.loadSampleData({sample_cols: [], columns: [], sample: []});
  2467. $(document).trigger("error", xhr.responseText);
  2468. self.isGuessingFieldTypes(false);
  2469. viewModel.isLoading(false);
  2470. guessFieldTypesXhr = null;
  2471. });
  2472. };
  2473. self.loadSampleData = function(resp) {
  2474. resp.columns.forEach(function (entry, i, arr) {
  2475. if (self.destination.outputFormat() === 'table' && self.source.inputFormat() != 'rdbms') {
  2476. entry.type = MAPPINGS.get(MAPPINGS.SOLR_TO_HIVE, entry.type, 'string');
  2477. } else if (self.destination.outputFormat() === 'index') {
  2478. entry.type = MAPPINGS.get(MAPPINGS.HIVE_TO_SOLR, entry.type, entry.type);
  2479. }
  2480. arr[i] = loadField(entry, self.destination, i);
  2481. });
  2482. self.source.sampleCols(resp.sample_cols ? resp.sample_cols : resp.columns);
  2483. self.source.sample(resp.sample);
  2484. self.destination.columns(resp.columns);
  2485. if (self.destination.tableFormat() == 'kudu' && $.grep(resp.columns, function(column) { return column.name() == 'id' }).length > 0) {
  2486. self.destination.primaryKeys(['id']); // Auto select ID column
  2487. }
  2488. };
  2489. self.isIndexing = ko.observable(false);
  2490. self.indexingError = ko.observable(false);
  2491. self.indexingSuccess = ko.observable(false);
  2492. self.destination.indexerRunJob.subscribe(function () {
  2493. self.destination.columns().forEach(function (column) {
  2494. column.operations.removeAll();
  2495. });
  2496. });
  2497. self.operationTypesFiltered = ko.pureComputed(function () {
  2498. var indexerRunJob = self.destination.indexerRunJob();
  2499. return self.operationTypes.filter(function (operation) {
  2500. return indexerRunJob || operation.name === 'split'; // We only support split for now with CSV API
  2501. }).map(function (o) {
  2502. return o.name;
  2503. });
  2504. });
  2505. self.commands = ko.observable([]);
  2506. self.showCommands = function() {
  2507. self.indexFile({show: true});
  2508. };
  2509. self.indexFile = function (options) {
  2510. var options = options || {};
  2511. if (!self.readyToIndex()) {
  2512. return;
  2513. }
  2514. $(".jHueNotify").remove();
  2515. self.indexingStarted(true);
  2516. %if not is_embeddable:
  2517. viewModel.isLoading(true);
  2518. self.isIndexing(true);
  2519. $.post("${ url('indexer:importer_submit') }", {
  2520. "source": ko.mapping.toJSON(self.source),
  2521. "destination": ko.mapping.toJSON(self.destination)
  2522. }, function (resp) {
  2523. if (resp.status !== 0) {
  2524. $(document).trigger("error", resp.message);
  2525. self.indexingStarted(false);
  2526. self.isIndexing(false);
  2527. } else if (resp.on_success_url) {
  2528. $.jHueNotify.info("${ _('Creation success.') }");
  2529. if (resp.pubSubUrl) {
  2530. huePubSub.publish(notebook.pubSubUrl);
  2531. }
  2532. huePubSub.publish('open.link', resp.on_success_url);
  2533. } else {
  2534. self.showCreate(true);
  2535. self.editorId(resp.history_id);
  2536. self.jobId(resp.handle.id);
  2537. $('#importerNotebook').html($('#importerNotebook-progress').html());
  2538. self.editorVM = new EditorViewModel(resp.history_uuid, '', {
  2539. user: '${ user.username }',
  2540. userId: ${ user.id },
  2541. languages: [{name: "Java", type: "java"}, {name: "Hive SQL", type: "hive"}], // TODO reuse
  2542. snippetViewSettings: {
  2543. java : {
  2544. snippetIcon: 'fa-file-archive-o '
  2545. },
  2546. hive: {
  2547. placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
  2548. aceMode: 'ace/mode/hive',
  2549. snippetImage: '${ static("beeswax/art/icon_beeswax_48.png") }',
  2550. sqlDialect: true
  2551. },
  2552. impala: {
  2553. placeHolder: '${ _("Example: SELECT * FROM tablename, or press CTRL + space") }',
  2554. aceMode: 'ace/mode/impala',
  2555. snippetImage: '${ static("impala/art/icon_impala_48.png") }',
  2556. sqlDialect: true
  2557. },
  2558. sqoop1: {
  2559. placeHolder: '${ _("Example: import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir hdfs://localhost:8020/user/foo -m 1") }',
  2560. snippetImage: '${ static("sqoop/art/icon_sqoop_48.png") }'
  2561. }
  2562. }
  2563. });
  2564. self.editorVM.editorMode(true);
  2565. self.editorVM.isNotificationManager(true);
  2566. ko.cleanNode($("#importerNotebook")[0]);
  2567. ko.applyBindings(self.editorVM, $("#importerNotebook")[0]);
  2568. self.editorVM.openNotebook(resp.history_uuid, null, true, function(){
  2569. self.editorVM.selectedNotebook().snippets()[0].progress.subscribe(function(val){
  2570. if (val === 100) {
  2571. self.indexingStarted(false);
  2572. self.isIndexing(false);
  2573. self.indexingSuccess(true);
  2574. }
  2575. });
  2576. self.editorVM.selectedNotebook().snippets()[0].status.subscribe(function(val){
  2577. if (val === 'failed') {
  2578. self.isIndexing(false);
  2579. self.indexingStarted(false);
  2580. self.indexingError(true);
  2581. } else if (val === 'available') {
  2582. var snippet = self.editorVM.selectedNotebook().snippets()[0]; // Could be native to editor at some point
  2583. if (!snippet.result.handle().has_more_statements) {
  2584. if (self.editorVM.selectedNotebook().onSuccessUrl()) {
  2585. var match = snippet.statement_raw().match(/CREATE TABLE `([^`]+)`/i);
  2586. if (match) {
  2587. var db = match[1];
  2588. dataCatalog.getEntry({ connector: snippet.connector(), namespace: self.namespace(), compute: self.compute(), path: [ db ]}).done(function (dbEntry) {
  2589. dbEntry.clearCache({ silenceErrors: true }).done(function () {
  2590. huePubSub.publish('open.link', self.editorVM.selectedNotebook().onSuccessUrl());
  2591. })
  2592. });
  2593. } else {
  2594. dataCatalog.getEntry({ connector: snippet.connector(), namespace: self.namespace(), compute: self.compute(), path: []}).done(function (sourceEntry) {
  2595. sourceEntry.clearCache({ silenceErrors: true }).done(function () {
  2596. huePubSub.publish('open.link', self.editorVM.selectedNotebook().onSuccessUrl());
  2597. })
  2598. });
  2599. }
  2600. }
  2601. } else { // Perform last DROP statement execute
  2602. snippet.execute();
  2603. }
  2604. }
  2605. });
  2606. self.editorVM.selectedNotebook().snippets()[0].checkStatus();
  2607. });
  2608. }
  2609. viewModel.isLoading(false);
  2610. }).fail(function (xhr, textStatus, errorThrown) {
  2611. $(document).trigger("error", xhr.responseText);
  2612. viewModel.isLoading(false);
  2613. self.indexingStarted(false);
  2614. self.isIndexing(false);
  2615. });
  2616. % else:
  2617. $.post("${ url('indexer:importer_submit') }", {
  2618. "source": ko.mapping.toJSON(self.source),
  2619. "destination": ko.mapping.toJSON(self.destination),
  2620. "start_time": ko.mapping.toJSON((new Date()).getTime()),
  2621. "show_command": options.show || ''
  2622. }, function (resp) {
  2623. self.indexingStarted(false);
  2624. if (resp.status === 0) {
  2625. if (resp.history_uuid) {
  2626. $.jHueNotify.info("${ _('Task submitted') }");
  2627. huePubSub.publish('notebook.task.submitted', resp);
  2628. } else if (resp.on_success_url) {
  2629. if (resp.pub_sub_url) {
  2630. huePubSub.publish(resp.pub_sub_url);
  2631. }
  2632. $.jHueNotify.info("${ _('Creation success') }");
  2633. if (resp.errors && resp.errors.length) {
  2634. $.jHueNotify.warn("${ _('Skipped records: ') }" + resp.errors.join(', '));
  2635. }
  2636. huePubSub.publish('open.link', resp.on_success_url);
  2637. } else if (resp.commands) {
  2638. self.commands(resp.commands);
  2639. $('#showCommandsModal').modal('show');
  2640. }
  2641. } else {
  2642. $(document).trigger("error", resp && resp.message ? resp.message : '${ _("Error importing") }');
  2643. }
  2644. }).fail(function (xhr) {
  2645. self.indexingStarted(false);
  2646. $(document).trigger("error", xhr.responseText);
  2647. });
  2648. % endif
  2649. hueAnalytics.log('importer', 'submit/' + self.source.inputFormat() + '/' + self.destination.outputFormat());
  2650. };
  2651. self.removeOperation = function (operation, operationList) {
  2652. operationList.remove(operation);
  2653. hueAnalytics.log('importer', 'step/removeOperation');
  2654. };
  2655. self.addOperation = function (field) {
  2656. field.operations.push(new Operation("split"));
  2657. hueAnalytics.log('importer', 'step/addOperation');
  2658. };
  2659. self.load = function (state) {
  2660. self.source.name(state.name);
  2661. self.source.show(state.show);
  2662. self.source.path(state.path);
  2663. self.destination.columns.removeAll();
  2664. if (state.format && 'type' in state.format) {
  2665. var koFormat = ko.mapping.fromJS(new FileType(state.format.type, state.format));
  2666. self.source.format(koFormat);
  2667. }
  2668. if (state.columns) {
  2669. state.columns.forEach(function (currCol) {
  2670. self.destination.columns.push(loadField(currCol));
  2671. });
  2672. }
  2673. }
  2674. };
  2675. var loadDefaultField = function (options) {
  2676. if (!options.name) {
  2677. options.name = getNewFieldName();
  2678. }
  2679. return loadField($.extend({}, ${default_field_type | n}, options));
  2680. };
  2681. var loadField = function (currField, parent, idx) {
  2682. var koField = ko.mapping.fromJS(currField);
  2683. if (koField.name && parent) {
  2684. koField.name.subscribe(function (newVal) {
  2685. if (newVal.indexOf(',') > -1) {
  2686. var fields = newVal.split(',');
  2687. fields.forEach(function (val, i) {
  2688. if (i + idx < parent.columns().length) {
  2689. parent.columns()[i + idx].name(val);
  2690. }
  2691. });
  2692. }
  2693. parent.bulkColumnNames(parent.columns().map(function (item) {
  2694. return item.name()
  2695. }).join(','));
  2696. });
  2697. }
  2698. koField.operations.removeAll();
  2699. currField.operations.forEach(function (operationData) {
  2700. var operation = new Operation(operationData.type);
  2701. operation.load(operationData);
  2702. koField.operations.push(operation);
  2703. });
  2704. var autoExpand = function (newVal) {
  2705. if ((newVal === 'array' || newVal === 'map' || newVal === 'struct') && koField.nested().length === 0) {
  2706. koField.nested.push(loadDefaultField({level: koField.level() + 1}));
  2707. }
  2708. };
  2709. koField.type.subscribe(autoExpand);
  2710. koField.keyType.subscribe(autoExpand);
  2711. return koField;
  2712. };
  2713. var IndexerViewModel = function () {
  2714. var self = this;
  2715. self.apiHelper = window.apiHelper;
  2716. self.sourceType = window.location.getParameter('sourceType', true) || '${ source_type }';
  2717. self.namespaceId = window.location.getParameter('namespace', true);
  2718. self.computeId = window.location.getParameter('compute', true);
  2719. self.assistAvailable = ko.observable(true);
  2720. self.isLeftPanelVisible = ko.observable();
  2721. self.apiHelper.withTotalStorage('assist', 'assist_panel_visible', self.isLeftPanelVisible, true);
  2722. self.loadDefaultField = loadDefaultField;
  2723. self.createWizard = new CreateWizard(self);
  2724. // Wizard related
  2725. self.wizardEnabled = ko.observable(false);
  2726. self.currentStep = ko.observable(self.createWizard.prefill.target_type() === 'database' ? 2 : 1);
  2727. self.currentStep.subscribe(function () {
  2728. %if is_embeddable:
  2729. $('.page-content').scrollTop(0);
  2730. %else:
  2731. $('.content-panel').scrollTop(0);
  2732. %endif
  2733. });
  2734. self.previousStepVisible = ko.pureComputed(function(){
  2735. return self.currentStep() > 1 && (self.createWizard.destination.outputFormat() !== 'database' || self.createWizard.source.inputFormat() === 'rdbms');
  2736. });
  2737. self.nextStepVisible = ko.pureComputed(function(){
  2738. return self.currentStep() < 3 && self.wizardEnabled();
  2739. });
  2740. self.nextStep = function () {
  2741. if (self.nextStepVisible()) {
  2742. self.currentStep(self.currentStep() + 1);
  2743. hueAnalytics.log('importer', 'step/' + self.currentStep());
  2744. }
  2745. };
  2746. self.previousStep = function () {
  2747. if (self.previousStepVisible()) {
  2748. self.currentStep(self.currentStep() - 1);
  2749. hueAnalytics.log('importer', 'step/' + self.currentStep());
  2750. }
  2751. };
  2752. self.isLoading = ko.observable(false);
  2753. };
  2754. var viewModel;
  2755. function resizeElements () {
  2756. var $contentPanel = $('#importerComponents').find('.content-panel-inner');
  2757. $('.step-indicator-fixed').width($contentPanel.width());
  2758. document.styleSheets[0].addRule('.form-actions','margin-left: -11px !important');
  2759. document.styleSheets[0].addRule('.step-indicator li:first-child:before','max-width: ' + ($contentPanel.find('.step-indicator li:first-child .caption').width()) + 'px');
  2760. document.styleSheets[0].addRule('.step-indicator li:first-child:before','left: ' + ($contentPanel.find('.step-indicator li:first-child .caption').width()/2) + 'px');
  2761. document.styleSheets[0].addRule('.step-indicator li:last-child:before','max-width: ' + ($contentPanel.find('.step-indicator li:last-child .caption').width()) + 'px');
  2762. document.styleSheets[0].addRule('.step-indicator li:last-child:before','right: ' + ($contentPanel.find('.step-indicator li:last-child .caption').width()/2) + 'px');
  2763. }
  2764. $(document).ready(function () {
  2765. viewModel = new IndexerViewModel();
  2766. ko.applyBindings(viewModel, $('#importerComponents')[0]);
  2767. var draggableMeta = {};
  2768. huePubSub.subscribe('draggable.text.meta', function (meta) {
  2769. draggableMeta = meta;
  2770. });
  2771. huePubSub.subscribe('importer.show.bulkeditor', function (meta) {
  2772. $('#fieldsBulkEditor').modal('show');
  2773. }, 'importer');
  2774. huePubSub.subscribe('split.panel.resized', resizeElements);
  2775. hueUtils.waitForRendered('.step-indicator li:first-child .caption', function(el){ return el.width() < $('#importerComponents').find('.content-panel-inner').width()/2 }, resizeElements);
  2776. $(window).on('resize', resizeElements);
  2777. $('.importer-droppable').droppable({
  2778. accept: ".draggableText",
  2779. drop: function (e, ui) {
  2780. var text = ui.helper.text();
  2781. var generatedName = 'idx';
  2782. switch (draggableMeta.type){
  2783. case 'sql':
  2784. if (draggableMeta.table !== '') {
  2785. generatedName += draggableMeta.table;
  2786. viewModel.createWizard.source.inputFormat('table');
  2787. viewModel.createWizard.source.table(draggableMeta.table);
  2788. }
  2789. break;
  2790. case 'hdfs':
  2791. generatedName += draggableMeta.definition.name;
  2792. viewModel.createWizard.source.inputFormat('file');
  2793. viewModel.createWizard.source.path(draggableMeta.definition.path);
  2794. break;
  2795. case 'document':
  2796. if (draggableMeta.definition.type === 'query-hive') {
  2797. generatedName += draggableMeta.definition.name;
  2798. viewModel.createWizard.source.inputFormat('query');
  2799. viewModel.createWizard.source.draggedQuery(draggableMeta.definition.uuid);
  2800. }
  2801. break;
  2802. }
  2803. if (generatedName !== 'idx' && viewModel.createWizard.source.name() === '') {
  2804. viewModel.createWizard.source.name(generatedName);
  2805. }
  2806. }
  2807. });
  2808. });
  2809. })();
  2810. </script>
  2811. </span>
  2812. %if not is_embeddable:
  2813. ${ commonfooter(request, messages) | n,unicode }
  2814. %endif