models2.py 102 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import json
  18. import logging
  19. import re
  20. import time
  21. import uuid
  22. from datetime import datetime, timedelta
  23. from dateutil.parser import parse
  24. from string import Template
  25. from django.core.urlresolvers import reverse
  26. from django.utils.encoding import force_unicode
  27. from django.utils.translation import ugettext as _
  28. from desktop.conf import USE_DEFAULT_CONFIGURATION
  29. from desktop.lib import django_mako
  30. from desktop.lib.exceptions_renderable import PopupException
  31. from desktop.lib.i18n import smart_str
  32. from desktop.lib.json_utils import JSONEncoderForHTML
  33. from desktop.models import DefaultConfiguration, Document2, Document
  34. from hadoop.fs.hadoopfs import Hdfs
  35. from hadoop.fs.exceptions import WebHdfsException
  36. from liboozie.submission2 import Submission
  37. from liboozie.submission2 import create_directories
  38. from oozie.conf import REMOTE_SAMPLE_DIR
  39. from oozie.utils import utc_datetime_format, UTC_TIME_FORMAT, convert_to_server_timezone
  40. from oozie.importlib.workflows import generate_v2_graph_nodes, MalformedWfDefException, InvalidTagWithNamespaceException
  41. from liboozie.oozie_api import get_oozie
  42. LOG = logging.getLogger(__name__)
  43. class Job(object):
  44. def find_all_parameters(self, with_lib_path=True):
  45. params = self.find_parameters()
  46. for param in self.parameters:
  47. params[param['name'].strip()] = param['value']
  48. if params.get('nominal_time') == '':
  49. params['nominal_time'] = datetime.today().strftime(UTC_TIME_FORMAT)
  50. return [{'name': name, 'value': value} for name, value in params.iteritems() if with_lib_path or name != 'oozie.use.system.libpath']
  51. @classmethod
  52. def get_workspace(cls, user):
  53. return (REMOTE_SAMPLE_DIR.get() + '/hue-oozie-$TIME').replace('$USER', user.username).replace('$TIME', str(time.time()))
  54. @property
  55. def validated_name(self):
  56. good_name = []
  57. for c in self.name[:40]:
  58. if not good_name:
  59. if not re.match('[a-zA-Z_\{\$\}]', c):
  60. c = '_'
  61. else:
  62. if not re.match('[\-_a-zA-Z0-9\{\$\}]', c):
  63. c = '_'
  64. good_name.append(c)
  65. return ''.join(good_name)
  66. def __str__(self):
  67. return '%s' % force_unicode(self.name)
  68. def deployment_dir(self):
  69. return None
  70. def check_workspace(self, fs, user):
  71. # Create optional default root workspace for the first submission
  72. if REMOTE_SAMPLE_DIR.get() == REMOTE_SAMPLE_DIR.config.default_value:
  73. create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
  74. Submission(user, self, fs, None, {})._create_dir(self.deployment_dir)
  75. Submission(user, self, fs, None, {})._create_dir(Hdfs.join(self.deployment_dir, 'lib'))
  76. def import_workspace(self, fs, source_deployment_dir, owner):
  77. try:
  78. fs.copy_remote_dir(source_deployment_dir, self.deployment_dir, owner=owner)
  79. except WebHdfsException, e:
  80. msg = _('The copy of the deployment directory failed: %s.') % e
  81. LOG.error(msg)
  82. raise PopupException(msg)
  83. class WorkflowConfiguration(object):
  84. APP_NAME = 'oozie-workflow'
  85. SLA_DEFAULT = [
  86. {'key': 'enabled', 'value': False}, # Always first element
  87. {'key': 'nominal-time', 'value': '${nominal_time}'},
  88. {'key': 'should-start', 'value': ''},
  89. {'key': 'should-end', 'value': '${30 * MINUTES}'},
  90. {'key': 'max-duration', 'value': ''},
  91. {'key': 'alert-events', 'value': ''},
  92. {'key': 'alert-contact', 'value': ''},
  93. {'key': 'notification-msg', 'value': ''},
  94. {'key': 'upstream-apps', 'value': ''},
  95. ]
  96. PROPERTIES = [
  97. {
  98. "multiple": True,
  99. "defaultValue": [
  100. {
  101. 'name': 'oozie.use.system.libpath',
  102. 'value': True
  103. }
  104. ],
  105. "value": [
  106. {
  107. 'name': 'oozie.use.system.libpath',
  108. 'value': True
  109. }
  110. ],
  111. "nice_name": _("Variables"),
  112. "key": "parameters",
  113. "help_text": _("Add one or more Oozie workflow job parameters."),
  114. "type": "parameters"
  115. }, {
  116. "multiple": False,
  117. "defaultValue": '',
  118. "value": '',
  119. "nice_name": _("Workspace"),
  120. "key": "deployment_dir",
  121. "help_text": _("Specify the deployment directory."),
  122. "type": "hdfs-file"
  123. }, {
  124. "multiple": True,
  125. "defaultValue": [],
  126. "value": [],
  127. "nice_name": _("Hadoop Properties"),
  128. "key": "properties",
  129. "help_text": _("Hadoop configuration properties."),
  130. "type": "settings"
  131. }, {
  132. "multiple": False,
  133. "defaultValue": True,
  134. "value": True,
  135. "nice_name": _("Show graph arrows"),
  136. "key": "show_arrows",
  137. "help_text": _("Toggles display of graph arrows."),
  138. "type": "boolean"
  139. }, {
  140. "multiple": False,
  141. "defaultValue": "uri:oozie:workflow:0.5",
  142. "value": "uri:oozie:workflow:0.5",
  143. "nice_name": _("Version"),
  144. "key": "schema_version",
  145. "help_text": _("Oozie XML Schema Version"),
  146. "type": "string",
  147. "options": [
  148. "uri:oozie:workflow:0.5",
  149. "uri:oozie:workflow:0.4.5",
  150. "uri:oozie:workflow:0.4",
  151. ]
  152. }, {
  153. "multiple": False,
  154. "defaultValue": '',
  155. "value": '',
  156. "nice_name": _("Job XML"),
  157. "key": "job_xml",
  158. "help_text": _("Oozie Job XML file"),
  159. "type": "hdfs-file"
  160. }, {
  161. "multiple": False,
  162. "defaultValue": False,
  163. "value": False,
  164. "nice_name": _("SLA Enabled"),
  165. "key": "sla_enabled",
  166. "help_text": _("SLA Enabled"),
  167. "type": "boolean"
  168. }, {
  169. "multiple": False,
  170. "defaultValue": SLA_DEFAULT,
  171. "value": SLA_DEFAULT,
  172. "nice_name": _("SLA Configuration"),
  173. "key": "sla",
  174. "help_text": _("Oozie SLA properties"),
  175. "type": "settings",
  176. "options": [prop['key'] for prop in SLA_DEFAULT]
  177. }
  178. ]
  179. class Workflow(Job):
  180. XML_FILE_NAME = 'workflow.xml'
  181. PROPERTY_APP_PATH = 'oozie.wf.application.path'
  182. HUE_ID = 'hue-id-w'
  183. def __init__(self, data=None, document=None, workflow=None, user=None):
  184. self.document = document
  185. if document is not None:
  186. self.data = document.data
  187. elif data is not None:
  188. self.data = data
  189. else:
  190. if not workflow:
  191. workflow = self.get_default_workflow()
  192. workflow['properties'] = self.get_workflow_properties_for_user(user, workflow)
  193. self.data = json.dumps({
  194. 'layout': [{
  195. "size":12, "rows":[
  196. {"widgets":[{"size":12, "name":"Start", "id":"3f107997-04cc-8733-60a9-a4bb62cebffc", "widgetType":"start-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]},
  197. {"widgets":[{"size":12, "name":"End", "id":"33430f0f-ebfa-c3ec-f237-3e77efa03d0a", "widgetType":"end-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]},
  198. {"widgets":[{"size":12, "name":"Kill", "id":"17c9c895-5a16-7443-bb81-f34b30b21548", "widgetType":"kill-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]}
  199. ],
  200. "drops":[ "temp"],
  201. "klass":"card card-home card-column span12"
  202. }],
  203. 'workflow': workflow
  204. })
  205. @classmethod
  206. def get_application_path_key(cls):
  207. return 'oozie.wf.application.path'
  208. @classmethod
  209. def gen_workflow_data_from_xml(cls, user, oozie_workflow):
  210. node_list = []
  211. try:
  212. node_list = generate_v2_graph_nodes(oozie_workflow.definition)
  213. except MalformedWfDefException, e:
  214. LOG.exception("Could not find any nodes in Workflow definition. Maybe it's malformed?")
  215. except InvalidTagWithNamespaceException, e:
  216. LOG.exception(
  217. "Tag with namespace %(namespace)s is not valid. Please use one of the following namespaces: %(namespaces)s" % {
  218. 'namespace': e.namespace,
  219. 'namespaces': e.namespaces
  220. })
  221. _to_lowercase(node_list)
  222. adj_list = _create_graph_adjaceny_list(node_list)
  223. node_hierarchy = ['start']
  224. _get_hierarchy_from_adj_list(adj_list, adj_list['start']['ok_to'], node_hierarchy)
  225. _update_adj_list(adj_list)
  226. wf_rows = _create_workflow_layout(node_hierarchy, adj_list)
  227. data = {'layout': [{}], 'workflow': {}}
  228. if wf_rows:
  229. data['layout'][0]['rows'] = wf_rows
  230. wf_nodes = []
  231. _dig_nodes(node_hierarchy, adj_list, user, wf_nodes)
  232. data['workflow']['nodes'] = wf_nodes
  233. data['workflow']['id'] = '123'
  234. data['workflow']['properties'] = cls.get_workflow_properties_for_user(user, workflow=None)
  235. data['workflow']['properties'].update({
  236. 'deployment_dir': '/user/hue/oozie/workspaces/hue-oozie-1452553957.19'
  237. })
  238. return data
  239. @classmethod
  240. def get_default_workflow(cls):
  241. return {
  242. "id": None,
  243. "uuid": None,
  244. "name": "My Workflow",
  245. "nodes": [
  246. {"id": "3f107997-04cc-8733-60a9-a4bb62cebffc", "name": "Start", "type": "start-widget", "properties": {},
  247. "children": [{'to': '33430f0f-ebfa-c3ec-f237-3e77efa03d0a'}]},
  248. {"id": "33430f0f-ebfa-c3ec-f237-3e77efa03d0a", "name": "End", "type": "end-widget", "properties": {},
  249. "children": []},
  250. {"id": "17c9c895-5a16-7443-bb81-f34b30b21548", "name": "Kill", "type": "kill-widget",
  251. "properties": {'message': _('Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]')},
  252. "children": []}
  253. ]
  254. }
  255. @classmethod
  256. def get_workflow_properties_for_user(cls, user, workflow=None):
  257. workflow = workflow if workflow is not None else {}
  258. properties = workflow.get('properties', None)
  259. if not properties:
  260. config = None
  261. properties = cls.get_properties()
  262. if user is not None:
  263. if USE_DEFAULT_CONFIGURATION.get():
  264. config = DefaultConfiguration.objects.get_configuration_for_user(app=WorkflowConfiguration.APP_NAME, user=user)
  265. if config is not None:
  266. properties.update(config.properties_dict)
  267. properties.update({
  268. 'wf1_id': None,
  269. 'description': ''
  270. })
  271. return properties
  272. @staticmethod
  273. def get_properties():
  274. return dict((prop['key'], prop['value']) for prop in WorkflowConfiguration.PROPERTIES)
  275. @property
  276. def id(self):
  277. return self.document.id
  278. @property
  279. def uuid(self):
  280. return self.document.uuid
  281. @property
  282. def name(self):
  283. _data = self.get_data()
  284. return _data['workflow']['name']
  285. @property
  286. def deployment_dir(self):
  287. _data = self.get_data()
  288. return _data['workflow']['properties']['deployment_dir']
  289. @property
  290. def parameters(self):
  291. _data = self.get_data()
  292. return _data['workflow']['properties']['parameters']
  293. @property
  294. def sla_enabled(self):
  295. _data = self.get_data()
  296. return _data['workflow']['properties']['sla'][0].get('value')
  297. @property
  298. def has_some_slas(self):
  299. return self.sla_enabled or any([node.sla_enabled for node in self.nodes])
  300. @property
  301. def credentials(self):
  302. return list(set([cred for node in self.nodes for cred in node.data['properties']['credentials']]))
  303. @property
  304. def sla(self):
  305. _data = self.get_data()
  306. return _data['workflow']['properties']['sla']
  307. @property
  308. def nodes(self):
  309. _data = self.get_data()
  310. return [Node(node) for node in _data['workflow']['nodes']]
  311. def find_parameters(self):
  312. params = set()
  313. for param in find_dollar_braced_variables(self.name):
  314. params.add(param)
  315. if self.sla_enabled:
  316. for param in find_json_parameters(self.sla):
  317. params.add(param)
  318. for node in self.nodes:
  319. params.update(node.find_parameters())
  320. return dict([(param, '') for param in list(params)])
  321. def get_json(self):
  322. _data = self.get_data()
  323. return json.dumps(_data)
  324. def get_data(self):
  325. _data = json.loads(self.data)
  326. if self.document is not None:
  327. _data['workflow']['id'] = self.document.id
  328. _data['workflow']['dependencies'] = list(self.document.dependencies.values('uuid', ))
  329. else:
  330. _data['workflow']['dependencies'] = []
  331. if 'parameters' not in _data['workflow']['properties']:
  332. _data['workflow']['properties']['parameters'] = [
  333. {'name': 'oozie.use.system.libpath', 'value': True},
  334. ]
  335. if 'show_arrows' not in _data['workflow']['properties']:
  336. _data['workflow']['properties']['show_arrows'] = True
  337. for node in _data['workflow']['nodes']:
  338. if 'credentials' in node['properties']: # If node is an Action
  339. if 'retry_max' not in node['properties']: # When displaying a workflow
  340. node['properties']['retry_max'] = []
  341. if 'retry_interval' not in node['properties']:
  342. node['properties']['retry_interval'] = []
  343. # Backward compatibility
  344. _upgrade_older_node(node)
  345. return _data
  346. def to_xml(self, mapping=None):
  347. if mapping is None:
  348. mapping = {}
  349. tmpl = 'editor2/gen/workflow.xml.mako'
  350. data = self.get_data()
  351. nodes = [node for node in self.nodes if node.name != 'End'] + [node for node in self.nodes if
  352. node.name == 'End'] # End at the end
  353. node_mapping = dict([(node.id, node) for node in nodes])
  354. sub_wfs_ids = [node.data['properties']['workflow'] for node in nodes if node.data['type'] == 'subworkflow']
  355. workflow_mapping = dict(
  356. [(workflow.uuid, Workflow(document=workflow)) for workflow in Document2.objects.filter(uuid__in=sub_wfs_ids)])
  357. xml = re.sub(re.compile('>\s*\n+', re.MULTILINE), '>\n', django_mako.render_to_string(tmpl, {
  358. 'wf': self,
  359. 'workflow': data['workflow'],
  360. 'nodes': nodes,
  361. 'mapping': mapping,
  362. 'node_mapping': node_mapping,
  363. 'workflow_mapping': workflow_mapping
  364. }))
  365. return force_unicode(xml.strip())
  366. def get_absolute_url(self):
  367. return reverse('oozie:edit_workflow') + '?workflow=%s' % self.id
  368. def override_subworkflow_id(self, sub_wf_action, workflow_id):
  369. _data = self.get_data()
  370. action = [_action for _action in _data['workflow']['nodes'] if _action['id'] == sub_wf_action.id]
  371. if action:
  372. action[0]['properties']['job_properties'].append({'name': Workflow.HUE_ID, 'value': workflow_id})
  373. self.data = json.dumps(_data)
  374. def update_name(self, name):
  375. _data = self.get_data()
  376. _data['workflow']['name'] = name
  377. self.data = json.dumps(_data)
  378. def set_workspace(self, user):
  379. _data = json.loads(self.data)
  380. _data['workflow']['properties']['deployment_dir'] = Job.get_workspace(user)
  381. self.data = json.dumps(_data)
  382. def create_single_action_workflow_data(self, node_id):
  383. _data = json.loads(self.data)
  384. start_node = [node for node in _data['workflow']['nodes'] if node['name'] == 'Start'][0]
  385. submit_node = [node for node in _data['workflow']['nodes'] if node['id'] == node_id][0]
  386. end_node = [node for node in _data['workflow']['nodes'] if node['name'] == 'End'][0]
  387. kill_node = [node for node in _data['workflow']['nodes'] if node['name'] == 'Kill'][0]
  388. # Modify children to point Start -> Submit_node -> End/Kill
  389. start_node['children'] = [{'to': submit_node['id']}]
  390. submit_node['children'] = [{'to': end_node['id']}, {'error': kill_node['id']}]
  391. _data['workflow']['properties']['deployment_dir'] = None
  392. # Recursively find the widget node
  393. def _get_node(rows, node_id):
  394. for row in rows:
  395. if not row['widgets']:
  396. for col in row['columns']:
  397. node = _get_node(col['rows'], node_id)
  398. if node:
  399. return node
  400. elif row['widgets'][0]['id'] == node_id:
  401. return row
  402. # Create wf data with above nodes
  403. return json.dumps({
  404. 'layout': [{
  405. "size": 12,
  406. "rows": [
  407. [row for row in _data['layout'][0]['rows'] if row['widgets'] and row['widgets'][0]['name'] == 'Start'][0],
  408. _get_node(_data['layout'][0]['rows'], node_id),
  409. [row for row in _data['layout'][0]['rows'] if row['widgets'] and row['widgets'][0]['name'] == 'End'][0],
  410. [row for row in _data['layout'][0]['rows'] if row['widgets'] and row['widgets'][0]['name'] == 'Kill'][0]
  411. ],
  412. "drops": ["temp"],
  413. "klass": "card card-home card-column span12"
  414. }],
  415. 'workflow': {
  416. "id": None,
  417. "uuid": None,
  418. "name": _data['workflow']['name'],
  419. "properties": _data['workflow']['properties'],
  420. "nodes": [start_node, submit_node, end_node, kill_node]
  421. }
  422. })
  423. # Updates node_list to lowercase names
  424. # To avoid case-sensitive failures
  425. def _to_lowercase(node_list):
  426. for node in node_list:
  427. for key in node.keys():
  428. if type(node[key]) is str:
  429. node[key] = node[key].lower()
  430. def _update_adj_list(adj_list):
  431. uuids = {}
  432. id = 1
  433. for node in adj_list.keys():
  434. adj_list[node]['id'] = id
  435. # Oozie uses same action for streaming and mapreduce but Hue manages them differently
  436. if adj_list[node]['node_type'] == 'map-reduce':
  437. if 'streaming' in adj_list[node]['name']:
  438. adj_list[node]['node_type'] = 'streaming'
  439. else:
  440. adj_list[node]['node_type'] = 'mapreduce'
  441. elif adj_list[node]['node_type'] == 'sub-workflow':
  442. adj_list[node]['node_type'] = 'subworkflow'
  443. if adj_list[node]['node_type'] == 'kill':
  444. adj_list[node]['uuid'] = '17c9c895-5a16-7443-bb81-f34b30b21548'
  445. elif adj_list[node]['node_type'] == 'start':
  446. adj_list[node]['uuid'] = '3f107997-04cc-8733-60a9-a4bb62cebffc'
  447. elif adj_list[node]['node_type'] == 'end':
  448. adj_list[node]['uuid'] = '33430f0f-ebfa-c3ec-f237-3e77efa03d0a'
  449. else:
  450. adj_list[node]['uuid'] = node[-4:] + str(uuid.uuid4())[4:]
  451. uuids[id] = adj_list[node]['uuid']
  452. id += 1
  453. return adj_list
  454. def _dig_nodes(nodes, adj_list, user, wf_nodes):
  455. for node in nodes:
  456. if type(node) != list:
  457. node = adj_list[node]
  458. properties = {}
  459. if '%s-widget' % node['node_type'] in NODES:
  460. properties = dict(NODES['%s-widget' % node['node_type']].get_fields())
  461. if node['node_type'] == 'pig':
  462. properties['script_path'] = node.get('pig').get('script_path')
  463. elif node['node_type'] == 'spark':
  464. properties['class'] = node.get('spark').get('class')
  465. properties['jars'] = node.get('spark').get('jar')
  466. elif node['node_type'] == 'hive' or node['node_type'] == 'hive2':
  467. properties['script_path'] = node.get('hive').get('script')
  468. elif node['node_type'] == 'java':
  469. properties['main_class'] = node.get('java').get('main-class')
  470. elif node['node_type'] == 'sqoop':
  471. properties['command'] = node.get('sqoop').get('command')
  472. elif node['node_type'] == 'mapreduce':
  473. properties['job_properties'] = node.get('job_properties')
  474. elif node['node_type'] == 'shell':
  475. properties['shell_command'] = node.get('shell').get('command')
  476. elif node['node_type'] == 'ssh':
  477. properties['user'] = '%s@%s' % (node.get('ssh').get('user'), node.get('ssh').get('host'))
  478. properties['ssh_command'] = node.get('ssh').get('command')
  479. elif node['node_type'] == 'fs':
  480. fs_props = node.get('fs')
  481. # TBD: gather props for different fs operations
  482. elif node['node_type'] == 'email':
  483. properties['to'] = node.get('email').get('to')
  484. properties['subject'] = node.get('email').get('subject')
  485. #TBD: body doesn't show up
  486. properties['body'] = node.get('email').get('body')
  487. elif node['node_type'] == 'streaming':
  488. properties['mapper'] = node.get('streaming').get('mapper')
  489. properties['reducer'] = node.get('streaming').get('reducer')
  490. elif node['node_type'] == 'distcp':
  491. properties['distcp_parameters'] = node.get('params')
  492. elif node['node_type'] == 'subworkflow':
  493. properties['app-path'] = node.get('subworkflow').get('app-path')
  494. properties['workflow'] = node.get('uuid')
  495. properties['job_properties'] = []
  496. properties['sla'] = ''
  497. children = []
  498. if node['node_type'] in ('fork', 'decision'):
  499. for key in node.keys():
  500. if key.startswith('path'):
  501. children.append({'to': adj_list[node[key]]['uuid'], 'condition': '${ 1 gt 0 }'})
  502. if node['node_type'] == 'decision':
  503. children.append({'to': adj_list[node['default']]['uuid'], 'condition': '${ 1 gt 0 }'})
  504. else:
  505. if node.get('ok_to'):
  506. children.append({'to': adj_list[node['ok_to']]['uuid']})
  507. if node.get('error_to'):
  508. children.append({'error': adj_list[node['error_to']]['uuid']})
  509. wf_nodes.append({
  510. "id": node['uuid'],
  511. "name": '%s-%s' % (node['node_type'].split('-')[0], node['uuid'][:4]),
  512. "type": "%s-widget" % node['node_type'],
  513. "properties": properties,
  514. "children": children
  515. })
  516. else:
  517. _dig_nodes(node, adj_list, user, wf_nodes)
  518. def _create_workflow_layout(nodes, adj_list, size=12):
  519. wf_rows = []
  520. for node in nodes:
  521. if type(node) == list and len(node) == 1:
  522. node = node[0]
  523. if type(node) != list:
  524. wf_rows.append({"widgets":[{"size":size, "name": adj_list[node]['node_type'], "id": adj_list[node]['uuid'], "widgetType": "%s-widget" % adj_list[node]['node_type'], "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size, "columns":[]}]})
  525. else:
  526. if adj_list[node[0]]['node_type'] in ('fork', 'decision'):
  527. wf_rows.append({"widgets":[{"size":size, "name": adj_list[node[0]]['name'], "id": adj_list[node[0]]['uuid'], "widgetType": "%s-widget" % adj_list[node[0]]['node_type'], "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size, "columns":[]}]})
  528. wf_rows.append({
  529. "id": str(uuid.uuid4()),
  530. "widgets":[
  531. ],
  532. "columns":[
  533. {
  534. "id": str(uuid.uuid4()),
  535. "size": (size / len(node[1])),
  536. "rows":
  537. [{
  538. "id": str(uuid.uuid4()),
  539. "widgets": c['widgets'],
  540. "columns":c.get('columns') or []
  541. } for c in col],
  542. "klass":"card card-home card-column span%s" % (size / len(node[1]))
  543. }
  544. for col in [_create_workflow_layout(item, adj_list, size) for item in node[1]]
  545. ]
  546. })
  547. if adj_list[node[0]]['node_type'] == 'fork':
  548. wf_rows.append({"widgets":[{"size":size, "name": adj_list[node[2]]['name'], "id": adj_list[node[2]]['uuid'], "widgetType": "%s-widget" % adj_list[node[2]]['node_type'], "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size, "columns":[]}]})
  549. else:
  550. wf_rows.append(_create_workflow_layout(node, adj_list, size))
  551. return wf_rows
  552. def _get_hierarchy_from_adj_list(adj_list, curr_node, node_hierarchy):
  553. _get_hierarchy_from_adj_list_helper(adj_list, curr_node, node_hierarchy)
  554. # Add End and Kill nodes to node_hierarchy
  555. node_hierarchy.append([adj_list[key]['name'] for key in adj_list.keys() if adj_list[key]['node_type'] == 'kill'])
  556. node_hierarchy.append([adj_list[key]['name'] for key in adj_list.keys() if adj_list[key]['node_type'] == 'end'])
  557. def _get_hierarchy_from_adj_list_helper(adj_list, curr_node, node_hierarchy):
  558. if not curr_node or adj_list[curr_node]['node_type'] in ('join', 'end', 'kill'):
  559. return curr_node
  560. elif adj_list[curr_node]['node_type'] in ('fork', 'decision'):
  561. branch_nodes = []
  562. branch_nodes.append(curr_node)
  563. join_node = None
  564. children = []
  565. for key in adj_list[curr_node].keys():
  566. if key.startswith('path'):
  567. child = []
  568. return_node = _get_hierarchy_from_adj_list_helper(adj_list, adj_list[curr_node][key], child)
  569. join_node = return_node if not join_node else join_node
  570. if child:
  571. children.append(child)
  572. branch_nodes.append(children)
  573. if adj_list[curr_node]['node_type'] == 'fork':
  574. branch_nodes.append(join_node)
  575. node_hierarchy.append(branch_nodes)
  576. return _get_hierarchy_from_adj_list_helper(adj_list, adj_list[join_node]['ok_to'], node_hierarchy)
  577. node_hierarchy.append(branch_nodes)
  578. return join_node
  579. else:
  580. node_hierarchy.append(curr_node)
  581. return _get_hierarchy_from_adj_list_helper(adj_list, adj_list[curr_node]['ok_to'], node_hierarchy)
  582. def _create_graph_adjaceny_list(nodes):
  583. start_node = [node for node in nodes if node.get('node_type') == 'start'][0]
  584. adj_list = {'start': start_node}
  585. for node in nodes:
  586. if node and node.get('node_type') != 'start':
  587. adj_list[node['name']] = node
  588. return adj_list
  589. class Node():
  590. def __init__(self, data):
  591. self.data = data
  592. self._augment_data()
  593. def to_xml(self, mapping=None, node_mapping=None, workflow_mapping=None):
  594. if mapping is None:
  595. mapping = {}
  596. if node_mapping is None:
  597. node_mapping = {}
  598. if workflow_mapping is None:
  599. workflow_mapping = {}
  600. if self.data['type'] in ('hive2', 'hive-document') and not self.data['properties']['jdbc_url']:
  601. self.data['properties']['jdbc_url'] = _get_hiveserver2_url()
  602. if self.data['type'] == 'fork':
  603. links = [link for link in self.data['children'] if link['to'] in node_mapping]
  604. if len(links) != len(self.data['children']):
  605. LOG.warn('Fork has some children links that do not exist, ignoring them: links %s, existing links %s, links %s, existing links %s' \
  606. % (len(links), len(self.data['children']), links, self.data['children']))
  607. self.data['children'] = links
  608. data = {
  609. 'node': self.data,
  610. 'mapping': mapping,
  611. 'node_mapping': node_mapping,
  612. 'workflow_mapping': workflow_mapping
  613. }
  614. return django_mako.render_to_string(self.get_template_name(), data)
  615. @property
  616. def id(self):
  617. return self.data['id']
  618. @property
  619. def name(self):
  620. return self.data['name']
  621. @property
  622. def sla_enabled(self):
  623. return 'sla' in self.data['properties'] and self.data['properties']['sla'] and self.data['properties']['sla'][0].get('value')
  624. def _augment_data(self):
  625. self.data['type'] = self.data['type'].replace('-widget', '')
  626. self.data['uuid'] = self.data['id']
  627. # Action Node
  628. if 'credentials' not in self.data['properties']:
  629. self.data['properties']['credentials'] = []
  630. if 'prepares' not in self.data['properties']:
  631. self.data['properties']['prepares'] = []
  632. if 'job_xml' not in self.data['properties']:
  633. self.data['properties']['job_xml'] = []
  634. if 'properties' not in self.data['properties']:
  635. self.data['properties']['properties'] = []
  636. if 'params' not in self.data['properties']:
  637. self.data['properties']['params'] = []
  638. if 'files' not in self.data['properties']:
  639. self.data['properties']['files'] = []
  640. if 'archives' not in self.data['properties']:
  641. self.data['properties']['archives'] = []
  642. if 'sla' not in self.data['properties']:
  643. self.data['properties']['sla'] = WorkflowConfiguration.SLA_DEFAULT
  644. if 'retry_max' not in self.data['properties']:
  645. self.data['properties']['retry_max'] = []
  646. if 'retry_interval' not in self.data['properties']:
  647. self.data['properties']['retry_interval'] = []
  648. # Backward compatibility
  649. _upgrade_older_node(self.data)
  650. def get_template_name(self):
  651. return 'editor2/gen/workflow-%s.xml.mako' % self.data['type']
  652. def find_parameters(self):
  653. return find_parameters(self) + (find_parameters(self, ['sla']) if self.sla_enabled else [])
  654. def _upgrade_older_node(node):
  655. if node['type'] in ('sqoop', 'sqoop-widget') and 'arguments' not in node['properties']:
  656. node['properties']['arguments'] = node['properties']['parameters']
  657. if node['type'] in ('kill', 'kill-widget') and 'to' not in node['properties']:
  658. node['properties']['enableMail'] = False
  659. node['properties']['to'] = ''
  660. node['properties']['cc'] = ''
  661. node['properties']['subject'] = ''
  662. node['properties']['body'] = ''
  663. if node['type'] == 'email-widget' and 'bcc' not in node['properties']:
  664. node['properties']['bcc'] = ''
  665. node['properties']['content_type'] = 'text/plain'
  666. node['properties']['attachment'] = ''
  667. if node['type'] == 'spark-widget' and 'files' not in node['properties']:
  668. node['properties']['files'] = []
  669. class Action(object):
  670. @classmethod
  671. def get_fields(cls):
  672. credentials = [cls.DEFAULT_CREDENTIALS] if hasattr(cls, 'DEFAULT_CREDENTIALS') else []
  673. return [(f['name'], f['value']) for f in cls.FIELDS.itervalues()] + [('sla', WorkflowConfiguration.SLA_DEFAULT), ('credentials', credentials)]
  674. class StartNode(Action):
  675. TYPE = 'start'
  676. FIELDS = {}
  677. class EndNode(Action):
  678. TYPE = 'end'
  679. FIELDS = {}
  680. class PigAction(Action):
  681. TYPE = 'pig'
  682. FIELDS = {
  683. 'script_path': {
  684. 'name': 'script_path',
  685. 'label': _('Script'),
  686. 'value': '',
  687. 'help_text': _('Path to the script on HDFS.'),
  688. 'type': ''
  689. },
  690. 'parameters': {
  691. 'name': 'parameters',
  692. 'label': _('Parameters'),
  693. 'value': [],
  694. 'help_text': _('The Pig parameters of the script without -param. e.g. INPUT=${inputDir}'),
  695. 'type': ''
  696. },
  697. 'arguments': {
  698. 'name': 'arguments',
  699. 'label': _('Arguments'),
  700. 'value': [],
  701. 'help_text': _('The Pig parameters of the script as is. e.g. -param, INPUT=${inputDir}'),
  702. 'type': ''
  703. },
  704. # Common
  705. 'files': {
  706. 'name': 'files',
  707. 'label': _('Files'),
  708. 'value': [],
  709. 'help_text': _('Files put in the running directory.'),
  710. 'type': ''
  711. },
  712. 'archives': {
  713. 'name': 'archives',
  714. 'label': _('Archives'),
  715. 'value': [],
  716. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  717. 'type': ''
  718. },
  719. 'job_properties': {
  720. 'name': 'job_properties',
  721. 'label': _('Hadoop job properties'),
  722. 'value': [],
  723. 'help_text': _('value, e.g. production'),
  724. 'type': ''
  725. },
  726. 'prepares': {
  727. 'name': 'prepares',
  728. 'label': _('Prepares'),
  729. 'value': [],
  730. 'help_text': _('Path to manipulate before starting the application.'),
  731. 'type': ''
  732. },
  733. 'job_xml': {
  734. 'name': 'job_xml',
  735. 'label': _('Job XML'),
  736. 'value': [],
  737. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  738. 'type': ''
  739. },
  740. 'retry_max': {
  741. 'name': 'retry_max',
  742. 'label': _('Max retry'),
  743. 'value': [],
  744. 'help_text': _('Number of times, default is 3'),
  745. 'type': ''
  746. },
  747. 'retry_interval': {
  748. 'name': 'retry_interval',
  749. 'label': _('Retry interval'),
  750. 'value': [],
  751. 'help_text': _('Wait time in minutes, default is 10'),
  752. 'type': ''
  753. }
  754. }
  755. @classmethod
  756. def get_mandatory_fields(cls):
  757. return [cls.FIELDS['script_path']]
  758. class JavaAction(Action):
  759. TYPE = 'java'
  760. FIELDS = {
  761. 'jar_path': {
  762. 'name': 'jar_path',
  763. 'label': _('Jar name'),
  764. 'value': '',
  765. 'help_text': _('Path to the jar on HDFS.'),
  766. 'type': ''
  767. },
  768. 'main_class': {
  769. 'name': 'main_class',
  770. 'label': _('Main class'),
  771. 'value': '',
  772. 'help_text': _('Java class. e.g. org.apache.hadoop.examples.Grep'),
  773. 'type': 'text'
  774. },
  775. 'arguments': {
  776. 'name': 'arguments',
  777. 'label': _('Arguments'),
  778. 'value': [],
  779. 'help_text': _('Arguments of the main method. The value of each arg element is considered a single argument '
  780. 'and they are passed to the main method in the same order.'),
  781. 'type': ''
  782. },
  783. 'java_opts': {
  784. 'name': 'java_opts',
  785. 'label': _('Java options'),
  786. 'value': [],
  787. 'help_text': _('Parameters for the JVM, e.g. -Dprop1=a -Dprop2=b'),
  788. 'type': ''
  789. },
  790. 'capture_output': {
  791. 'name': 'capture_output',
  792. 'label': _('Capture output'),
  793. 'value': False,
  794. 'help_text': _('Capture output of the stdout of the %(program)s command execution. The %(program)s '
  795. 'command output must be in Java Properties file format and it must not exceed 2KB. '
  796. 'From within the workflow definition, the output of an %(program)s action node is accessible '
  797. 'via the String action:output(String node, String key) function') % {'program': TYPE.title()},
  798. 'type': ''
  799. },
  800. # Common
  801. 'files': {
  802. 'name': 'files',
  803. 'label': _('Files'),
  804. 'value': [],
  805. 'help_text': _('Files put in the running directory.'),
  806. 'type': ''
  807. },
  808. 'archives': {
  809. 'name': 'archives',
  810. 'label': _('Archives'),
  811. 'value': [],
  812. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  813. 'type': ''
  814. },
  815. 'job_properties': {
  816. 'name': 'job_properties',
  817. 'label': _('Hadoop job properties'),
  818. 'value': [],
  819. 'help_text': _('value, e.g. production'),
  820. 'type': ''
  821. },
  822. 'prepares': {
  823. 'name': 'prepares',
  824. 'label': _('Prepares'),
  825. 'value': [],
  826. 'help_text': _('Path to manipulate before starting the application.'),
  827. 'type': ''
  828. },
  829. 'job_xml': {
  830. 'name': 'job_xml',
  831. 'label': _('Job XML'),
  832. 'value': [],
  833. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  834. 'type': ''
  835. },
  836. 'retry_max': {
  837. 'name': 'retry_max',
  838. 'label': _('Max retry'),
  839. 'value': [],
  840. 'help_text': _('Number of times, default is 3'),
  841. 'type': ''
  842. },
  843. 'retry_interval': {
  844. 'name': 'retry_interval',
  845. 'label': _('Retry interval'),
  846. 'value': [],
  847. 'help_text': _('Wait time in minutes, default is 10'),
  848. 'type': ''
  849. }
  850. }
  851. @classmethod
  852. def get_mandatory_fields(cls):
  853. return [cls.FIELDS['jar_path'], cls.FIELDS['main_class']]
  854. class HiveAction(Action):
  855. TYPE = 'hive'
  856. DEFAULT_CREDENTIALS = 'hcat'
  857. FIELDS = {
  858. 'script_path': {
  859. 'name': 'script_path',
  860. 'label': _('Script'),
  861. 'value': '',
  862. 'help_text': _('Path to the script on HDFS.'),
  863. 'type': ''
  864. },
  865. 'parameters': {
  866. 'name': 'parameters',
  867. 'label': _('Parameters'),
  868. 'value': [],
  869. 'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}') % {'type': TYPE.title()},
  870. 'type': ''
  871. },
  872. # Common
  873. 'files': {
  874. 'name': 'files',
  875. 'label': _('Files'),
  876. 'value': [],
  877. 'help_text': _('Files put in the running directory.'),
  878. 'type': ''
  879. },
  880. 'archives': {
  881. 'name': 'archives',
  882. 'label': _('Archives'),
  883. 'value': [],
  884. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  885. 'type': ''
  886. },
  887. 'job_properties': {
  888. 'name': 'job_properties',
  889. 'label': _('Hadoop job properties'),
  890. 'value': [],
  891. 'help_text': _('value, e.g. production'),
  892. 'type': ''
  893. },
  894. 'prepares': {
  895. 'name': 'prepares',
  896. 'label': _('Prepares'),
  897. 'value': [],
  898. 'help_text': _('Path to manipulate before starting the application.'),
  899. 'type': ''
  900. },
  901. 'hive_xml': {
  902. 'name': 'hive_xml',
  903. 'label': _('Hive XML'),
  904. 'value': '',
  905. 'help_text': _('Refer to a hive-site.xml for connecting to Hive'),
  906. 'type': ''
  907. },
  908. 'retry_max': {
  909. 'name': 'retry_max',
  910. 'label': _('Max retry'),
  911. 'value': [],
  912. 'help_text': _('Number of times, default is 3'),
  913. 'type': ''
  914. },
  915. 'retry_interval': {
  916. 'name': 'retry_interval',
  917. 'label': _('Retry interval'),
  918. 'value': [],
  919. 'help_text': _('Wait time in minutes, default is 10'),
  920. 'type': ''
  921. }
  922. }
  923. @classmethod
  924. def get_mandatory_fields(cls):
  925. return [cls.FIELDS['script_path'], cls.FIELDS['hive_xml']]
  926. def _get_hiveserver2_url():
  927. try:
  928. from beeswax.hive_site import hiveserver2_jdbc_url
  929. return hiveserver2_jdbc_url()
  930. except Exception, e:
  931. # Might fail is Hive is disabled
  932. LOG.warn('Could not guess HiveServer2 URL: %s' % smart_str(e))
  933. return 'jdbc:hive2://localhost:10000/default'
  934. class HiveServer2Action(Action):
  935. TYPE = 'hive2'
  936. DEFAULT_CREDENTIALS = 'hive2'
  937. FIELDS = {
  938. 'script_path': {
  939. 'name': 'script_path',
  940. 'label': _('Script'),
  941. 'value': '',
  942. 'help_text': _('Path to the script on HDFS.'),
  943. 'type': ''
  944. },
  945. 'parameters': {
  946. 'name': 'parameters',
  947. 'label': _('Parameters'),
  948. 'value': [],
  949. 'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}') % {'type': TYPE.title()},
  950. 'type': ''
  951. },
  952. # Common
  953. 'jdbc_url': {
  954. 'name': 'jdbc_url',
  955. 'label': _('HiveServer2 URL'),
  956. 'value': "",
  957. 'help_text': _('e.g. jdbc:hive2://localhost:10000/default. JDBC URL for the Hive Server 2.'),
  958. 'type': ''
  959. },
  960. 'password': {
  961. 'name': 'password',
  962. 'label': _('Password'),
  963. 'value': '',
  964. 'help_text': _('The password element must contain the password of the current user. However, the password is only used if Hive Server 2 is backed by '
  965. 'something requiring a password (e.g. LDAP); non-secured Hive Server 2 or Kerberized Hive Server 2 don\'t require a password.'),
  966. 'type': ''
  967. },
  968. 'files': {
  969. 'name': 'files',
  970. 'label': _('Files'),
  971. 'value': [],
  972. 'help_text': _('Files put in the running directory.'),
  973. 'type': ''
  974. },
  975. 'archives': {
  976. 'name': 'archives',
  977. 'label': _('Archives'),
  978. 'value': [],
  979. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  980. 'type': ''
  981. },
  982. 'job_properties': {
  983. 'name': 'job_properties',
  984. 'label': _('Hadoop job properties'),
  985. 'value': [],
  986. 'help_text': _('value, e.g. production'),
  987. 'type': ''
  988. },
  989. 'prepares': {
  990. 'name': 'prepares',
  991. 'label': _('Prepares'),
  992. 'value': [],
  993. 'help_text': _('Path to manipulate before starting the application.'),
  994. 'type': ''
  995. },
  996. 'job_xml': {
  997. 'name': 'job_xml',
  998. 'label': _('Job XML'),
  999. 'value': '',
  1000. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  1001. 'type': ''
  1002. },
  1003. 'retry_max': {
  1004. 'name': 'retry_max',
  1005. 'label': _('Max retry'),
  1006. 'value': [],
  1007. 'help_text': _('Number of times, default is 3'),
  1008. 'type': ''
  1009. },
  1010. 'retry_interval': {
  1011. 'name': 'retry_interval',
  1012. 'label': _('Retry interval'),
  1013. 'value': [],
  1014. 'help_text': _('Wait time in minutes, default is 10'),
  1015. 'type': ''
  1016. }
  1017. }
  1018. @classmethod
  1019. def get_mandatory_fields(cls):
  1020. return [cls.FIELDS['script_path']]
  1021. class SubWorkflowAction(Action):
  1022. TYPE = 'subworkflow'
  1023. FIELDS = {
  1024. 'workflow': {
  1025. 'name': 'workflow',
  1026. 'label': _('Sub-workflow'),
  1027. 'value': None,
  1028. 'help_text': _('The sub-workflow application to include. You must own all the sub-workflows'),
  1029. 'type': 'workflow'
  1030. },
  1031. 'propagate_configuration': {
  1032. 'name': 'propagate_configuration',
  1033. 'label': _('Propagate configuration'),
  1034. 'value': True,
  1035. 'help_text': _('If the workflow job configuration should be propagated to the child workflow.'),
  1036. 'type': ''
  1037. },
  1038. 'job_properties': {
  1039. 'name': 'job_properties',
  1040. 'label': _('Hadoop job properties'),
  1041. 'value': [],
  1042. 'help_text': _('Can be used to specify the job properties that are required to run the child workflow job.'),
  1043. 'type': ''
  1044. },
  1045. 'retry_max': {
  1046. 'name': 'retry_max',
  1047. 'label': _('Max retry'),
  1048. 'value': [],
  1049. 'help_text': _('Number of times, default is 3'),
  1050. 'type': ''
  1051. },
  1052. 'retry_interval': {
  1053. 'name': 'retry_interval',
  1054. 'label': _('Retry interval'),
  1055. 'value': [],
  1056. 'help_text': _('Wait time in minutes, default is 10'),
  1057. 'type': ''
  1058. }
  1059. }
  1060. @classmethod
  1061. def get_mandatory_fields(cls):
  1062. return [cls.FIELDS['workflow']]
  1063. class SqoopAction(Action):
  1064. TYPE = 'sqoop'
  1065. FIELDS = {
  1066. 'command': {
  1067. 'name': 'command',
  1068. 'label': _('Sqoop command'),
  1069. 'value': 'import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir hdfs://localhost:8020/user/foo -m 1',
  1070. 'help_text': _('The full %(type)s command. Either put it here or split it by spaces and insert the parts as multiple parameters below.') % {'type': TYPE},
  1071. 'type': 'textarea'
  1072. },
  1073. 'arguments': {
  1074. 'name': 'arguments',
  1075. 'label': _('Arguments'),
  1076. 'value': [],
  1077. 'help_text': _('If no command is specified, split the command by spaces and insert the %(type)s parameters '
  1078. 'here e.g. import, --connect, jdbc:hsqldb:file:db.hsqldb, ...') % {'type': TYPE},
  1079. 'type': ''
  1080. },
  1081. # Common
  1082. 'files': {
  1083. 'name': 'files',
  1084. 'label': _('Files'),
  1085. 'value': [],
  1086. 'help_text': _('Files put in the running directory.'),
  1087. 'type': ''
  1088. },
  1089. 'archives': {
  1090. 'name': 'archives',
  1091. 'label': _('Archives'),
  1092. 'value': [],
  1093. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  1094. 'type': ''
  1095. },
  1096. 'job_properties': {
  1097. 'name': 'job_properties',
  1098. 'label': _('Hadoop job properties'),
  1099. 'value': [],
  1100. 'help_text': _('value, e.g. production'),
  1101. 'type': ''
  1102. },
  1103. 'prepares': {
  1104. 'name': 'prepares',
  1105. 'label': _('Prepares'),
  1106. 'value': [],
  1107. 'help_text': _('Path to manipulate before starting the application.'),
  1108. 'type': ''
  1109. },
  1110. 'job_xml': {
  1111. 'name': 'job_xml',
  1112. 'label': _('Job XML'),
  1113. 'value': '',
  1114. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  1115. 'type': ''
  1116. },
  1117. 'retry_max': {
  1118. 'name': 'retry_max',
  1119. 'label': _('Max retry'),
  1120. 'value': [],
  1121. 'help_text': _('Number of times, default is 3'),
  1122. 'type': ''
  1123. },
  1124. 'retry_interval': {
  1125. 'name': 'retry_interval',
  1126. 'label': _('Retry interval'),
  1127. 'value': [],
  1128. 'help_text': _('Wait time in minutes, default is 10'),
  1129. 'type': ''
  1130. }
  1131. }
  1132. @classmethod
  1133. def get_mandatory_fields(cls):
  1134. return [cls.FIELDS['command']]
  1135. class MapReduceAction(Action):
  1136. TYPE = 'mapreduce'
  1137. FIELDS = {
  1138. 'jar_path': {
  1139. 'name': 'jar_path',
  1140. 'label': _('Jar name'),
  1141. 'value': '',
  1142. 'help_text': _('Path to the jar on HDFS.'),
  1143. 'type': ''
  1144. },
  1145. # Common
  1146. 'files': {
  1147. 'name': 'files',
  1148. 'label': _('Files'),
  1149. 'value': [],
  1150. 'help_text': _('Files put in the running directory.'),
  1151. 'type': ''
  1152. },
  1153. 'archives': {
  1154. 'name': 'archives',
  1155. 'label': _('Archives'),
  1156. 'value': [],
  1157. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  1158. 'type': ''
  1159. },
  1160. 'job_properties': {
  1161. 'name': 'job_properties',
  1162. 'label': _('Hadoop job properties'),
  1163. 'value': [],
  1164. 'help_text': _('value, e.g. production'),
  1165. 'type': ''
  1166. },
  1167. 'prepares': {
  1168. 'name': 'prepares',
  1169. 'label': _('Prepares'),
  1170. 'value': [],
  1171. 'help_text': _('Path to manipulate before starting the application.'),
  1172. 'type': ''
  1173. },
  1174. 'job_xml': {
  1175. 'name': 'job_xml',
  1176. 'label': _('Job XML'),
  1177. 'value': '',
  1178. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  1179. 'type': ''
  1180. },
  1181. 'retry_max': {
  1182. 'name': 'retry_max',
  1183. 'label': _('Max retry'),
  1184. 'value': [],
  1185. 'help_text': _('Number of times, default is 3'),
  1186. 'type': ''
  1187. },
  1188. 'retry_interval': {
  1189. 'name': 'retry_interval',
  1190. 'label': _('Retry interval'),
  1191. 'value': [],
  1192. 'help_text': _('Wait time in minutes, default is 10'),
  1193. 'type': ''
  1194. }
  1195. }
  1196. @classmethod
  1197. def get_mandatory_fields(cls):
  1198. return [cls.FIELDS['jar_path']]
  1199. class ShellAction(Action):
  1200. TYPE = 'shell'
  1201. FIELDS = {
  1202. 'shell_command': {
  1203. 'name': 'shell_command',
  1204. 'label': _('Shell command'),
  1205. 'value': '',
  1206. 'help_text': _('Shell command to execute, e.g script.sh'),
  1207. 'type': ''
  1208. },
  1209. 'arguments': {
  1210. 'name': 'arguments',
  1211. 'label': _('Arguments'),
  1212. 'value': [],
  1213. 'help_text': _('One arg, e.g. -l, --help'),
  1214. 'type': ''
  1215. },
  1216. 'env_var': {
  1217. 'name': 'env_var',
  1218. 'label': _('Environment variables'),
  1219. 'value': [],
  1220. 'help_text': _('e.g. MAX=10 or PATH=$PATH:mypath'),
  1221. 'type': ''
  1222. },
  1223. 'capture_output': {
  1224. 'name': 'capture_output',
  1225. 'label': _('Capture output'),
  1226. 'value': True,
  1227. 'help_text': _('Capture output of the stdout of the %(program)s command execution. The %(program)s '
  1228. 'command output must be in Java Properties file format and it must not exceed 2KB. '
  1229. 'From within the workflow definition, the output of an %(program)s action node is accessible '
  1230. 'via the String action:output(String node, String key) function') % {'program': TYPE},
  1231. 'type': ''
  1232. },
  1233. # Common
  1234. 'files': {
  1235. 'name': 'files',
  1236. 'label': _('Files'),
  1237. 'value': [],
  1238. 'help_text': _('Files put in the running directory.'),
  1239. 'type': ''
  1240. },
  1241. 'archives': {
  1242. 'name': 'archives',
  1243. 'label': _('Archives'),
  1244. 'value': [],
  1245. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  1246. 'type': ''
  1247. },
  1248. 'job_properties': {
  1249. 'name': 'job_properties',
  1250. 'label': _('Hadoop job properties'),
  1251. 'value': [],
  1252. 'help_text': _('value, e.g. production'),
  1253. 'type': ''
  1254. },
  1255. 'prepares': {
  1256. 'name': 'prepares',
  1257. 'label': _('Prepares'),
  1258. 'value': [],
  1259. 'help_text': _('Path to manipulate before starting the application.'),
  1260. 'type': ''
  1261. },
  1262. 'job_xml': {
  1263. 'name': 'job_xml',
  1264. 'label': _('Job XML'),
  1265. 'value': '',
  1266. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  1267. 'type': ''
  1268. },
  1269. 'retry_max': {
  1270. 'name': 'retry_max',
  1271. 'label': _('Max retry'),
  1272. 'value': [],
  1273. 'help_text': _('Number of times, default is 3'),
  1274. 'type': ''
  1275. },
  1276. 'retry_interval': {
  1277. 'name': 'retry_interval',
  1278. 'label': _('Retry interval'),
  1279. 'value': [],
  1280. 'help_text': _('Wait time in minutes, default is 10'),
  1281. 'type': ''
  1282. }
  1283. }
  1284. @classmethod
  1285. def get_mandatory_fields(cls):
  1286. return [cls.FIELDS['shell_command']]
  1287. class SshAction(Action):
  1288. TYPE = 'ssh'
  1289. FIELDS = {
  1290. 'host': {
  1291. 'name': 'host',
  1292. 'label': _('User and Host'),
  1293. 'value': 'user@host.com',
  1294. 'help_text': _('Where the shell will be executed.'),
  1295. 'type': 'text'
  1296. },
  1297. 'ssh_command': {
  1298. 'name': 'ssh_command',
  1299. 'label': _('Ssh command'),
  1300. 'value': 'ls',
  1301. 'help_text': _('The path of the Shell command to execute.'),
  1302. 'type': 'textarea'
  1303. },
  1304. 'arguments': {
  1305. 'name': 'arguments',
  1306. 'label': _('Arguments'),
  1307. 'value': [],
  1308. 'help_text': _('One arg, e.g. -l, --help'),
  1309. 'type': ''
  1310. },
  1311. 'capture_output': {
  1312. 'name': 'capture_output',
  1313. 'label': _('Capture output'),
  1314. 'value': True,
  1315. 'help_text': _('Capture output of the stdout of the %(program)s command execution. The %(program)s '
  1316. 'command output must be in Java Properties file format and it must not exceed 2KB. '
  1317. 'From within the workflow definition, the output of an %(program)s action node is accessible '
  1318. 'via the String action:output(String node, String key) function') % {'program': TYPE},
  1319. 'type': ''
  1320. },
  1321. # Common
  1322. 'retry_max': {
  1323. 'name': 'retry_max',
  1324. 'label': _('Max retry'),
  1325. 'value': [],
  1326. 'help_text': _('Number of times, default is 3'),
  1327. 'type': ''
  1328. },
  1329. 'retry_interval': {
  1330. 'name': 'retry_interval',
  1331. 'label': _('Retry interval'),
  1332. 'value': [],
  1333. 'help_text': _('Wait time in minutes, default is 10'),
  1334. 'type': ''
  1335. }
  1336. }
  1337. @classmethod
  1338. def get_mandatory_fields(cls):
  1339. return [cls.FIELDS['host'], cls.FIELDS['ssh_command']]
  1340. class FsAction(Action):
  1341. TYPE = 'fs'
  1342. FIELDS = {
  1343. 'deletes': {
  1344. 'name': 'deletes',
  1345. 'label': _('Delete path'),
  1346. 'value': [],
  1347. 'help_text': _('Deletes recursively all content.'),
  1348. 'type': ''
  1349. },
  1350. 'mkdirs': {
  1351. 'name': 'mkdirs',
  1352. 'label': _('Create directory'),
  1353. 'value': [],
  1354. 'help_text': _('Sub directories are created if needed.'),
  1355. 'type': ''
  1356. },
  1357. 'moves': {
  1358. 'name': 'moves',
  1359. 'label': _('Move file or directory'),
  1360. 'value': [],
  1361. 'help_text': _('Destination.'),
  1362. 'type': ''
  1363. },
  1364. 'chmods': {
  1365. 'name': 'chmods',
  1366. 'label': _('Change permissions'),
  1367. 'value': [],
  1368. 'help_text': _('File or directory.'),
  1369. 'type': ''
  1370. },
  1371. 'touchzs': {
  1372. 'name': 'touchzs',
  1373. 'label': _('Create or touch a file'),
  1374. 'value': [],
  1375. 'help_text': _('Or update its modification date.'),
  1376. 'type': ''
  1377. },
  1378. 'chgrps': {
  1379. 'name': 'chgrps',
  1380. 'label': _('Change the group'),
  1381. 'value': [],
  1382. 'help_text': _('File or directory.'),
  1383. 'type': ''
  1384. },
  1385. # Common
  1386. 'retry_max': {
  1387. 'name': 'retry_max',
  1388. 'label': _('Max retry'),
  1389. 'value': [],
  1390. 'help_text': _('Number of times, default is 3'),
  1391. 'type': ''
  1392. },
  1393. 'retry_interval': {
  1394. 'name': 'retry_interval',
  1395. 'label': _('Retry interval'),
  1396. 'value': [],
  1397. 'help_text': _('Wait time in minutes, default is 10'),
  1398. 'type': ''
  1399. }
  1400. }
  1401. @classmethod
  1402. def get_mandatory_fields(cls):
  1403. return []
  1404. class EmailAction(Action):
  1405. TYPE = 'email'
  1406. FIELDS = {
  1407. 'to': {
  1408. 'name': 'to',
  1409. 'label': _('To addresses'),
  1410. 'value': '',
  1411. 'help_text': _('Comma-separated values'),
  1412. 'type': 'text'
  1413. },
  1414. 'cc': {
  1415. 'name': 'cc',
  1416. 'label': _('cc'),
  1417. 'value': '',
  1418. 'help_text': _('Comma-separated values'),
  1419. 'type': 'text'
  1420. },
  1421. 'bcc': {
  1422. 'name': 'bcc',
  1423. 'label': _('bcc'),
  1424. 'value': '',
  1425. 'help_text': _('Comma-separated values'),
  1426. 'type': 'text'
  1427. },
  1428. 'subject': {
  1429. 'name': 'subject',
  1430. 'label': _('Subject'),
  1431. 'value': '',
  1432. 'help_text': _('Plain-text'),
  1433. 'type': 'text'
  1434. },
  1435. 'body': {
  1436. 'name': 'body',
  1437. 'label': _('Body'),
  1438. 'value': '',
  1439. 'help_text': _('Plain-text'),
  1440. 'type': 'textarea'
  1441. },
  1442. 'attachment': {
  1443. 'name': 'attachment',
  1444. 'label': _('Attachment'),
  1445. 'value': '',
  1446. 'help_text': _('Comma separated list of HDFS files.'),
  1447. 'type': ''
  1448. },
  1449. 'content_type': {
  1450. 'name': 'content_type',
  1451. 'label': _('Content-type'),
  1452. 'value': 'text/plain',
  1453. 'help_text': _('Default is text/plain'),
  1454. 'type': 'text'
  1455. },
  1456. # Common
  1457. 'retry_max': {
  1458. 'name': 'retry_max',
  1459. 'label': _('Max retry'),
  1460. 'value': [],
  1461. 'help_text': _('Number of times, default is 3'),
  1462. 'type': ''
  1463. },
  1464. 'retry_interval': {
  1465. 'name': 'retry_interval',
  1466. 'label': _('Retry interval'),
  1467. 'value': [],
  1468. 'help_text': _('Wait time in minutes, default is 10'),
  1469. 'type': ''
  1470. }
  1471. }
  1472. @classmethod
  1473. def get_mandatory_fields(cls):
  1474. return [cls.FIELDS['to'], cls.FIELDS['subject'], cls.FIELDS['body']]
  1475. class StreamingAction(Action):
  1476. TYPE = 'streaming'
  1477. FIELDS = {
  1478. 'mapper': {
  1479. 'name': 'mapper',
  1480. 'label': _('Mapper'),
  1481. 'value': '',
  1482. 'help_text': _('The executable/script to be used as mapper.'),
  1483. 'type': ''
  1484. },
  1485. 'reducer': {
  1486. 'name': 'reducer',
  1487. 'label': _('Reducer'),
  1488. 'value': '',
  1489. 'help_text': _('The executable/script to be used as reducer.'),
  1490. 'type': ''
  1491. },
  1492. # Common
  1493. 'files': {
  1494. 'name': 'files',
  1495. 'label': _('Files'),
  1496. 'value': [],
  1497. 'help_text': _('Files put in the running directory.')
  1498. },
  1499. 'archives': {
  1500. 'name': 'archives',
  1501. 'label': _('Archives'),
  1502. 'value': [],
  1503. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.')
  1504. },
  1505. 'job_properties': {
  1506. 'name': 'job_properties',
  1507. 'label': _('Hadoop job properties'),
  1508. 'value': [],
  1509. 'help_text': _('value, e.g. production')
  1510. },
  1511. 'prepares': {
  1512. 'name': 'prepares',
  1513. 'label': _('Prepares'),
  1514. 'value': [],
  1515. 'help_text': _('Path to manipulate before starting the application.')
  1516. },
  1517. 'job_xml': {
  1518. 'name': 'job_xml',
  1519. 'label': _('Job XML'),
  1520. 'value': '',
  1521. 'help_text': _('Refer to a Hadoop JobConf job.xml')
  1522. },
  1523. 'retry_max': {
  1524. 'name': 'retry_max',
  1525. 'label': _('Max retry'),
  1526. 'value': [],
  1527. 'help_text': _('Number of times, default is 3'),
  1528. 'type': ''
  1529. },
  1530. 'retry_interval': {
  1531. 'name': 'retry_interval',
  1532. 'label': _('Retry interval'),
  1533. 'value': [],
  1534. 'help_text': _('Wait time in minutes, default is 10'),
  1535. 'type': ''
  1536. }
  1537. }
  1538. @classmethod
  1539. def get_mandatory_fields(cls):
  1540. return [cls.FIELDS['mapper'], cls.FIELDS['reducer']]
  1541. class DistCpAction(Action):
  1542. TYPE = 'distcp'
  1543. FIELDS = {
  1544. 'distcp_parameters': {
  1545. 'name': 'distcp_parameters',
  1546. 'label': _('Arguments'),
  1547. 'value': [{'value': ''}, {'value': ''}],
  1548. 'help_text': _('Options first, then source / destination paths'),
  1549. 'type': 'distcp'
  1550. },
  1551. # Common
  1552. 'prepares': {
  1553. 'name': 'prepares',
  1554. 'label': _('Prepares'),
  1555. 'value': [],
  1556. 'help_text': _('Path to manipulate before starting the application.')
  1557. },
  1558. 'job_properties': {
  1559. 'name': 'job_properties',
  1560. 'label': _('Hadoop job properties'),
  1561. 'value': [],
  1562. 'help_text': _('value, e.g. production')
  1563. },
  1564. 'java_opts': {
  1565. 'name': 'java_opts',
  1566. 'label': _('Java options'),
  1567. 'value': '',
  1568. 'help_text': _('Parameters for the JVM, e.g. -Dprop1=a -Dprop2=b')
  1569. },
  1570. 'retry_max': {
  1571. 'name': 'retry_max',
  1572. 'label': _('Max retry'),
  1573. 'value': [],
  1574. 'help_text': _('Number of times, default is 3'),
  1575. 'type': ''
  1576. },
  1577. 'retry_interval': {
  1578. 'name': 'retry_interval',
  1579. 'label': _('Retry interval'),
  1580. 'value': [],
  1581. 'help_text': _('Wait time in minutes, default is 10'),
  1582. 'type': ''
  1583. }
  1584. }
  1585. @classmethod
  1586. def get_mandatory_fields(cls):
  1587. return [cls.FIELDS['distcp_parameters']]
  1588. class SparkAction(Action):
  1589. TYPE = 'spark'
  1590. FIELDS = {
  1591. 'spark_master': {
  1592. 'name': 'spark_master',
  1593. 'label': _('Spark Master'),
  1594. 'value': 'local[*]',
  1595. 'help_text': _('Ex: spark://host:port, mesos://host:port, yarn, or local.'),
  1596. 'type': ''
  1597. },
  1598. 'mode': {
  1599. 'name': 'mode',
  1600. 'label': _('Mode'),
  1601. 'value': 'client',
  1602. 'help_text': _('e.g. client,cluster'),
  1603. 'type': ''
  1604. },
  1605. 'app_name': {
  1606. 'name': 'app_name',
  1607. 'label': _('App name'),
  1608. 'value': 'MySpark',
  1609. 'help_text': _('The name of the spark application'),
  1610. 'type': ''
  1611. },
  1612. 'files': {
  1613. 'name': 'files',
  1614. 'label': _('Files'),
  1615. 'value': [],
  1616. 'help_text': _('Files put in the running directory.'),
  1617. 'type': ''
  1618. },
  1619. 'class': {
  1620. 'name': 'class',
  1621. 'label': _('Main class'),
  1622. 'value': '',
  1623. 'help_text': _("e.g. org.apache.spark.examples.mllib.JavaALS."),
  1624. 'type': 'text'
  1625. },
  1626. 'jars': {
  1627. 'name': 'jars',
  1628. 'label': _('Jars/py files'),
  1629. 'value': '',
  1630. 'help_text': _('Comma separated list of jars or python HDFS files.'),
  1631. 'type': ''
  1632. },
  1633. 'spark_opts': {
  1634. 'name': 'spark_opts',
  1635. 'label': _('Options list'),
  1636. 'value': '',
  1637. 'help_text': _('Ex: --executor-memory 20G --num-executors 50'),
  1638. 'type': ''
  1639. },
  1640. 'spark_arguments': {
  1641. 'name': 'spark_arguments',
  1642. 'label': _('Arguments'),
  1643. 'value': [],
  1644. 'help_text': _('Arguments, one by one, e.g. 1000, /path/a.')
  1645. },
  1646. # Common
  1647. 'job_properties': {
  1648. 'name': 'job_properties',
  1649. 'label': _('Hadoop job properties'),
  1650. 'value': [],
  1651. 'help_text': _('value, e.g. production')
  1652. },
  1653. 'prepares': {
  1654. 'name': 'prepares',
  1655. 'label': _('Prepares'),
  1656. 'value': [],
  1657. 'help_text': _('Path to manipulate before starting the application.')
  1658. },
  1659. 'job_xml': {
  1660. 'name': 'job_xml',
  1661. 'label': _('Job XML'),
  1662. 'value': '',
  1663. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  1664. 'type': ''
  1665. },
  1666. 'retry_max': {
  1667. 'name': 'retry_max',
  1668. 'label': _('Max retry'),
  1669. 'value': [],
  1670. 'help_text': _('Number of times, default is 3'),
  1671. 'type': ''
  1672. },
  1673. 'retry_interval': {
  1674. 'name': 'retry_interval',
  1675. 'label': _('Retry interval'),
  1676. 'value': [],
  1677. 'help_text': _('Wait time in minutes, default is 10'),
  1678. 'type': ''
  1679. }
  1680. }
  1681. @classmethod
  1682. def get_mandatory_fields(cls):
  1683. return [cls.FIELDS['spark_master'], cls.FIELDS['mode'], cls.FIELDS['jars']]
  1684. class KillAction(Action):
  1685. TYPE = 'kill'
  1686. FIELDS = {
  1687. 'message': {
  1688. 'name': 'message',
  1689. 'label': _('Message'),
  1690. 'value': _('Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]'),
  1691. 'help_text': _('Message to display when the workflow fails. Can contain some EL functions.'),
  1692. 'type': 'textarea'
  1693. }
  1694. }
  1695. @classmethod
  1696. def get_mandatory_fields(cls):
  1697. return [cls.FIELDS['message']]
  1698. class JoinAction(Action):
  1699. TYPE = 'join'
  1700. FIELDS = {}
  1701. @classmethod
  1702. def get_mandatory_fields(cls):
  1703. return []
  1704. class GenericAction(Action):
  1705. TYPE = 'generic'
  1706. FIELDS = {
  1707. 'xml': {
  1708. 'name': 'xml',
  1709. 'label': _('XML of the action'),
  1710. 'value': '<my_action>\n</my_action>',
  1711. 'help_text': _('Insert verbatim the XML of the action to insert into the workflow.'),
  1712. 'type': 'textarea'
  1713. }
  1714. }
  1715. @classmethod
  1716. def get_mandatory_fields(cls):
  1717. return [cls.FIELDS['xml']]
  1718. class ForkNode(Action):
  1719. TYPE = 'fork'
  1720. FIELDS = {}
  1721. @classmethod
  1722. def get_mandatory_fields(cls):
  1723. return []
  1724. class HiveDocumentAction(Action):
  1725. TYPE = 'hive-document'
  1726. DEFAULT_CREDENTIALS = 'hive2'
  1727. FIELDS = {
  1728. 'uuid': {
  1729. 'name': 'uuid',
  1730. 'label': _('Hive query'),
  1731. 'value': '',
  1732. 'help_text': _('Select a saved Hive query you want to schedule.'),
  1733. 'type': 'hive'
  1734. },
  1735. 'parameters': {
  1736. 'name': 'parameters',
  1737. 'label': _('Parameters'),
  1738. 'value': [],
  1739. 'help_text': _('The %(type)s parameters of the script. E.g. N=5, INPUT=${inputDir}') % {'type': TYPE.title()},
  1740. 'type': ''
  1741. },
  1742. # Common
  1743. 'jdbc_url': {
  1744. 'name': 'jdbc_url',
  1745. 'label': _('HiveServer2 URL'),
  1746. 'value': "",
  1747. 'help_text': _('e.g. jdbc:hive2://localhost:10000/default. JDBC URL for the Hive Server 2.'),
  1748. 'type': ''
  1749. },
  1750. 'password': {
  1751. 'name': 'password',
  1752. 'label': _('Password'),
  1753. 'value': '',
  1754. 'help_text': _('The password element must contain the password of the current user. However, the password is only used if Hive Server 2 is backed by '
  1755. 'something requiring a password (e.g. LDAP); non-secured Hive Server 2 or Kerberized Hive Server 2 don\'t require a password.'),
  1756. 'type': ''
  1757. },
  1758. 'files': {
  1759. 'name': 'files',
  1760. 'label': _('Files'),
  1761. 'value': [],
  1762. 'help_text': _('Files put in the running directory.'),
  1763. 'type': ''
  1764. },
  1765. 'archives': {
  1766. 'name': 'archives',
  1767. 'label': _('Archives'),
  1768. 'value': [],
  1769. 'help_text': _('zip, tar and tgz/tar.gz uncompressed into the running directory.'),
  1770. 'type': ''
  1771. },
  1772. 'job_properties': {
  1773. 'name': 'job_properties',
  1774. 'label': _('Hadoop job properties'),
  1775. 'value': [],
  1776. 'help_text': _('value, e.g. production'),
  1777. 'type': ''
  1778. },
  1779. 'prepares': {
  1780. 'name': 'prepares',
  1781. 'label': _('Prepares'),
  1782. 'value': [],
  1783. 'help_text': _('Path to manipulate before starting the application.'),
  1784. 'type': ''
  1785. },
  1786. 'job_xml': {
  1787. 'name': 'job_xml',
  1788. 'label': _('Job XML'),
  1789. 'value': '',
  1790. 'help_text': _('Refer to a Hadoop JobConf job.xml'),
  1791. 'type': ''
  1792. },
  1793. 'retry_max': {
  1794. 'name': 'retry_max',
  1795. 'label': _('Max retry'),
  1796. 'value': [],
  1797. 'help_text': _('Number of times, default is 3'),
  1798. 'type': ''
  1799. },
  1800. 'retry_interval': {
  1801. 'name': 'retry_interval',
  1802. 'label': _('Retry interval'),
  1803. 'value': [],
  1804. 'help_text': _('Wait time in minutes, default is 10'),
  1805. 'type': ''
  1806. }
  1807. }
  1808. @classmethod
  1809. def get_mandatory_fields(cls):
  1810. return [cls.FIELDS['uuid']]
  1811. class DecisionNode(Action):
  1812. TYPE = 'decision'
  1813. FIELDS = {}
  1814. @classmethod
  1815. def get_mandatory_fields(cls):
  1816. return []
  1817. NODES = {
  1818. 'start-widget': StartNode,
  1819. 'end-widget': EndNode,
  1820. 'pig-widget': PigAction,
  1821. 'java-widget': JavaAction,
  1822. 'hive-widget': HiveAction,
  1823. 'hive2-widget': HiveServer2Action,
  1824. 'sqoop-widget': SqoopAction,
  1825. 'mapreduce-widget': MapReduceAction,
  1826. 'subworkflow-widget': SubWorkflowAction,
  1827. 'shell-widget': ShellAction,
  1828. 'ssh-widget': SshAction,
  1829. 'fs-widget': FsAction,
  1830. 'email-widget': EmailAction,
  1831. 'streaming-widget': StreamingAction,
  1832. 'distcp-widget': DistCpAction,
  1833. 'kill-widget': KillAction,
  1834. 'join-widget': JoinAction,
  1835. 'fork-widget': ForkNode,
  1836. 'decision-widget': DecisionNode,
  1837. 'spark-widget': SparkAction,
  1838. 'generic-widget': GenericAction,
  1839. 'hive-document-widget': HiveDocumentAction
  1840. }
  1841. WORKFLOW_NODE_PROPERTIES = {}
  1842. for node in NODES.itervalues():
  1843. WORKFLOW_NODE_PROPERTIES.update(node.FIELDS)
  1844. def find_parameters(instance, fields=None):
  1845. """Find parameters in the given fields"""
  1846. if fields is None:
  1847. fields = NODES['%s-widget' % instance.data['type']].FIELDS.keys()
  1848. params = []
  1849. for field in fields:
  1850. data = instance.data['properties'][field]
  1851. if field == 'sla' and not instance.sla_enabled:
  1852. continue
  1853. if isinstance(data, list):
  1854. params.extend(find_json_parameters(data))
  1855. elif isinstance(data, basestring):
  1856. for match in Template.pattern.finditer(data):
  1857. name = match.group('braced')
  1858. if name is not None:
  1859. params.append(name)
  1860. return params
  1861. def find_json_parameters(fields):
  1862. # Input is list of json dict
  1863. params = []
  1864. for field in fields:
  1865. for data in field.values():
  1866. if isinstance(data, basestring):
  1867. for match in Template.pattern.finditer(data):
  1868. name = match.group('braced')
  1869. if name is not None:
  1870. params.append(name)
  1871. return params
  1872. def find_dollar_variables(text):
  1873. return re.findall('[^\n\\\\]\$([^\{ \'\"\-;\(\)]+)', text, re.MULTILINE)
  1874. def find_dollar_braced_variables(text):
  1875. vars = set()
  1876. for var in re.findall('\$\{([A-Za-z0-9:_-]+)\}', text, re.MULTILINE):
  1877. if ':' in var:
  1878. var = var.split(':', 1)[1]
  1879. vars.add(var)
  1880. return list(vars)
  1881. def import_workflow_from_hue_3_7(old_wf):
  1882. """
  1883. Example of data to transform
  1884. [<Start: start>, <Pig: Pig>, [<Kill: kill>], [<End: end>]]
  1885. [<Start: start>, <Java: TeraGenWorkflow>, <Java: TeraSort>, [<Kill: kill>], [<End: end>]]
  1886. [<Start: start>, [<Fork: fork-34>, [[<Mapreduce: Sleep-1>, <Mapreduce: Sleep-10>], [<Mapreduce: Sleep-5>, [<Fork: fork-38>, [[<Mapreduce: Sleep-3>], [<Mapreduce: Sleep-4>]], <Join: join-39>]]], <Join: join-35>], [<Kill: kill>], [<End: end>]]
  1887. """
  1888. uuids = {}
  1889. old_nodes = old_wf.get_hierarchy()
  1890. wf = Workflow()
  1891. wf_rows = []
  1892. wf_nodes = []
  1893. data = wf.get_data()
  1894. # UUIDs node mapping
  1895. for node in old_wf.node_list:
  1896. if node.name == 'kill':
  1897. node_uuid = '17c9c895-5a16-7443-bb81-f34b30b21548'
  1898. elif node.name == 'start':
  1899. node_uuid = '3f107997-04cc-8733-60a9-a4bb62cebffc'
  1900. elif node.name == 'end':
  1901. node_uuid = '33430f0f-ebfa-c3ec-f237-3e77efa03d0a'
  1902. else:
  1903. node_uuid = str(uuid.uuid4())
  1904. uuids[node.id] = node_uuid
  1905. # Workflow
  1906. data['workflow']['uuid'] = str(uuid.uuid4())
  1907. data['workflow']['name'] = old_wf.name
  1908. data['workflow']['properties']['properties'] = json.loads(old_wf.job_properties)
  1909. data['workflow']['properties']['job_xml'] = old_wf.job_xml
  1910. data['workflow']['properties']['description'] = old_wf.description
  1911. data['workflow']['properties']['schema_version'] = old_wf.schema_version
  1912. data['workflow']['properties']['deployment_dir'] = old_wf.deployment_dir
  1913. data['workflow']['properties']['parameters'] = json.loads(old_wf.parameters)
  1914. data['workflow']['properties']['description'] = old_wf.description
  1915. data['workflow']['properties']['sla'] = old_wf.sla
  1916. data['workflow']['properties']['sla_enabled'] = old_wf.sla_enabled
  1917. data['workflow']['properties']['imported'] = True
  1918. data['workflow']['properties']['wf1_id'] = old_wf.id
  1919. # Layout
  1920. rows = data['layout'][0]['rows']
  1921. def _create_layout(nodes, size=12):
  1922. wf_rows = []
  1923. for node in nodes:
  1924. if type(node) == list and len(node) == 1:
  1925. node = node[0]
  1926. if type(node) != list:
  1927. wf_rows.append({"widgets":[{"size":size, "name": node.name.title(), "id": uuids[node.id], "widgetType": "%s-widget" % node.node_type, "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size, "columns":[]}]})
  1928. else:
  1929. if node[0].node_type == 'fork':
  1930. wf_rows.append({"widgets":[{"size":size, "name": 'Fork', "id": uuids[node[0].id], "widgetType": "%s-widget" % node[0].node_type, "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size, "columns":[]}]})
  1931. wf_rows.append({
  1932. "id": str(uuid.uuid4()),
  1933. "widgets":[
  1934. ],
  1935. "columns":[
  1936. {
  1937. "id": str(uuid.uuid4()),
  1938. "size": (size / len(node[1])),
  1939. "rows":
  1940. [{
  1941. "id": str(uuid.uuid4()),
  1942. "widgets": c['widgets'],
  1943. "columns":[]
  1944. }
  1945. for c in col] if type(col) == list else [{
  1946. "id": str(uuid.uuid4()),
  1947. "widgets": col['widgets'],
  1948. "columns":[]
  1949. }
  1950. ]
  1951. ,
  1952. "klass":"card card-home card-column span%s" % (size / len(node[1]))
  1953. }
  1954. for col in _create_layout(node[1], size)
  1955. ]
  1956. })
  1957. wf_rows.append({"widgets":[{"size":size, "name": 'Join', "id": uuids[node[2].id], "widgetType": "%s-widget" % node[2].node_type, "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span%s" % size, "columns":[]}]})
  1958. else:
  1959. wf_rows.append(_create_layout(node, size))
  1960. return wf_rows
  1961. wf_rows = _create_layout(old_nodes)
  1962. if wf_rows:
  1963. data['layout'][0]['rows'] = [data['layout'][0]['rows'][0]] + wf_rows + [data['layout'][0]['rows'][-1]]
  1964. # Content
  1965. def _dig_nodes(nodes):
  1966. for node in nodes:
  1967. if type(node) != list:
  1968. properties = {}
  1969. if '%s-widget' % node.node_type in NODES:
  1970. properties = dict(NODES['%s-widget' % node.node_type].get_fields())
  1971. if node.node_type == 'pig':
  1972. properties['script_path'] = node.script_path
  1973. properties['parameters'] = [param for param in json.loads(node.params) if param['value'] != '-param']
  1974. properties['files'] = [{'value': f} for f in json.loads(node.files)]
  1975. properties['archives'] = json.loads(node.archives)
  1976. properties['job_properties'] = json.loads(node.job_properties)
  1977. properties['prepares'] = json.loads(node.prepares)
  1978. properties['job_xml'] = node.job_xml
  1979. properties['description'] = node.description
  1980. properties['sla'] = node.sla
  1981. properties['sla_enabled'] = node.sla_enabled
  1982. elif node.node_type == 'hive':
  1983. properties['script_path'] = node.script_path
  1984. properties['parameters'] = [param for param in json.loads(node.params) if param['value'] != '-param']
  1985. properties['files'] = [{'value': f} for f in json.loads(node.files)]
  1986. properties['archives'] = json.loads(node.archives)
  1987. properties['job_properties'] = json.loads(node.job_properties)
  1988. properties['prepares'] = json.loads(node.prepares)
  1989. properties['hive_xml'] = node.job_xml
  1990. properties['description'] = node.description
  1991. properties['sla'] = node.sla
  1992. properties['sla_enabled'] = node.sla_enabled
  1993. elif node.node_type == 'java':
  1994. properties['jar_path'] = node.jar_path
  1995. properties['main_class'] = node.main_class
  1996. properties['arguments'] = [{'value': arg} for arg in node.args.split(' ')]
  1997. properties['java_opts'] = node.java_opts
  1998. properties['capture_output'] = node.capture_output
  1999. properties['files'] = [{'value': f} for f in json.loads(node.files)]
  2000. properties['archives'] = json.loads(node.archives)
  2001. properties['job_properties'] = json.loads(node.job_properties)
  2002. properties['prepares'] = json.loads(node.prepares)
  2003. properties['job_xml'] = node.job_xml
  2004. properties['description'] = node.description
  2005. properties['sla'] = node.sla
  2006. properties['sla_enabled'] = node.sla_enabled
  2007. elif node.node_type == 'sqoop':
  2008. properties['command'] = node.script_path
  2009. properties['parameters'] = json.loads(node.params)
  2010. properties['files'] = [{'value': f} for f in json.loads(node.files)]
  2011. properties['archives'] = json.loads(node.archives)
  2012. properties['job_properties'] = json.loads(node.job_properties)
  2013. properties['prepares'] = json.loads(node.prepares)
  2014. properties['job_xml'] = node.job_xml
  2015. properties['description'] = node.description
  2016. properties['sla'] = node.sla
  2017. properties['sla_enabled'] = node.sla_enabled
  2018. elif node.node_type == 'mapreduce':
  2019. properties['jar_path'] = node.jar_path
  2020. properties['files'] = [{'value': f} for f in json.loads(node.files)]
  2021. properties['archives'] = json.loads(node.archives)
  2022. properties['job_properties'] = json.loads(node.job_properties)
  2023. properties['prepares'] = json.loads(node.prepares)
  2024. properties['job_xml'] = node.job_xml
  2025. properties['description'] = node.description
  2026. properties['sla'] = node.sla
  2027. properties['sla_enabled'] = node.sla_enabled
  2028. elif node.node_type == 'shell':
  2029. properties['shell_command'] = node.command
  2030. properties['arguments'] = json.loads(node.params)
  2031. properties['capture_output'] = node.capture_output
  2032. properties['files'] = [{'value': f} for f in json.loads(node.files)]
  2033. properties['archives'] = json.loads(node.archives)
  2034. properties['job_properties'] = json.loads(node.job_properties)
  2035. properties['prepares'] = json.loads(node.prepares)
  2036. properties['job_xml'] = node.job_xml
  2037. properties['description'] = node.description
  2038. properties['sla'] = node.sla
  2039. properties['sla_enabled'] = node.sla_enabled
  2040. elif node.node_type == 'ssh':
  2041. properties['user'] = '%s@%s' % (node.user, node.host)
  2042. properties['ssh_command'] = node.command
  2043. properties['params'] = json.loads(node.params)
  2044. properties['capture_output'] = node.capture_output
  2045. properties['description'] = node.description
  2046. properties['sla'] = node.sla
  2047. properties['sla_enabled'] = node.sla_enabled
  2048. elif node.node_type == 'fs':
  2049. properties['deletes'] = [{'value': f['name']} for f in json.loads(node.deletes)]
  2050. properties['mkdirs'] = [{'value': f['name']} for f in json.loads(node.mkdirs)]
  2051. properties['moves'] = json.loads(node.moves)
  2052. chmods = json.loads(node.chmods)
  2053. for c in chmods:
  2054. c['value'] = c['path']
  2055. c['dir_files'] = False
  2056. properties['chmods'] = chmods
  2057. properties['touchzs'] = [{'value': f['name']} for f in json.loads(node.touchzs)]
  2058. properties['description'] = node.description
  2059. properties['sla'] = node.sla
  2060. properties['sla_enabled'] = node.sla_enabled
  2061. elif node.node_type == 'email':
  2062. properties['to'] = node.to
  2063. properties['cc'] = node.cc
  2064. properties['subject'] = node.subject
  2065. properties['body'] = node.body
  2066. properties['description'] = node.description
  2067. properties['sla'] = node.sla
  2068. properties['sla_enabled'] = node.sla_enabled
  2069. elif node.node_type == 'streaming':
  2070. properties['mapper'] = node.mapper
  2071. properties['reducer'] = node.reducer
  2072. properties['files'] = [{'value': f} for f in json.loads(node.files)]
  2073. properties['archives'] = json.loads(node.archives)
  2074. properties['job_properties'] = json.loads(node.job_properties)
  2075. properties['prepares'] = json.loads(node.prepares)
  2076. properties['job_xml'] = node.job_xml
  2077. properties['description'] = node.description
  2078. properties['sla'] = node.sla
  2079. properties['sla_enabled'] = node.sla_enabled
  2080. elif node.node_type == 'distcp':
  2081. properties['distcp_parameters'] = json.loads(node.params)
  2082. properties['java_opts'] = node.job_xml
  2083. properties['job_properties'] = json.loads(node.job_properties)
  2084. properties['prepares'] = json.loads(node.prepares)
  2085. properties['description'] = node.description
  2086. properties['sla'] = node.sla
  2087. properties['sla_enabled'] = node.sla_enabled
  2088. wf_nodes.append({
  2089. "id": uuids[node.id],
  2090. "name": '%s-%s' % (node.node_type.split('-')[0], uuids[node.id][:4]),
  2091. "type": "%s-widget" % node.node_type,
  2092. "properties": properties,
  2093. "children":[{('to' if link.name in ('ok', 'start') else link.name): uuids[link.child.get_full_node().id]} for link in node.get_children_links()]
  2094. })
  2095. else:
  2096. _dig_nodes(node)
  2097. _dig_nodes(old_nodes)
  2098. data['workflow']['nodes'] = wf_nodes
  2099. return Workflow(data=json.dumps(data))
  2100. class Coordinator(Job):
  2101. XML_FILE_NAME = 'coordinator.xml'
  2102. PROPERTY_APP_PATH = 'oozie.coord.application.path'
  2103. HUE_ID = 'hue-id-c'
  2104. def __init__(self, data=None, json_data=None, document=None):
  2105. self.document = document
  2106. if document is not None:
  2107. self._data = json.loads(document.data)
  2108. elif json_data is not None:
  2109. self._data = json.loads(json_data)
  2110. elif data is not None:
  2111. self._data = data
  2112. else:
  2113. self._data = {
  2114. 'id': None,
  2115. 'uuid': None,
  2116. 'name': 'My Coordinator',
  2117. 'variables': [], # Aka workflow parameters
  2118. 'properties': {
  2119. 'description': '',
  2120. 'deployment_dir': '',
  2121. 'schema_version': 'uri:oozie:coordinator:0.2',
  2122. 'frequency_number': 1,
  2123. 'frequency_unit': 'days',
  2124. 'cron_frequency': '0 0 * * *',
  2125. 'cron_advanced': False,
  2126. 'timezone': '',
  2127. 'start': '${start_date}',
  2128. 'end': '${end_date}',
  2129. 'workflow': None,
  2130. 'timeout': None,
  2131. 'concurrency': None,
  2132. 'execution': None,
  2133. 'throttle': None,
  2134. 'job_xml': '',
  2135. 'credentials': [],
  2136. 'parameters': [
  2137. {'name': 'oozie.use.system.libpath', 'value': True},
  2138. {'name': 'start_date', 'value': datetime.today().strftime('%Y-%m-%dT%H:%M')},
  2139. {'name': 'end_date', 'value': (datetime.today() + timedelta(days=7)).strftime('%Y-%m-%dT%H:%M')}
  2140. ],
  2141. 'sla': WorkflowConfiguration.SLA_DEFAULT
  2142. }
  2143. }
  2144. @property
  2145. def id(self):
  2146. return self.document.id
  2147. @property
  2148. def uuid(self):
  2149. return self.document.uuid
  2150. def get_data_for_json(self):
  2151. _data = self.data.copy()
  2152. start_date = filter(lambda a: a['name'] == 'start_date', self._data['properties']['parameters'])
  2153. if start_date and type(start_date[0]['value']) == datetime:
  2154. start_date[0]['value'] = start_date[0]['value'].strftime('%Y-%m-%dT%H:%M:%S')
  2155. end_date = filter(lambda a: a['name'] == 'end_date', self._data['properties']['parameters'])
  2156. if end_date and type(end_date[0]['value']) == datetime:
  2157. end_date[0]['value'] = end_date[0]['value'].strftime('%Y-%m-%dT%H:%M:%S')
  2158. return _data
  2159. def to_json(self):
  2160. return json.dumps(self.get_data_for_json())
  2161. def to_json_for_html(self):
  2162. return json.dumps(self.get_data_for_json(), cls=JSONEncoderForHTML)
  2163. @property
  2164. def data(self):
  2165. if type(self._data['properties']['start']) != datetime and not '$' in self._data['properties']['start']:
  2166. self._data['properties']['start'] = parse(self._data['properties']['start'])
  2167. if type(self._data['properties']['end']) != datetime and not '$' in self._data['properties']['end']:
  2168. self._data['properties']['end'] = parse(self._data['properties']['end'])
  2169. if self.document is not None:
  2170. self._data['id'] = self.document.id
  2171. return self._data
  2172. @property
  2173. def name(self):
  2174. return self.data['name']
  2175. def set_workspace(self, user):
  2176. self.data['properties']['deployment_dir'] = Job.get_workspace(user)
  2177. @property
  2178. def deployment_dir(self):
  2179. return self.data['properties']['deployment_dir']
  2180. def find_parameters(self):
  2181. params = set()
  2182. for param in find_dollar_braced_variables(self.name):
  2183. params.add(param)
  2184. for param in find_json_parameters([self.data['properties']]):
  2185. params.add(param)
  2186. for param in find_json_parameters(self.data['variables']):
  2187. if param not in ('MINUTE', 'HOUR', 'DAY', 'MONTH', 'YEAR') and not param.startswith('coord:'):
  2188. params.add(param)
  2189. if self.sla_enabled:
  2190. for param in find_json_parameters(self.sla):
  2191. params.add(param)
  2192. # Get missed params from workflow
  2193. for prop in self.workflow.find_parameters():
  2194. if not prop in params:
  2195. params.add(prop)
  2196. # Remove the ones filled up by coordinator
  2197. removable_names = [ds['workflow_variable'] for ds in self.data['variables']]
  2198. return dict([(param, '') for param in list(params) if param not in removable_names])
  2199. @property
  2200. def sla_enabled(self):
  2201. return self.data['properties']['sla'][0].get('value')
  2202. @property
  2203. def sla(self):
  2204. return self.data['properties']['sla']
  2205. @property
  2206. def parameters(self):
  2207. return self.data['properties']['parameters']
  2208. @property
  2209. def datasets(self):
  2210. return self.inputDatasets + self.outputDatasets
  2211. @property
  2212. def inputDatasets(self):
  2213. return [Dataset(dataset, self) for dataset in self.data['variables'] if dataset['dataset_type'] == 'input_path']
  2214. @property
  2215. def outputDatasets(self):
  2216. return [Dataset(dataset, self) for dataset in self.data['variables'] if dataset['dataset_type'] == 'output_path']
  2217. @property
  2218. def start_server_tz(self):
  2219. return self.data['properties']['start']
  2220. @property
  2221. def end_server_tz(self):
  2222. return self.data['properties']['end']
  2223. @property
  2224. def frequency(self):
  2225. return '${coord:%(unit)s(%(number)d)}' % {'unit': self.data['properties']['frequency_unit'], 'number': self.data['properties']['frequency_number']}
  2226. @property
  2227. def cron_frequency(self):
  2228. data_dict = self.data['properties']
  2229. if 'cron_frequency' in data_dict:
  2230. return data_dict['cron_frequency']
  2231. else:
  2232. # Backward compatibility
  2233. freq = '0 0 * * *'
  2234. if data_dict['frequency_number'] == 1:
  2235. if data_dict['frequency_unit'] == 'minutes':
  2236. freq = '* * * * *'
  2237. elif data_dict['frequency_unit'] == 'hours':
  2238. freq = '0 * * * *'
  2239. elif data_dict['frequency_unit'] == 'days':
  2240. freq = '0 0 * * *'
  2241. elif data_dict['frequency_unit'] == 'months':
  2242. freq = '0 0 0 * *'
  2243. return {'frequency': freq, 'isAdvancedCron': False}
  2244. def to_xml(self, mapping=None):
  2245. if mapping is None:
  2246. mapping = {}
  2247. tmpl = "editor2/gen/coordinator.xml.mako"
  2248. return re.sub(re.compile('\s*\n+', re.MULTILINE), '\n', django_mako.render_to_string(tmpl, {'coord': self, 'mapping': mapping})).encode('utf-8', 'xmlcharrefreplace')
  2249. def clear_workflow_params(self):
  2250. # Repopulated in the config properties
  2251. self.data['variables'] = [dataset for dataset in self.data['variables'] if dataset['dataset_type'] != 'parameter']
  2252. @property
  2253. def properties(self):
  2254. props = [{'name': dataset['workflow_variable'], 'value': dataset['dataset_variable']} for dataset in self.data['variables'] if dataset['dataset_type'] == 'parameter']
  2255. props += self.data['properties']['parameters']
  2256. return props
  2257. @property
  2258. def workflow(self):
  2259. if self.document is None:
  2260. raise PopupException(_('Cannot return workflow since document attribute is None.'))
  2261. wf_doc = Document2.objects.get_by_uuid(user=self.document.owner, uuid=self.data['properties']['workflow'])
  2262. return Workflow(document=wf_doc)
  2263. def get_absolute_url(self):
  2264. return reverse('oozie:edit_coordinator') + '?coordinator=%s' % self.id
  2265. @classmethod
  2266. def get_application_path_key(cls):
  2267. return 'oozie.coord.application.path'
  2268. class Dataset():
  2269. def __init__(self, data, coordinator):
  2270. self._data = data
  2271. self.coordinator = coordinator
  2272. @property
  2273. def data(self):
  2274. self._data['name'] = self._data['workflow_variable']
  2275. return self._data
  2276. @property
  2277. def frequency(self):
  2278. if self.data['same_frequency']:
  2279. if self.coordinator.cron_frequency == '* * * * *':
  2280. frequency_unit = 'minutes'
  2281. elif self.coordinator.cron_frequency == '0 * * * *':
  2282. frequency_unit = 'hours'
  2283. elif self.coordinator.cron_frequency == '0 0 * * *':
  2284. frequency_unit = 'days'
  2285. elif self.coordinator.cron_frequency == '0 0 0 * *':
  2286. frequency_unit = 'months'
  2287. else:
  2288. raise PopupException(_('The frequency of the workflow parameter "%s" cannot be guessed from the frequency of the coordinator.'
  2289. ' It so needs to be specified manually.') % self.data['name'])
  2290. frequency_number = 1
  2291. else:
  2292. frequency_unit = self.data['frequency_unit']
  2293. frequency_number = self.data['frequency_number']
  2294. return '${coord:%(unit)s(%(number)s)}' % {'unit': frequency_unit, 'number': frequency_number}
  2295. @property
  2296. def start_server_tz(self):
  2297. if self.data['same_start']:
  2298. return self.coordinator.start_server_tz
  2299. else:
  2300. return convert_to_server_timezone(self.data['start'], self.data['timezone'])
  2301. @property
  2302. def timezone(self):
  2303. if self.data['same_timezone']:
  2304. return self.coordinator.data['properties']['timezone']
  2305. else:
  2306. return self.data['timezone']
  2307. @property
  2308. def start_instance(self):
  2309. if not self.is_advanced_start_instance:
  2310. return int(self.data['advanced_start_instance'])
  2311. else:
  2312. return 0
  2313. @property
  2314. def is_advanced_start_instance(self):
  2315. return not self.is_int(self.data['advanced_start_instance'])
  2316. def is_int(self, text):
  2317. try:
  2318. int(text)
  2319. return True
  2320. except ValueError:
  2321. return False
  2322. @property
  2323. def end_instance(self):
  2324. if not self.is_advanced_end_instance:
  2325. return int(self.data['advanced_end_instance'])
  2326. else:
  2327. return 0
  2328. @property
  2329. def is_advanced_end_instance(self):
  2330. return not self.is_int(self.data['advanced_end_instance'])
  2331. class Bundle(Job):
  2332. XML_FILE_NAME = 'bundle.xml'
  2333. PROPERTY_APP_PATH = 'oozie.bundle.application.path'
  2334. HUE_ID = 'hue-id-b'
  2335. def __init__(self, data=None, json_data=None, document=None):
  2336. self.document = document
  2337. if document is not None:
  2338. self._data = json.loads(document.data)
  2339. elif json_data is not None:
  2340. self._data = json.loads(json_data)
  2341. elif data is not None:
  2342. self._data = data
  2343. else:
  2344. self._data = {
  2345. 'id': None,
  2346. 'uuid': None,
  2347. 'name': 'My Bundle',
  2348. 'coordinators': [],
  2349. 'properties': {
  2350. 'description': '',
  2351. 'deployment_dir': '',
  2352. 'schema_version': 'uri:oozie:bundle:0.2',
  2353. 'kickoff': datetime.today(),
  2354. 'parameters': [{'name': 'oozie.use.system.libpath', 'value': 'true'}]
  2355. }
  2356. }
  2357. @property
  2358. def id(self):
  2359. return self.document.id
  2360. @property
  2361. def uuid(self):
  2362. return self.document.uuid
  2363. def get_data_for_json(self):
  2364. _data = self.data.copy()
  2365. _data['properties']['kickoff'] = _data['properties']['kickoff'].strftime('%Y-%m-%dT%H:%M:%S')
  2366. return _data
  2367. def to_json(self):
  2368. return json.dumps(self.get_data_for_json())
  2369. def to_json_for_html(self):
  2370. return json.dumps(self.get_data_for_json(), cls=JSONEncoderForHTML)
  2371. @property
  2372. def data(self):
  2373. if type(self._data['properties']['kickoff']) == unicode:
  2374. self._data['properties']['kickoff'] = parse(self._data['properties']['kickoff'])
  2375. if self.document is not None:
  2376. self._data['id'] = self.document.id
  2377. return self._data
  2378. def to_xml(self, mapping=None):
  2379. if mapping is None:
  2380. mapping = {}
  2381. mapping.update(dict(list(self.get_coordinator_docs().values('uuid', 'name'))))
  2382. tmpl = "editor2/gen/bundle.xml.mako"
  2383. return force_unicode(
  2384. re.sub(re.compile('\s*\n+', re.MULTILINE), '\n', django_mako.render_to_string(tmpl, {
  2385. 'bundle': self,
  2386. 'mapping': mapping
  2387. })))
  2388. def get_coordinator_docs(self):
  2389. coordinator_ids = [coordinator['coordinator'] for coordinator in self.data['coordinators']]
  2390. return Document2.objects.filter(type='oozie-coordinator2', uuid__in=coordinator_ids)
  2391. def get_coordinator_objects(self):
  2392. return [Coordinator(document=doc) for doc in self.get_coordinator_docs()]
  2393. @property
  2394. def name(self):
  2395. return self.data['name']
  2396. @property
  2397. def parameters(self):
  2398. return self.data['properties']['parameters']
  2399. @property
  2400. def kick_off_time_utc(self):
  2401. return utc_datetime_format(self.data['properties']['kickoff'])
  2402. def set_workspace(self, user):
  2403. self.data['properties']['deployment_dir'] = Job.get_workspace(user)
  2404. @property
  2405. def deployment_dir(self):
  2406. return self.data['properties']['deployment_dir']
  2407. def find_parameters(self):
  2408. params = set()
  2409. for param in find_dollar_braced_variables(self.name):
  2410. params.add(param)
  2411. for coord in self.get_coordinator_objects():
  2412. params.update(coord.find_parameters())
  2413. for param in find_json_parameters([self.data['properties']]):
  2414. params.add(param)
  2415. # Remove the ones filled up by bundle
  2416. removable_names = [p['name'] for coord in self.data['coordinators'] for p in coord['properties']]
  2417. return dict([(param, '') for param in list(params) if param not in removable_names])
  2418. def get_absolute_url(self):
  2419. return reverse('oozie:edit_bundle') + '?bundle=%s' % self.id
  2420. @classmethod
  2421. def get_application_path_key(cls):
  2422. return 'oozie.bundle.application.path'
  2423. class History(object):
  2424. @classmethod
  2425. def get_workflow_from_config(self, conf_dict):
  2426. try:
  2427. doc = Document2.objects.get(type='oozie-workflow2', id=conf_dict.get(Workflow.HUE_ID))
  2428. return Workflow(document=doc)
  2429. except Document2.DoesNotExist:
  2430. pass
  2431. @classmethod
  2432. def get_coordinator_from_config(self, conf_dict):
  2433. try:
  2434. doc = Document2.objects.get(type='oozie-coordinator2', id=conf_dict.get(Coordinator.HUE_ID))
  2435. return Coordinator(document=doc)
  2436. except Document2.DoesNotExist:
  2437. pass
  2438. @classmethod
  2439. def get_bundle_from_config(self, conf_dict):
  2440. try:
  2441. doc = Document2.objects.get(type='oozie-bundle2', id=conf_dict.get(Bundle.HUE_ID))
  2442. return Bundle(document=doc)
  2443. except Document2.DoesNotExist:
  2444. pass
  2445. class WorkflowBuilder():
  2446. """
  2447. Focus on building nodes, not the UI layout.
  2448. """
  2449. def create_hive_document_workflow(self, name, parameters, user):
  2450. api = get_oozie(user)
  2451. credentials = [HiveDocumentAction.DEFAULT_CREDENTIALS] if api.security_enabled else []
  2452. params = [{u'value': u'%s=${%s}' % (p, p)} for p in parameters]
  2453. data = json.dumps({'workflow': {
  2454. u'name': name,
  2455. u'versions': [u'uri:oozie:workflow:0.4', u'uri:oozie:workflow:0.4.5'
  2456. , u'uri:oozie:workflow:0.5'],
  2457. u'isDirty': False,
  2458. u'movedNode': None,
  2459. u'linkMapping': {
  2460. u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a': [],
  2461. u'3f107997-04cc-8733-60a9-a4bb62cebffc': [u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c'
  2462. ],
  2463. u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c': [u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a'
  2464. ],
  2465. u'17c9c895-5a16-7443-bb81-f34b30b21548': [],
  2466. },
  2467. u'nodeIds': [u'3f107997-04cc-8733-60a9-a4bb62cebffc',
  2468. u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a',
  2469. u'17c9c895-5a16-7443-bb81-f34b30b21548',
  2470. u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c'],
  2471. u'id': 47,
  2472. u'nodes': [{
  2473. u'name': u'Start',
  2474. u'properties': {},
  2475. u'actionParametersFetched': False,
  2476. u'id': u'3f107997-04cc-8733-60a9-a4bb62cebffc',
  2477. u'type': u'start-widget',
  2478. u'children': [{u'to': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c'
  2479. }],
  2480. u'actionParameters': [],
  2481. }, {
  2482. u'name': u'End',
  2483. u'properties': {},
  2484. u'actionParametersFetched': False,
  2485. u'id': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a',
  2486. u'type': u'end-widget',
  2487. u'children': [],
  2488. u'actionParameters': [],
  2489. }, {
  2490. u'name': u'Kill',
  2491. u'properties': {
  2492. u'body': u'',
  2493. u'cc': u'',
  2494. u'to': u'',
  2495. u'enableMail': False,
  2496. u'message': u'Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]'
  2497. ,
  2498. u'subject': u'',
  2499. },
  2500. u'actionParametersFetched': False,
  2501. u'id': u'17c9c895-5a16-7443-bb81-f34b30b21548',
  2502. u'type': u'kill-widget',
  2503. u'children': [],
  2504. u'actionParameters': [],
  2505. }, {
  2506. u'name': u'hive-0aec',
  2507. u'actionParametersUI': [],
  2508. u'properties': {
  2509. u'files': [],
  2510. u'job_xml': u'',
  2511. u'uuid': uuid,
  2512. u'parameters': params,
  2513. u'retry_interval': [],
  2514. u'retry_max': [],
  2515. u'job_properties': [],
  2516. u'sla': [
  2517. {u'key': u'enabled', u'value': False},
  2518. {u'key': u'nominal-time', u'value': u'${nominal_time}'},
  2519. {u'key': u'should-start', u'value': u''},
  2520. {u'key': u'should-end', u'value': u'${30 * MINUTES}'},
  2521. {u'key': u'max-duration', u'value': u''},
  2522. {u'key': u'alert-events', u'value': u''},
  2523. {u'key': u'alert-contact', u'value': u''},
  2524. {u'key': u'notification-msg', u'value': u''},
  2525. {u'key': u'upstream-apps', u'value': u''},
  2526. ],
  2527. u'archives': [],
  2528. u'prepares': [],
  2529. u'credentials': credentials,
  2530. u'password': u'',
  2531. u'jdbc_url': u'',
  2532. },
  2533. u'actionParametersFetched': False,
  2534. u'id': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c',
  2535. u'type': u'hive-document-widget',
  2536. u'children': [{u'to': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a'},
  2537. {u'error': u'17c9c895-5a16-7443-bb81-f34b30b21548'
  2538. }],
  2539. u'actionParameters': [],
  2540. }],
  2541. u'properties': {
  2542. u'job_xml': u'',
  2543. u'description': u'',
  2544. u'wf1_id': None,
  2545. u'sla_enabled': False,
  2546. u'deployment_dir': u'/user/hue/oozie/workspaces/hue-oozie-1459474214.27',
  2547. u'schema_version': u'uri:oozie:workflow:0.5',
  2548. u'sla': [
  2549. {u'key': u'enabled', u'value': False},
  2550. {u'key': u'nominal-time', u'value': u'${nominal_time}'},
  2551. {u'key': u'should-start', u'value': u''},
  2552. {u'key': u'should-end', u'value': u'${30 * MINUTES}'},
  2553. {u'key': u'max-duration', u'value': u''},
  2554. {u'key': u'alert-events', u'value': u''},
  2555. {u'key': u'alert-contact', u'value': u''},
  2556. {u'key': u'notification-msg', u'value': u''},
  2557. {u'key': u'upstream-apps', u'value': u''},
  2558. ],
  2559. u'show_arrows': True,
  2560. u'parameters': [{u'name': u'oozie.use.system.libpath',
  2561. u'value': True}],
  2562. u'properties': [],
  2563. },
  2564. u'nodeNamesMapping': {
  2565. u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a': u'End',
  2566. u'3f107997-04cc-8733-60a9-a4bb62cebffc': u'Start',
  2567. u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c': u'hive-0aec',
  2568. u'17c9c895-5a16-7443-bb81-f34b30b21548': u'Kill',
  2569. },
  2570. u'uuid': u'433922e5-e616-dfe0-1cba-7fe744c9305c',
  2571. }, 'layout': [{
  2572. u'oozieRows': [{
  2573. u'enableOozieDropOnBefore': True,
  2574. u'enableOozieDropOnSide': True,
  2575. u'enableOozieDrop': False,
  2576. u'widgets': [{
  2577. u'status': u'',
  2578. u'logsURL': u'',
  2579. u'name': u'Hive',
  2580. u'widgetType': u'hive-document-widget',
  2581. u'oozieMovable': True,
  2582. u'ooziePropertiesExpanded': False,
  2583. u'externalIdUrl': u'',
  2584. u'properties': {},
  2585. u'isLoading': True,
  2586. u'offset': 0,
  2587. u'actionURL': u'',
  2588. u'progress': 0,
  2589. u'klass': u'card card-widget span12',
  2590. u'oozieExpanded': False,
  2591. u'id': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c',
  2592. u'size': 12,
  2593. }],
  2594. u'id': u'32e1ea1a-812b-6878-9719-ff7b8407bf46',
  2595. u'columns': [],
  2596. }],
  2597. u'rows': [{
  2598. u'enableOozieDropOnBefore': True,
  2599. u'enableOozieDropOnSide': True,
  2600. u'enableOozieDrop': False,
  2601. u'widgets': [{
  2602. u'status': u'',
  2603. u'logsURL': u'',
  2604. u'name': u'Start',
  2605. u'widgetType': u'start-widget',
  2606. u'oozieMovable': False,
  2607. u'ooziePropertiesExpanded': False,
  2608. u'externalIdUrl': u'',
  2609. u'properties': {},
  2610. u'isLoading': True,
  2611. u'offset': 0,
  2612. u'actionURL': u'',
  2613. u'progress': 0,
  2614. u'klass': u'card card-widget span12',
  2615. u'oozieExpanded': False,
  2616. u'id': u'3f107997-04cc-8733-60a9-a4bb62cebffc',
  2617. u'size': 12,
  2618. }],
  2619. u'id': u'798dc16a-d366-6305-d2b3-2d5a6f6c4f4b',
  2620. u'columns': [],
  2621. }, {
  2622. u'enableOozieDropOnBefore': True,
  2623. u'enableOozieDropOnSide': True,
  2624. u'enableOozieDrop': False,
  2625. u'widgets': [{
  2626. u'status': u'',
  2627. u'logsURL': u'',
  2628. u'name': u'Hive',
  2629. u'widgetType': u'hive-document-widget',
  2630. u'oozieMovable': True,
  2631. u'ooziePropertiesExpanded': False,
  2632. u'externalIdUrl': u'',
  2633. u'properties': {},
  2634. u'isLoading': True,
  2635. u'offset': 0,
  2636. u'actionURL': u'',
  2637. u'progress': 0,
  2638. u'klass': u'card card-widget span12',
  2639. u'oozieExpanded': False,
  2640. u'id': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c',
  2641. u'size': 12,
  2642. }],
  2643. u'id': u'32e1ea1a-812b-6878-9719-ff7b8407bf46',
  2644. u'columns': [],
  2645. }, {
  2646. u'enableOozieDropOnBefore': True,
  2647. u'enableOozieDropOnSide': True,
  2648. u'enableOozieDrop': False,
  2649. u'widgets': [{
  2650. u'status': u'',
  2651. u'logsURL': u'',
  2652. u'name': u'End',
  2653. u'widgetType': u'end-widget',
  2654. u'oozieMovable': False,
  2655. u'ooziePropertiesExpanded': False,
  2656. u'externalIdUrl': u'',
  2657. u'properties': {},
  2658. u'isLoading': True,
  2659. u'offset': 0,
  2660. u'actionURL': u'',
  2661. u'progress': 0,
  2662. u'klass': u'card card-widget span12',
  2663. u'oozieExpanded': False,
  2664. u'id': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a',
  2665. u'size': 12,
  2666. }],
  2667. u'id': u'f2cf152d-8c82-2f4f-5d67-2e18c99e59c4',
  2668. u'columns': [],
  2669. }, {
  2670. u'enableOozieDropOnBefore': True,
  2671. u'enableOozieDropOnSide': True,
  2672. u'enableOozieDrop': False,
  2673. u'widgets': [{
  2674. u'status': u'',
  2675. u'logsURL': u'',
  2676. u'name': u'Kill',
  2677. u'widgetType': u'kill-widget',
  2678. u'oozieMovable': True,
  2679. u'ooziePropertiesExpanded': False,
  2680. u'externalIdUrl': u'',
  2681. u'properties': {},
  2682. u'isLoading': True,
  2683. u'offset': 0,
  2684. u'actionURL': u'',
  2685. u'progress': 0,
  2686. u'klass': u'card card-widget span12',
  2687. u'oozieExpanded': False,
  2688. u'id': u'17c9c895-5a16-7443-bb81-f34b30b21548',
  2689. u'size': 12,
  2690. }],
  2691. u'id': u'01afcf1b-fa7a-e093-b613-ce52c5531a04',
  2692. u'columns': [],
  2693. }],
  2694. u'oozieEndRow': {
  2695. u'enableOozieDropOnBefore': True,
  2696. u'enableOozieDropOnSide': True,
  2697. u'enableOozieDrop': False,
  2698. u'widgets': [{
  2699. u'status': u'',
  2700. u'logsURL': u'',
  2701. u'name': u'End',
  2702. u'widgetType': u'end-widget',
  2703. u'oozieMovable': False,
  2704. u'ooziePropertiesExpanded': False,
  2705. u'externalIdUrl': u'',
  2706. u'properties': {},
  2707. u'isLoading': True,
  2708. u'offset': 0,
  2709. u'actionURL': u'',
  2710. u'progress': 0,
  2711. u'klass': u'card card-widget span12',
  2712. u'oozieExpanded': False,
  2713. u'id': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a',
  2714. u'size': 12,
  2715. }],
  2716. u'id': u'f2cf152d-8c82-2f4f-5d67-2e18c99e59c4',
  2717. u'columns': [],
  2718. },
  2719. u'oozieKillRow': {
  2720. u'enableOozieDropOnBefore': True,
  2721. u'enableOozieDropOnSide': True,
  2722. u'enableOozieDrop': False,
  2723. u'widgets': [{
  2724. u'status': u'',
  2725. u'logsURL': u'',
  2726. u'name': u'Kill',
  2727. u'widgetType': u'kill-widget',
  2728. u'oozieMovable': True,
  2729. u'ooziePropertiesExpanded': False,
  2730. u'externalIdUrl': u'',
  2731. u'properties': {},
  2732. u'isLoading': True,
  2733. u'offset': 0,
  2734. u'actionURL': u'',
  2735. u'progress': 0,
  2736. u'klass': u'card card-widget span12',
  2737. u'oozieExpanded': False,
  2738. u'id': u'17c9c895-5a16-7443-bb81-f34b30b21548',
  2739. u'size': 12,
  2740. }],
  2741. u'id': u'01afcf1b-fa7a-e093-b613-ce52c5531a04',
  2742. u'columns': [],
  2743. },
  2744. u'enableOozieDropOnAfter': True,
  2745. u'oozieStartRow': {
  2746. u'enableOozieDropOnBefore': True,
  2747. u'enableOozieDropOnSide': True,
  2748. u'enableOozieDrop': False,
  2749. u'widgets': [{
  2750. u'status': u'',
  2751. u'logsURL': u'',
  2752. u'name': u'Start',
  2753. u'widgetType': u'start-widget',
  2754. u'oozieMovable': False,
  2755. u'ooziePropertiesExpanded': False,
  2756. u'externalIdUrl': u'',
  2757. u'properties': {},
  2758. u'isLoading': True,
  2759. u'offset': 0,
  2760. u'actionURL': u'',
  2761. u'progress': 0,
  2762. u'klass': u'card card-widget span12',
  2763. u'oozieExpanded': False,
  2764. u'id': u'3f107997-04cc-8733-60a9-a4bb62cebffc',
  2765. u'size': 12,
  2766. }],
  2767. u'id': u'798dc16a-d366-6305-d2b3-2d5a6f6c4f4b',
  2768. u'columns': [],
  2769. },
  2770. u'klass': u'card card-home card-column span12',
  2771. u'enableOozieDropOnBefore': True,
  2772. u'drops': [u'temp'],
  2773. u'id': u'672ff75a-d841-72c3-c616-c9d45ec97649',
  2774. u'size': 12,
  2775. }]}
  2776. )
  2777. workflow_doc = Document2.objects.create(name=name, type='oozie-workflow2', owner=user, data=data)
  2778. Document.objects.link(workflow_doc, owner=workflow_doc.owner, name=workflow_doc.name, description=workflow_doc.description, extra='workflow2')
  2779. return workflow_doc