|
@@ -37,6 +37,7 @@ from desktop.lib.django_test_util import make_logged_in_client
|
|
|
from desktop.lib.test_utils import grant_access, add_permission, add_to_group, reformat_json, reformat_xml
|
|
from desktop.lib.test_utils import grant_access, add_permission, add_to_group, reformat_json, reformat_xml
|
|
|
from desktop.models import Document, Document2
|
|
from desktop.models import Document, Document2
|
|
|
|
|
|
|
|
|
|
+from hadoop import cluster as originalCluster
|
|
|
from hadoop.pseudo_hdfs4 import is_live_cluster
|
|
from hadoop.pseudo_hdfs4 import is_live_cluster
|
|
|
from jobsub.models import OozieDesign, OozieMapreduceAction
|
|
from jobsub.models import OozieDesign, OozieMapreduceAction
|
|
|
from liboozie import oozie_api
|
|
from liboozie import oozie_api
|
|
@@ -65,7 +66,8 @@ class MockOozieApi:
|
|
|
{u'status': u'KILLED', u'run': 0, u'startTime': u'Mon, 30 Jul 2012 22:31:08 GMT', u'appName': u'WordCount2', u'lastModTime': u'Mon, 30 Jul 2012 22:32:20 GMT', u'actions': [], u'acl': None, u'appPath': None, u'externalId': '-', u'consoleUrl': u'http://runreal:11000/oozie?job=0000011-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Mon, 30 Jul 2012 22:31:08 GMT', u'toString': u'Workflow id[0000011-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Mon, 30 Jul 2012 22:32:20 GMT', u'id': u'0000011-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'},
|
|
{u'status': u'KILLED', u'run': 0, u'startTime': u'Mon, 30 Jul 2012 22:31:08 GMT', u'appName': u'WordCount2', u'lastModTime': u'Mon, 30 Jul 2012 22:32:20 GMT', u'actions': [], u'acl': None, u'appPath': None, u'externalId': '-', u'consoleUrl': u'http://runreal:11000/oozie?job=0000011-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Mon, 30 Jul 2012 22:31:08 GMT', u'toString': u'Workflow id[0000011-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Mon, 30 Jul 2012 22:32:20 GMT', u'id': u'0000011-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'},
|
|
|
{u'status': u'SUCCEEDED', u'run': 0, u'startTime': u'Mon, 30 Jul 2012 22:20:48 GMT', u'appName': u'WordCount3', u'lastModTime': u'Mon, 30 Jul 2012 22:22:00 GMT', u'actions': [], u'acl': None, u'appPath': None, u'externalId': '', u'consoleUrl': u'http://runreal:11000/oozie?job=0000009-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Mon, 30 Jul 2012 22:20:48 GMT', u'toString': u'Workflow id[0000009-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Mon, 30 Jul 2012 22:22:00 GMT', u'id': u'0000009-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'},
|
|
{u'status': u'SUCCEEDED', u'run': 0, u'startTime': u'Mon, 30 Jul 2012 22:20:48 GMT', u'appName': u'WordCount3', u'lastModTime': u'Mon, 30 Jul 2012 22:22:00 GMT', u'actions': [], u'acl': None, u'appPath': None, u'externalId': '', u'consoleUrl': u'http://runreal:11000/oozie?job=0000009-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Mon, 30 Jul 2012 22:20:48 GMT', u'toString': u'Workflow id[0000009-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Mon, 30 Jul 2012 22:22:00 GMT', u'id': u'0000009-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'},
|
|
|
{u'status': u'SUCCEEDED', u'run': 0, u'startTime': u'Mon, 30 Jul 2012 22:16:58 GMT', u'appName': u'WordCount4', u'lastModTime': u'Mon, 30 Jul 2012 22:18:10 GMT', u'actions': [], u'acl': None, u'appPath': None, u'externalId': None, u'consoleUrl': u'http://runreal:11000/oozie?job=0000008-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Mon, 30 Jul 2012 22:16:58 GMT', u'toString': u'Workflow id[0000008-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Mon, 30 Jul 2012 22:18:10 GMT', u'id': u'0000008-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'},
|
|
{u'status': u'SUCCEEDED', u'run': 0, u'startTime': u'Mon, 30 Jul 2012 22:16:58 GMT', u'appName': u'WordCount4', u'lastModTime': u'Mon, 30 Jul 2012 22:18:10 GMT', u'actions': [], u'acl': None, u'appPath': None, u'externalId': None, u'consoleUrl': u'http://runreal:11000/oozie?job=0000008-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Mon, 30 Jul 2012 22:16:58 GMT', u'toString': u'Workflow id[0000008-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Mon, 30 Jul 2012 22:18:10 GMT', u'id': u'0000008-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'},
|
|
|
- {u'status': u'SUCCEEDED', u'run': 0, u'startTime': u'Fri, 24 Jul 2015 14:56:08 AEST', u'appName': u'WordCount5', u'lastModTime': u'Fri, 24 Jul 2015 14:57:17 AEST', u'actions': [], u'acl': None, u'appPath': None, u'externalId': None, u'consoleUrl': u'http://runreal:11000/oozie?job=0000008-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Fri, 24 Jul 2015 14:56:08 AEST', u'toString': u'Workflow id[0000007-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Fri, 24 Jul 2015 14:57:17 AEST', u'id': u'0000007-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'}]
|
|
|
|
|
|
|
+ {u'status': u'SUCCEEDED', u'run': 0, u'startTime': u'Fri, 24 Jul 2015 14:56:08 AEST', u'appName': u'WordCount5', u'lastModTime': u'Fri, 24 Jul 2015 14:57:17 AEST', u'actions': [], u'acl': None, u'appPath': None, u'externalId': None, u'consoleUrl': u'http://runreal:11000/oozie?job=0000008-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Fri, 24 Jul 2015 14:56:08 AEST', u'toString': u'Workflow id[0000007-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Fri, 24 Jul 2015 14:57:17 AEST', u'id': u'0000007-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'},
|
|
|
|
|
+ {u'status': u'RUNNING', u'run': 0, u'startTime': u'Mon, 30 Jul 2012 22:35:48 GMT', u'appName': u'WordCount1', u'lastModTime': u'Mon, 30 Jul 2012 22:37:00 GMT', u'actions': [], u'acl': None, u'appPath': None, u'externalId': 'job_201208072118_0044', u'consoleUrl': u'http://runreal:11000/oozie?job=0000006-120725142744176-oozie-oozi-W', u'conf': None, u'parentId': None, u'createdTime': u'Mon, 30 Jul 2012 22:35:48 GMT', u'toString': u'Workflow id[0000006-120725142744176-oozie-oozi-W] status[SUCCEEDED]', u'endTime': u'Mon, 30 Jul 2012 22:37:00 GMT', u'id': u'0000006-120725142744176-oozie-oozi-W', u'group': None, u'user': u'test'}]
|
|
|
|
|
|
|
|
WORKFLOW_IDS = [wf['id'] for wf in JSON_WORKFLOW_LIST]
|
|
WORKFLOW_IDS = [wf['id'] for wf in JSON_WORKFLOW_LIST]
|
|
|
WORKFLOW_DICT = dict([(wf['id'], wf) for wf in JSON_WORKFLOW_LIST])
|
|
WORKFLOW_DICT = dict([(wf['id'], wf) for wf in JSON_WORKFLOW_LIST])
|
|
@@ -232,6 +234,27 @@ class MockOozieApi:
|
|
|
return {'status': "RUNNING"}
|
|
return {'status': "RUNNING"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+class MockFs():
|
|
|
|
|
+ def __init__(self, logical_name=None):
|
|
|
|
|
+
|
|
|
|
|
+ self.fs_defaultfs = 'hdfs://curacao:8020'
|
|
|
|
|
+ self.logical_name = logical_name if logical_name else ''
|
|
|
|
|
+
|
|
|
|
|
+ def setuser(self, user):
|
|
|
|
|
+ pass
|
|
|
|
|
+
|
|
|
|
|
+ def join(self, path1, path2):
|
|
|
|
|
+ return path1 + path2
|
|
|
|
|
+
|
|
|
|
|
+ def do_as_user(self, username, fn, *args, **kwargs):
|
|
|
|
|
+ return ''
|
|
|
|
|
+
|
|
|
|
|
+ def read(self, length=1024*1024):
|
|
|
|
|
+ return 'data'
|
|
|
|
|
+
|
|
|
|
|
+ def exists(self, path):
|
|
|
|
|
+ return True
|
|
|
|
|
+
|
|
|
class OozieMockBase(object):
|
|
class OozieMockBase(object):
|
|
|
|
|
|
|
|
def setUp(self):
|
|
def setUp(self):
|
|
@@ -249,14 +272,20 @@ class OozieMockBase(object):
|
|
|
add_to_group("test")
|
|
add_to_group("test")
|
|
|
self.user = User.objects.get(username='test')
|
|
self.user = User.objects.get(username='test')
|
|
|
self.wf = create_workflow(self.c, self.user)
|
|
self.wf = create_workflow(self.c, self.user)
|
|
|
|
|
+ self.original_fs = originalCluster.FS_CACHE["default"]
|
|
|
|
|
+ originalCluster.FS_CACHE["default"] = MockFs()
|
|
|
|
|
|
|
|
|
|
|
|
|
def tearDown(self):
|
|
def tearDown(self):
|
|
|
oozie_api.OozieApi = oozie_api.OriginalOozieApi
|
|
oozie_api.OozieApi = oozie_api.OriginalOozieApi
|
|
|
|
|
+ if originalCluster.FS_CACHE is None:
|
|
|
|
|
+ originalCluster.FS_CACHE = {}
|
|
|
|
|
+ originalCluster.FS_CACHE["default"] = self.original_fs
|
|
|
Workflow.objects.check_workspace = Workflow.objects.original_check_workspace
|
|
Workflow.objects.check_workspace = Workflow.objects.original_check_workspace
|
|
|
oozie_api._api_cache = None
|
|
oozie_api._api_cache = None
|
|
|
|
|
|
|
|
History.objects.all().delete()
|
|
History.objects.all().delete()
|
|
|
|
|
+
|
|
|
for coordinator in Coordinator.objects.all():
|
|
for coordinator in Coordinator.objects.all():
|
|
|
coordinator.delete(skip_trash=True)
|
|
coordinator.delete(skip_trash=True)
|
|
|
for bundle in Bundle.objects.all():
|
|
for bundle in Bundle.objects.all():
|
|
@@ -3283,6 +3312,15 @@ class TestDashboard(OozieMockBase):
|
|
|
response = self.c.get(reverse('oozie:rerun_oozie_coord', args=[MockOozieApi.WORKFLOW_IDS[0], '/path']))
|
|
response = self.c.get(reverse('oozie:rerun_oozie_coord', args=[MockOozieApi.WORKFLOW_IDS[0], '/path']))
|
|
|
assert_true('Rerun' in response.content, response.content)
|
|
assert_true('Rerun' in response.content, response.content)
|
|
|
|
|
|
|
|
|
|
+ def test_sync_coord_workflow(self):
|
|
|
|
|
+ wf_doc = save_temp_workflow(MockOozieApi.JSON_WORKFLOW_LIST[5], self.user)
|
|
|
|
|
+ reset = ENABLE_V2.set_for_testing(True)
|
|
|
|
|
+ try:
|
|
|
|
|
+ response = self.c.get(reverse('oozie:sync_coord_workflow', args=[MockOozieApi.WORKFLOW_IDS[5]]))
|
|
|
|
|
+ assert_equal([{'name':'Dryrun', 'value': False}, {'name':'ls_arg', 'value': '-l'}], response.context['params_form'].initial)
|
|
|
|
|
+ finally:
|
|
|
|
|
+ wf_doc.delete()
|
|
|
|
|
+ reset()
|
|
|
|
|
|
|
|
def test_rerun_coordinator_permissions(self):
|
|
def test_rerun_coordinator_permissions(self):
|
|
|
post_data = {
|
|
post_data = {
|
|
@@ -3914,3 +3952,51 @@ def synchronize_workflow_attributes(workflow_json, correct_workflow_json):
|
|
|
workflow_dict['attributes']['deployment_dir'] = correct_workflow_dict['attributes']['deployment_dir']
|
|
workflow_dict['attributes']['deployment_dir'] = correct_workflow_dict['attributes']['deployment_dir']
|
|
|
|
|
|
|
|
return reformat_json(workflow_dict)
|
|
return reformat_json(workflow_dict)
|
|
|
|
|
+
|
|
|
|
|
+def save_temp_workflow(wf, user):
|
|
|
|
|
+ data = json.dumps({
|
|
|
|
|
+ 'layout': [{
|
|
|
|
|
+ "size":12, "rows":[
|
|
|
|
|
+ {"widgets":[{"size":12, "name":"Start", "id":"3f107997-04cc-8733-60a9-a4bb62cebffc", "widgetType":"start-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]},
|
|
|
|
|
+ {"widgets":[{"size":12, "name":"End", "id":"33430f0f-ebfa-c3ec-f237-3e77efa03d0a", "widgetType":"end-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]},
|
|
|
|
|
+ {"widgets":[{"size":12, "name":"Kill", "id":"17c9c895-5a16-7443-bb81-f34b30b21548", "widgetType":"kill-widget", "properties":{}, "offset":0, "isLoading":False, "klass":"card card-widget span12"}]}
|
|
|
|
|
+ ],
|
|
|
|
|
+ "drops":[ "temp"],
|
|
|
|
|
+ "klass":"card card-home card-column span12"
|
|
|
|
|
+ }],
|
|
|
|
|
+ 'workflow': {
|
|
|
|
|
+ "id": None,
|
|
|
|
|
+ "uuid": None,
|
|
|
|
|
+ "name": "My Workflow",
|
|
|
|
|
+ "properties": {
|
|
|
|
|
+ "deployment_dir": "",
|
|
|
|
|
+ "description": "",
|
|
|
|
|
+ "job_xml": "",
|
|
|
|
|
+ "sla_enabled": False,
|
|
|
|
|
+ "schema_version": "uri:oozie:workflow:0.5",
|
|
|
|
|
+ "properties": [],
|
|
|
|
|
+ "parameters": [{'name':'Dryrun', 'value': False}, {'name':'ls_arg', 'value': '-l'}],
|
|
|
|
|
+ "sla": [
|
|
|
|
|
+ {'key': 'enabled', 'value': False}, # Always first element
|
|
|
|
|
+ {'key': 'nominal-time', 'value': '${nominal_time}'},
|
|
|
|
|
+ {'key': 'should-start', 'value': ''},
|
|
|
|
|
+ {'key': 'should-end', 'value': '${30 * MINUTES}'},
|
|
|
|
|
+ {'key': 'max-duration', 'value': ''},
|
|
|
|
|
+ {'key': 'alert-events', 'value': ''},
|
|
|
|
|
+ {'key': 'alert-contact', 'value': ''},
|
|
|
|
|
+ {'key': 'notification-msg', 'value': ''},
|
|
|
|
|
+ {'key': 'upstream-apps', 'value': ''},
|
|
|
|
|
+ ],
|
|
|
|
|
+ "show_arrows": True,
|
|
|
|
|
+ "wf1_id": None
|
|
|
|
|
+ },
|
|
|
|
|
+ "nodes":[
|
|
|
|
|
+ {"id":"3f107997-04cc-8733-60a9-a4bb62cebffc","name":"Start","type":"start-widget","properties":{},"children":[{'to': '33430f0f-ebfa-c3ec-f237-3e77efa03d0a'}]},
|
|
|
|
|
+ {"id":"33430f0f-ebfa-c3ec-f237-3e77efa03d0a","name":"End","type":"end-widget","properties":{},"children":[]},
|
|
|
|
|
+ {"id":"17c9c895-5a16-7443-bb81-f34b30b21548","name":"Kill","type":"kill-widget","properties":{'message': 'Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]'},"children":[]}
|
|
|
|
|
+ ]
|
|
|
|
|
+ }
|
|
|
|
|
+ })
|
|
|
|
|
+ workflow_doc = Document2.objects.create(name='test', type='oozie-workflow2', owner=user, data=data)
|
|
|
|
|
+ wf[u'conf'] = u'<configuration><property><name>hue-id-w</name><value>' + str(workflow_doc.id) + u'</value></property></configuration>'
|
|
|
|
|
+ return workflow_doc
|