|
|
@@ -231,7 +231,7 @@ def list_oozie_workflows(request):
|
|
|
return render('dashboard/list_oozie_workflows.mako', request, {
|
|
|
'user': request.user,
|
|
|
'jobs': [],
|
|
|
- 'has_job_edition_permission': has_job_edition_permission,
|
|
|
+ 'has_job_edition_permission': has_job_edition_permission,
|
|
|
})
|
|
|
|
|
|
|
|
|
@@ -347,8 +347,11 @@ def list_oozie_workflow(request, job_id):
|
|
|
if not hue_workflow and hue_coord and hue_coord.workflow.document:
|
|
|
hue_workflow = hue_coord.workflow
|
|
|
|
|
|
- if hue_coord and hue_coord.workflow and hue_coord.workflow.document: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user)
|
|
|
- if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user)
|
|
|
+ if hue_coord and hue_coord.workflow and hue_coord.workflow.document:
|
|
|
+ hue_coord.workflow.document.doc.get().can_read_or_exception(request.user)
|
|
|
+
|
|
|
+ if hue_workflow:
|
|
|
+ hue_workflow.document.doc.get().can_read_or_exception(request.user)
|
|
|
|
|
|
if hue_workflow:
|
|
|
full_node_list = hue_workflow.nodes
|
|
|
@@ -377,7 +380,8 @@ def list_oozie_workflow(request, job_id):
|
|
|
history = get_history().cross_reference_submission_history(request.user, job_id)
|
|
|
|
|
|
hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict)
|
|
|
- hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict)
|
|
|
+ hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or \
|
|
|
+ get_history().get_workflow_from_config(oozie_workflow.conf_dict)
|
|
|
|
|
|
if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id)
|
|
|
if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id)
|
|
|
@@ -400,7 +404,7 @@ def list_oozie_workflow(request, job_id):
|
|
|
workflow_graph = django_mako.render_to_string('dashboard/list_oozie_workflow_graph.mako', {})
|
|
|
return_obj = {
|
|
|
'id': oozie_workflow.id,
|
|
|
- 'status': oozie_workflow.status,
|
|
|
+ 'status': oozie_workflow.status,
|
|
|
'progress': oozie_workflow.get_progress(full_node_list),
|
|
|
'graph': workflow_graph,
|
|
|
'actions': massaged_workflow_actions_for_json(oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle),
|
|
|
@@ -445,7 +449,8 @@ def list_oozie_workflow(request, job_id):
|
|
|
'oozie_slas': oozie_slas,
|
|
|
'hue_workflow': hue_workflow,
|
|
|
'hue_coord': hue_coord,
|
|
|
- 'parameters': dict((var, val) for var, val in parameters.items() if var not in ParameterForm.NON_PARAMETERS and var != 'oozie.use.system.libpath' or var == 'oozie.wf.application.path'),
|
|
|
+ 'parameters': dict((var, val) for var, val in parameters.items() if var not in ParameterForm.NON_PARAMETERS and \
|
|
|
+ var != 'oozie.use.system.libpath' or var == 'oozie.wf.application.path'),
|
|
|
'has_job_edition_permission': has_job_edition_permission,
|
|
|
'workflow_graph': workflow_graph,
|
|
|
'layout_json': json.dumps(workflow_data.get('layout', ''), cls=JSONEncoderForHTML) if workflow_data else '',
|
|
|
@@ -463,7 +468,7 @@ def list_oozie_coordinator(request, job_id):
|
|
|
kwargs = {'cnt': 50, 'filters': []}
|
|
|
kwargs['offset'] = request.GET.get('offset', default=1)
|
|
|
if request.GET.getlist('status'):
|
|
|
- kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
|
|
|
+ kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
|
|
|
|
|
|
oozie_coordinator = check_job_access_permission(request, job_id, **kwargs)
|
|
|
|
|
|
@@ -487,7 +492,7 @@ def list_oozie_coordinator(request, job_id):
|
|
|
|
|
|
return_obj = {
|
|
|
'id': oozie_coordinator.id,
|
|
|
- 'status': oozie_coordinator.status,
|
|
|
+ 'status': oozie_coordinator.status,
|
|
|
'progress': oozie_coordinator.get_progress(),
|
|
|
'nextTime': format_time(oozie_coordinator.nextMaterializedTime),
|
|
|
'endTime': format_time(oozie_coordinator.endTime),
|
|
|
@@ -546,7 +551,7 @@ def list_oozie_bundle(request, job_id):
|
|
|
'id': oozie_bundle.id,
|
|
|
'user': oozie_bundle.user,
|
|
|
'name': oozie_bundle.bundleJobName,
|
|
|
- 'status': oozie_bundle.status,
|
|
|
+ 'status': oozie_bundle.status,
|
|
|
'progress': oozie_bundle.get_progress(),
|
|
|
'endTime': format_time(oozie_bundle.endTime),
|
|
|
'actions': massaged_bundle_actions_for_json(oozie_bundle),
|
|
|
@@ -602,7 +607,7 @@ def list_oozie_workflow_action(request, action):
|
|
|
def get_oozie_job_log(request, job_id):
|
|
|
oozie_api = get_oozie(request.user, api_version="v2")
|
|
|
check_job_access_permission(request, job_id)
|
|
|
- kwargs = {'logfilter' : []}
|
|
|
+ kwargs = {'logfilter': []}
|
|
|
|
|
|
if request.GET.get('format') == 'json':
|
|
|
if request.GET.get('recent'):
|
|
|
@@ -635,7 +640,8 @@ def list_oozie_info(request):
|
|
|
instrumentation = {}
|
|
|
metrics = {}
|
|
|
|
|
|
- if 'org.apache.oozie.service.MetricsInstrumentationService' in [c.strip() for c in configuration.get('oozie.services.ext', '').split(',')]:
|
|
|
+ metrics_instrumentation_service = 'org.apache.oozie.service.MetricsInstrumentationService'
|
|
|
+ if metrics_instrumentation_service in [c.strip() for c in configuration.get('oozie.services.ext', '').split(',')]:
|
|
|
api2 = get_oozie(request.user, api_version="v2")
|
|
|
metrics = api2.get_metrics()
|
|
|
else:
|
|
|
@@ -724,7 +730,8 @@ def sync_coord_workflow(request, job_id):
|
|
|
hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
|
|
|
hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
|
|
|
wf_application_path = job.conf_dict.get('wf_application_path') and Hdfs.urlsplit(job.conf_dict['wf_application_path'])[2] or ''
|
|
|
- coord_application_path = job.conf_dict.get('oozie.coord.application.path') and Hdfs.urlsplit(job.conf_dict['oozie.coord.application.path'])[2] or ''
|
|
|
+ coord_application_path = job.conf_dict.get('oozie.coord.application.path') and \
|
|
|
+ Hdfs.urlsplit(job.conf_dict['oozie.coord.application.path'])[2] or ''
|
|
|
properties = hue_coord and hue_coord.properties and dict([(param['name'], param['value']) for param in hue_coord.properties]) or None
|
|
|
|
|
|
if request.method == 'POST':
|
|
|
@@ -765,7 +772,8 @@ def sync_coord_workflow(request, job_id):
|
|
|
|
|
|
# Set previous values
|
|
|
if properties:
|
|
|
- new_params = dict([(key, properties[key]) if key in list(properties.keys()) else (key, new_params[key]) for key, value in new_params.items()])
|
|
|
+ new_params = dict(
|
|
|
+ [(key, properties[key]) if key in list(properties.keys()) else (key, new_params[key]) for key, value in new_params.items()])
|
|
|
|
|
|
initial_params = ParameterForm.get_initial_params(new_params)
|
|
|
params_form = ParametersFormSet(initial=initial_params)
|
|
|
@@ -823,7 +831,8 @@ def rerun_oozie_job(request, job_id, app_path=None):
|
|
|
return render('dashboard/rerun_workflow_popup.mako', request, {
|
|
|
'rerun_form': rerun_form,
|
|
|
'params_form': params_form,
|
|
|
- 'action': reverse('oozie:rerun_oozie_job', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) }),
|
|
|
+ 'action': reverse('oozie:rerun_oozie_job', kwargs={
|
|
|
+ 'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
|
|
|
'return_json': return_json,
|
|
|
'is_mini': request.GET.get('is_mini', False),
|
|
|
}, force_template=True)
|
|
|
@@ -887,7 +896,8 @@ def rerun_oozie_coordinator(request, job_id, app_path=None):
|
|
|
return render('dashboard/rerun_coord_popup.mako', request, {
|
|
|
'rerun_form': rerun_form,
|
|
|
'params_form': params_form,
|
|
|
- 'action': reverse('oozie:rerun_oozie_coord', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
|
|
|
+ 'action': reverse('oozie:rerun_oozie_coord', kwargs={
|
|
|
+ 'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
|
|
|
'return_json': return_json,
|
|
|
'is_mini': request.GET.get('is_mini', False),
|
|
|
}, force_template=True)
|
|
|
@@ -952,7 +962,8 @@ def rerun_oozie_bundle(request, job_id, app_path):
|
|
|
return render('dashboard/rerun_bundle_popup.mako', request, {
|
|
|
'rerun_form': rerun_form,
|
|
|
'params_form': params_form,
|
|
|
- 'action': reverse('oozie:rerun_oozie_bundle', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
|
|
|
+ 'action': reverse('oozie:rerun_oozie_bundle', kwargs={
|
|
|
+ 'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
|
|
|
}, force_template=True)
|
|
|
|
|
|
|
|
|
@@ -988,7 +999,14 @@ def submit_external_job(request, application_path):
|
|
|
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
|
|
|
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
|
|
|
application_name = os.path.basename(application_path)
|
|
|
- application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
|
|
|
+
|
|
|
+ if application_name == 'bundle.xml':
|
|
|
+ application_class = Bundle
|
|
|
+ elif application_name == 'coordinator.xml':
|
|
|
+ application_class = Coordinator
|
|
|
+ else:
|
|
|
+ application_class = get_workflow()
|
|
|
+
|
|
|
mapping[application_class.get_application_path_key()] = os.path.dirname(application_path)
|
|
|
|
|
|
try:
|
|
|
@@ -1006,7 +1024,14 @@ def submit_external_job(request, application_path):
|
|
|
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False)
|
|
|
else:
|
|
|
request.info(_('Oozie job submitted'))
|
|
|
- view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
|
|
|
+
|
|
|
+ if application_name == 'bundle.xml':
|
|
|
+ view = 'list_oozie_bundle'
|
|
|
+ elif application_name == 'coordinator.xml':
|
|
|
+ view = 'list_oozie_coordinator'
|
|
|
+ else:
|
|
|
+ view = 'list_oozie_workflow'
|
|
|
+
|
|
|
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
|
|
|
else:
|
|
|
request.error(_('Invalid submission form: %s' % params_form.errors))
|
|
|
@@ -1071,7 +1096,8 @@ def massaged_coordinator_actions_for_json(coordinator, oozie_bundle):
|
|
|
for action in coordinator_actions:
|
|
|
massaged_action = {
|
|
|
'id': action.id,
|
|
|
- 'url': action.externalId and reverse('oozie:list_oozie_workflow', kwargs={'job_id': action.externalId}) + '?%s' % '&'.join(related_job_ids) or '',
|
|
|
+ 'url': action.externalId and \
|
|
|
+ reverse('oozie:list_oozie_workflow', kwargs={'job_id': action.externalId}) + '?%s' % '&'.join(related_job_ids) or '',
|
|
|
'number': action.actionNumber,
|
|
|
'type': 'schedule-task',
|
|
|
'status': action.status,
|
|
|
@@ -1101,7 +1127,8 @@ def massaged_bundle_actions_for_json(bundle):
|
|
|
for action in bundle_actions:
|
|
|
massaged_action = {
|
|
|
'id': action.coordJobId,
|
|
|
- 'url': action.coordJobId and reverse('oozie:list_oozie_coordinator', kwargs={'job_id': action.coordJobId}) + '?bundle_job_id=%s' % bundle.id or '',
|
|
|
+ 'url': action.coordJobId and \
|
|
|
+ reverse('oozie:list_oozie_coordinator', kwargs={'job_id': action.coordJobId}) + '?bundle_job_id=%s' % bundle.id or '',
|
|
|
'name': action.coordJobName,
|
|
|
'type': action.type,
|
|
|
'status': action.status,
|
|
|
@@ -1160,7 +1187,8 @@ def massaged_oozie_jobs_for_json(oozie_jobs, user, just_sla=False):
|
|
|
|
|
|
for job in oozie_jobs:
|
|
|
if not just_sla or (just_sla and job.has_sla) and job.appName != 'pig-app-hue-script':
|
|
|
- last_modified_time_millis = hasattr(job, 'lastModTime') and job.lastModTime and (time.time() - time.mktime(job.lastModTime)) * 1000 or 0
|
|
|
+ last_modified_time_millis = hasattr(job, 'lastModTime') and \
|
|
|
+ job.lastModTime and (time.time() - time.mktime(job.lastModTime)) * 1000 or 0
|
|
|
duration_millis = job.durationTime
|
|
|
massaged_job = {
|
|
|
'id': job.id,
|
|
|
@@ -1169,8 +1197,10 @@ def massaged_oozie_jobs_for_json(oozie_jobs, user, just_sla=False):
|
|
|
'lastModTimeFormatted': last_modified_time_millis and format_duration_in_millis(last_modified_time_millis) or None,
|
|
|
'kickoffTime': hasattr(job, 'kickoffTime') and job.kickoffTime and format_time(job.kickoffTime) or '',
|
|
|
'kickoffTimeInMillis': hasattr(job, 'kickoffTime') and job.kickoffTime and time.mktime(catch_unicode_time(job.kickoffTime)) or 0,
|
|
|
- 'nextMaterializedTime': hasattr(job, 'nextMaterializedTime') and job.nextMaterializedTime and format_time(job.nextMaterializedTime) or '',
|
|
|
- 'nextMaterializedTimeInMillis': hasattr(job, 'nextMaterializedTime') and job.nextMaterializedTime and time.mktime(job.nextMaterializedTime) or 0,
|
|
|
+ 'nextMaterializedTime': hasattr(job, 'nextMaterializedTime') and \
|
|
|
+ job.nextMaterializedTime and format_time(job.nextMaterializedTime) or '',
|
|
|
+ 'nextMaterializedTimeInMillis': hasattr(job, 'nextMaterializedTime') and \
|
|
|
+ job.nextMaterializedTime and time.mktime(job.nextMaterializedTime) or 0,
|
|
|
'timeOut': hasattr(job, 'timeOut') and job.timeOut or None,
|
|
|
'endTime': job.endTime and format_time(job.endTime) or None,
|
|
|
'pauseTime': hasattr(job, 'pauseTime') and job.pauseTime and format_time(job.endTime) or None,
|
|
|
@@ -1187,9 +1217,9 @@ def massaged_oozie_jobs_for_json(oozie_jobs, user, just_sla=False):
|
|
|
'user': job.user,
|
|
|
'absoluteUrl': job.get_absolute_url(),
|
|
|
'canEdit': has_job_edition_permission(job, user),
|
|
|
- 'killUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id':job.id, 'action':'kill'}),
|
|
|
- 'suspendUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id':job.id, 'action':'suspend'}),
|
|
|
- 'resumeUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id':job.id, 'action':'resume'}),
|
|
|
+ 'killUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id': job.id, 'action': 'kill'}),
|
|
|
+ 'suspendUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id': job.id, 'action': 'suspend'}),
|
|
|
+ 'resumeUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id': job.id, 'action': 'resume'}),
|
|
|
'created': hasattr(job, 'createdTime') and job.createdTime and format_time(job.createdTime) or '',
|
|
|
'createdInMillis': job.submissionTime,
|
|
|
'startTime': hasattr(job, 'startTime') and format_time(job.startTime) or None,
|
|
|
@@ -1202,7 +1232,7 @@ def massaged_oozie_jobs_for_json(oozie_jobs, user, just_sla=False):
|
|
|
}
|
|
|
jobs.append(massaged_job)
|
|
|
|
|
|
- return { 'jobs': jobs }
|
|
|
+ return {'jobs': jobs}
|
|
|
|
|
|
|
|
|
def check_job_access_permission(request, job_id, **kwargs):
|
|
|
@@ -1254,7 +1284,9 @@ def check_job_edition_permission(oozie_job, user):
|
|
|
|
|
|
|
|
|
def has_job_edition_permission(oozie_job, user):
|
|
|
- return is_admin(user) or oozie_job.user == user.username or (oozie_job.group and user.groups.filter(name__in=oozie_job.group.split(",")).exists()) or (oozie_job.acl and user.username in oozie_job.acl.split(','))
|
|
|
+ return (is_admin(user) or oozie_job.user == user.username or \
|
|
|
+ (oozie_job.group and user.groups.filter(name__in=oozie_job.group.split(",")).exists()) or \
|
|
|
+ (oozie_job.acl and user.username in oozie_job.acl.split(',')))
|
|
|
|
|
|
|
|
|
def has_dashboard_jobs_access(user):
|