瀏覽代碼

HUE-833 [oozie] Add DistCp action

Add test
Add example
Move 'job_xml' to the bottom of the page
Romain Rigaux 13 年之前
父節點
當前提交
127e099

+ 0 - 0
apps/oozie/examples/distcp/empty


+ 155 - 21
apps/oozie/src/oozie/fixtures/initial_oozie_examples.json

@@ -8,8 +8,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/sleep",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "MapReduce job that sleeps"
     }
   },
@@ -22,8 +22,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/sleep",
       "schema_version": "uri:oozie:coordinator:0.2",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of MapReduce action"
     }
   },
@@ -36,8 +36,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/terasort",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of sequential Java actions"
     }
   },
@@ -50,8 +50,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/sleepfork",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of multiple forks"
     }
   },
@@ -64,8 +64,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/hive",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of Hive action"
     }
   },
@@ -78,8 +78,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/pig",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of Pig action"
     }
   },
@@ -92,8 +92,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/sqoop",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of Sqoop action"
     }
   },
@@ -106,8 +106,8 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/ssh",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of Ssh action"
     }
   },
@@ -120,11 +120,25 @@
       "parameters": "[]",
       "deployment_dir": "/user/hue/oozie/workspaces/shell",
       "schema_version": "uri:oozie:workflow:0.4",
-      "last_modified": "2012-10-02 00:00:00",
-      "owner": 1,
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 2,
       "description": "Example of Shell action"
     }
   },
+  {
+    "pk": 16,
+    "model": "oozie.job",
+    "fields": {
+      "is_shared": true,
+      "name": "DistCp",
+      "parameters": "[]",
+      "deployment_dir": "/user/hue/oozie/workspaces/distcp",
+      "schema_version": "uri:oozie:workflow:0.4",
+      "last_modified": "2012-11-05 00:00:00",
+      "owner": 1,
+      "description": "Example of DistCp action"
+    }
+  },
   {
     "pk": 5,
     "model": "oozie.workflow",
@@ -213,6 +227,17 @@
       "job_properties": "[]"
     }
   },
+  {
+    "pk": 16,
+    "model": "oozie.workflow",
+    "fields": {
+      "job_xml": "",
+      "start": 71,
+      "is_single": false,
+      "end": 70,
+      "job_properties": "[]"
+    }
+  },
   {
     "pk": 23,
     "model": "oozie.link",
@@ -713,6 +738,46 @@
       "child": 65
     }
   },
+  {
+    "pk": 92,
+    "model": "oozie.link",
+    "fields": {
+      "comment": "",
+      "name": "to",
+      "parent": 71,
+      "child": 72
+    }
+  },
+  {
+    "pk": 93,
+    "model": "oozie.link",
+    "fields": {
+      "comment": "",
+      "name": "related",
+      "parent": 71,
+      "child": 70
+    }
+  },
+  {
+    "pk": 94,
+    "model": "oozie.link",
+    "fields": {
+      "comment": "",
+      "name": "ok",
+      "parent": 72,
+      "child": 70
+    }
+  },
+  {
+    "pk": 95,
+    "model": "oozie.link",
+    "fields": {
+      "comment": "",
+      "name": "error",
+      "parent": 72,
+      "child": 69
+    }
+  },
   {
     "pk": 16,
     "model": "oozie.node",
@@ -1123,6 +1188,46 @@
       "description": "Execute a Python script printing its arguments"
     }
   },
+  {
+    "pk": 69,
+    "model": "oozie.node",
+    "fields": {
+      "node_type": "kill",
+      "workflow": 16,
+      "name": "kill",
+      "description": ""
+    }
+  },
+  {
+    "pk": 70,
+    "model": "oozie.node",
+    "fields": {
+      "node_type": "end",
+      "workflow": 16,
+      "name": "end",
+      "description": ""
+    }
+  },
+  {
+    "pk": 71,
+    "model": "oozie.node",
+    "fields": {
+      "node_type": "start",
+      "workflow": 16,
+      "name": "start",
+      "description": ""
+    }
+  },
+  {
+    "pk": 72,
+    "model": "oozie.node",
+    "fields": {
+      "node_type": "distcp",
+      "workflow": 16,
+      "name": "DistCp",
+      "description": "Copy some data and overwrite it if needed"
+    }
+  },
   {
     "pk": 19,
     "model": "oozie.mapreduce",
@@ -1295,6 +1400,16 @@
       "command": "hello.py"
     }
   },
+  {
+    "pk": 72,
+    "model": "oozie.distcp",
+    "fields": {
+      "prepares": "[]",
+      "job_xml": "",
+      "params": "[{\"value\":\"-overwrite\",\"type\":\"argument\"},{\"value\":\"-m\",\"type\":\"argument\"},{\"value\":\"${MAP_NUMBER}\",\"type\":\"argument\"},{\"value\":\"/user/hue/oozie/workspaces/data\",\"type\":\"argument\"},{\"value\":\"${OUTPUT}\",\"type\":\"argument\"}]",
+      "job_properties": "[]"
+    }
+  },
   {
     "pk": 18,
     "model": "oozie.start",
@@ -1351,6 +1466,13 @@
       "node_ptr": 67
     }
   },
+  {
+    "pk": 71,
+    "model": "oozie.start",
+    "fields": {
+      "node_ptr": 71
+    }
+  },
   {
     "pk": 17,
     "model": "oozie.end",
@@ -1391,6 +1513,11 @@
     "model": "oozie.end",
     "fields": {}
   },
+  {
+    "pk": 70,
+    "model": "oozie.end",
+    "fields": {}
+  },
   {
     "pk": 16,
     "model": "oozie.kill",
@@ -1447,6 +1574,13 @@
       "message": "Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]"
     }
   },
+  {
+    "pk": 69,
+    "model": "oozie.kill",
+    "fields": {
+      "message": "Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]"
+    }
+  },
   {
     "pk": 34,
     "model": "oozie.fork",
@@ -1471,12 +1605,12 @@
     "pk": 6,
     "model": "oozie.coordinator",
     "fields": {
-      "end": "2012-10-04 00:00:00",
+      "end": "2012-11-05 00:00:00",
       "concurrency": null,
       "frequency_number": 1,
       "workflow": 5,
       "frequency_unit": "days",
-      "start": "2012-10-01 00:00:00",
+      "start": "2012-11-01 00:00:00",
       "timeout": null,
       "timezone": "America/Los_Angeles",
       "throttle": null,
@@ -1492,7 +1626,7 @@
       "coordinator": 6,
       "frequency_unit": "days",
       "uri": "${DAY}",
-      "start": "2012-10-01 00:00:00",
+      "start": "2012-11-01 00:00:00",
       "timezone": "America/Los_Angeles",
       "done_flag": "",
       "description": ""

+ 16 - 1
apps/oozie/src/oozie/forms.py

@@ -22,7 +22,7 @@ from django.db.models import Q
 
 from desktop.lib.django_forms import MultiForm, SplitDateTimeWidget
 from oozie.models import Workflow, Node, Java, Mapreduce, Streaming, Coordinator,\
-  Dataset, DataInput, DataOutput, Pig, Link, Hive, Sqoop, Ssh, Shell
+  Dataset, DataInput, DataOutput, Pig, Link, Hive, Sqoop, Ssh, Shell, DistCp
 
 LOG = logging.getLogger(__name__)
 
@@ -188,6 +188,20 @@ class ShellForm(forms.ModelForm):
     }
 
 
+class DistCpForm(forms.ModelForm):
+  class Meta:
+    model = DistCp
+    exclude = NodeForm.Meta.ALWAYS_HIDE
+    widgets = {
+      'job_properties': forms.widgets.HiddenInput(),
+      'prepares': forms.widgets.HiddenInput(),
+      'params': forms.widgets.HiddenInput(),
+      'command': forms.TextInput(attrs={'class': 'pathChooser span5'}),
+      'description': forms.TextInput(attrs={'class': 'span5'}),
+      'job_xml': forms.TextInput(attrs={'class': 'span5'}),
+    }
+
+
 
 class LinkForm(forms.ModelForm):
   comment = forms.CharField(label='if', max_length=1024, required=True, widget=forms.TextInput(attrs={'class': 'span8'}))
@@ -307,6 +321,7 @@ _node_type_TO_FORM_CLS = {
   Sqoop.node_type: SqoopForm,
   Ssh.node_type: SshForm,
   Shell.node_type: ShellForm,
+  DistCp.node_type: DistCpForm,
 }
 
 

+ 1 - 1
apps/oozie/src/oozie/management/commands/oozie_setup.py

@@ -58,7 +58,7 @@ class Command(NoArgsCommand):
     sample, created = User.objects.get_or_create(username='sample')
     management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
     from oozie.models import Job
-    Job.objects.filter(owner__id=1, pk__lte=15).update(owner=sample)
+    Job.objects.filter(owner__id=1, pk__lte=16).update(owner=sample)
 
 
 # This should probably be refactored and some parts moved to the HDFS lib. Jobsub could be updated to.

+ 270 - 0
apps/oozie/src/oozie/migrations/0008_auto__add_distcp.py

@@ -0,0 +1,270 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+
+        # Adding model 'DistCp'
+        db.create_table('oozie_distcp', (
+            ('prepares', self.gf('django.db.models.fields.TextField')(default='[]')),
+            ('job_xml', self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True)),
+            ('params', self.gf('django.db.models.fields.TextField')(default='[]')),
+            ('job_properties', self.gf('django.db.models.fields.TextField')(default='[]')),
+            ('node_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Node'], unique=True, primary_key=True)),
+        ))
+        db.send_create_signal('oozie', ['DistCp'])
+
+
+    def backwards(self, orm):
+
+        # Deleting model 'DistCp'
+        db.delete_table('oozie_distcp')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'oozie.coordinator': {
+            'Meta': {'object_name': 'Coordinator', '_ormbases': ['oozie.Job']},
+            'concurrency': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'end': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 10, 11, 16, 8, 45, 499416)'}),
+            'execution': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'frequency_number': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'frequency_unit': ('django.db.models.fields.CharField', [], {'default': "'days'", 'max_length': '20'}),
+            'job_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Job']", 'unique': 'True', 'primary_key': 'True'}),
+            'start': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 10, 8, 16, 8, 45, 499386)'}),
+            'throttle': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'timeout': ('django.db.models.fields.SmallIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'timezone': ('django.db.models.fields.CharField', [], {'default': "'America/Los_Angeles'", 'max_length': '24'}),
+            'workflow': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['oozie.Workflow']", 'null': 'True'})
+        },
+        'oozie.datainput': {
+            'Meta': {'object_name': 'DataInput'},
+            'coordinator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['oozie.Coordinator']"}),
+            'dataset': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Dataset']", 'unique': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'})
+        },
+        'oozie.dataoutput': {
+            'Meta': {'object_name': 'DataOutput'},
+            'coordinator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['oozie.Coordinator']"}),
+            'dataset': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Dataset']", 'unique': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'})
+        },
+        'oozie.dataset': {
+            'Meta': {'object_name': 'Dataset'},
+            'coordinator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['oozie.Coordinator']"}),
+            'description': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024', 'blank': 'True'}),
+            'done_flag': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '64', 'blank': 'True'}),
+            'frequency_number': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'frequency_unit': ('django.db.models.fields.CharField', [], {'default': "'days'", 'max_length': '20'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'start': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 10, 8, 16, 8, 45, 500668)'}),
+            'timezone': ('django.db.models.fields.CharField', [], {'default': "'America/Los_Angeles'", 'max_length': '24'}),
+            'uri': ('django.db.models.fields.CharField', [], {'default': "'/data/${YEAR}${MONTH}${DAY}'", 'max_length': '1024'})
+        },
+        'oozie.distcp': {
+            'Meta': {'object_name': 'DistCp'},
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'params': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'prepares': ('django.db.models.fields.TextField', [], {'default': "'[]'"})
+        },
+        'oozie.end': {
+            'Meta': {'object_name': 'End'},
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'oozie.fork': {
+            'Meta': {'object_name': 'Fork'},
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'oozie.history': {
+            'Meta': {'object_name': 'History'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['oozie.Job']"}),
+            'oozie_job_id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'properties': ('django.db.models.fields.TextField', [], {}),
+            'submission_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'submitter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'oozie.hive': {
+            'Meta': {'object_name': 'Hive'},
+            'archives': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'files': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': '\'[{"name":"oozie.use.system.libpath","value":"true"},{"name":"oozie.hive.defaults","value":"hive-default.xml"}]\''}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'params': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'prepares': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'script_path': ('django.db.models.fields.CharField', [], {'max_length': '256'})
+        },
+        'oozie.java': {
+            'Meta': {'object_name': 'Java'},
+            'archives': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'args': ('django.db.models.fields.CharField', [], {'max_length': '4096', 'blank': 'True'}),
+            'files': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'jar_path': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'java_opts': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'main_class': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'prepares': ('django.db.models.fields.TextField', [], {'default': "'[]'"})
+        },
+        'oozie.job': {
+            'Meta': {'object_name': 'Job'},
+            'deployment_dir': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}),
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_shared': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True', 'blank': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'parameters': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'schema_version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'oozie.join': {
+            'Meta': {'object_name': 'Join'},
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'oozie.kill': {
+            'Meta': {'object_name': 'Kill'},
+            'message': ('django.db.models.fields.CharField', [], {'default': "'Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]'", 'max_length': '256'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'oozie.link': {
+            'Meta': {'object_name': 'Link'},
+            'child': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parent_node'", 'to': "orm['oozie.Node']"}),
+            'comment': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'parent': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'child_node'", 'to': "orm['oozie.Node']"})
+        },
+        'oozie.mapreduce': {
+            'Meta': {'object_name': 'Mapreduce'},
+            'archives': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'files': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'jar_path': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True'}),
+            'prepares': ('django.db.models.fields.TextField', [], {'default': "'[]'"})
+        },
+        'oozie.node': {
+            'Meta': {'object_name': 'Node'},
+            'children': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'parents'", 'symmetrical': 'False', 'through': "orm['oozie.Link']", 'to': "orm['oozie.Node']"}),
+            'description': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'node_type': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'workflow': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['oozie.Workflow']"})
+        },
+        'oozie.pig': {
+            'Meta': {'object_name': 'Pig'},
+            'archives': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'files': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': '\'[{"name":"oozie.use.system.libpath","value":"true"}]\''}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'params': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'prepares': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'script_path': ('django.db.models.fields.CharField', [], {'max_length': '256'})
+        },
+        'oozie.shell': {
+            'Meta': {'object_name': 'Shell'},
+            'archives': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'capture_output': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'command': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'files': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'params': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'prepares': ('django.db.models.fields.TextField', [], {'default': "'[]'"})
+        },
+        'oozie.sqoop': {
+            'Meta': {'object_name': 'Sqoop'},
+            'archives': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'files': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': '\'[{"name":"oozie.use.system.libpath","value":"true"}]\''}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'params': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'prepares': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'script_path': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'})
+        },
+        'oozie.ssh': {
+            'Meta': {'object_name': 'Ssh'},
+            'capture_output': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'command': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'params': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'user': ('django.db.models.fields.CharField', [], {'max_length': '64'})
+        },
+        'oozie.start': {
+            'Meta': {'object_name': 'Start'},
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True'})
+        },
+        'oozie.streaming': {
+            'Meta': {'object_name': 'Streaming'},
+            'archives': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'files': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': '\'[{"name":"oozie.use.system.libpath","value":"true"}]\''}),
+            'mapper': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'node_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Node']", 'unique': 'True', 'primary_key': 'True'}),
+            'reducer': ('django.db.models.fields.CharField', [], {'max_length': '512'})
+        },
+        'oozie.workflow': {
+            'Meta': {'object_name': 'Workflow', '_ormbases': ['oozie.Job']},
+            'end': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'end_workflow'", 'null': 'True', 'to': "orm['oozie.End']"}),
+            'is_single': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'job_properties': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'job_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['oozie.Job']", 'unique': 'True', 'primary_key': 'True'}),
+            'job_xml': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '512', 'blank': 'True'}),
+            'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
+        }
+    }
+
+    complete_apps = ['oozie']

+ 34 - 3
apps/oozie/src/oozie/models.py

@@ -625,6 +625,8 @@ class Node(models.Model):
       node = self.ssh
     elif self.node_type == Shell.node_type:
       node = self.shell
+    elif self.node_type == DistCp.node_type:
+      node = self.distcp
     elif self.node_type == Streaming.node_type:
       node = self.streaming
     elif self.node_type == Java.node_type:
@@ -992,7 +994,7 @@ class Ssh(Action):
                           help_text=_t('User executing the shell command.'))
   host = models.CharField(max_length=256, verbose_name=_t('Host'),
                          help_text=_t('Where the shell will be executed.'))
-  command = models.CharField(max_length=256, verbose_name=_t('Command'),
+  command = models.CharField(max_length=256, verbose_name=_t('%(type)s command') % {'type': node_type.title()},
                              help_text=_t('The command that will be executed.'))
   params = models.TextField(default="[]", verbose_name=_t('Arguments'),
                             help_text=_t('The arguments of the %(type)s command')  % {'type': node_type.title()})
@@ -1010,7 +1012,7 @@ class Shell(Action):
   PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares')
   node_type = 'shell'
 
-  command = models.CharField(max_length=256, blank=False, verbose_name=_t('Shell command'),
+  command = models.CharField(max_length=256, blank=False, verbose_name=_t('%(type)s command') % {'type': node_type.title()},
                              help_text=_t('The path of the Shell command to execute'))
   params = models.TextField(default="[]", verbose_name=_t('Arguments'),
                             help_text=_t('The arguments of Shell command can then be specified using one or more argument element.'))
@@ -1049,7 +1051,36 @@ class Shell(Action):
     return json.loads(self.prepares)
 
 
-Action.types = (Mapreduce.node_type, Streaming.node_type, Java.node_type, Pig.node_type, Hive.node_type, Sqoop.node_type, Ssh.node_type, Shell.node_type)
+class DistCp(Action):
+  PARAM_FIELDS = ('job_properties', 'params', 'prepares')
+  node_type = 'distcp'
+
+  params = models.TextField(default="[]", verbose_name=_t('Arguments'),
+                            help_text=_t('The arguments of the %(type)s command. Put options first then source paths then destination path.')
+                                        % {'type': node_type.title()})
+  job_properties = models.TextField(default='[]', verbose_name=_t('Job properties'),
+                                    help_text=_t('For the job configuration (e.g. mapred.job.queue.name=production'))
+  prepares = models.TextField(default="[]", verbose_name=_t('Prepares'),
+                              help_text=_t('List of absolute paths to delete then to create before starting the application. '
+                                           'This should be used exclusively for directory cleanup'))
+  job_xml = models.CharField(max_length=PATH_MAX, default='', blank=True, verbose_name=_t('Job XML'),
+                             help_text=_t('Refer to a Hadoop JobConf job.xml file bundled in the workflow deployment directory. '
+                                          'Properties specified in the configuration element override properties specified in the '
+                                          'files specified by any job-xml elements.'))
+
+
+  def get_properties(self):
+    return json.loads(self.job_properties)
+
+  def get_params(self):
+    return json.loads(self.params)
+
+  def get_prepares(self):
+    return json.loads(self.prepares)
+
+
+Action.types = (Mapreduce.node_type, Streaming.node_type, Java.node_type, Pig.node_type, Hive.node_type, Sqoop.node_type, Ssh.node_type, Shell.node_type,
+                DistCp.node_type)
 
 
 class ControlFlow(Node):

+ 6 - 0
apps/oozie/src/oozie/templates/editor/edit_workflow.mako

@@ -120,6 +120,12 @@ ${ layout.menubar(section='workflows') }
                   <i class="icon-plus"></i> ${ _('Ssh') }
                 </a>
                 <p/>
+                <p>
+                <a href="${ url('oozie:new_action', workflow=workflow.id, node_type='distcp', parent_action_id=workflow.end.get_parents()[0].id) }"
+                  title="${ _('Click to add to the end') }" class="btn">
+                  <i class="icon-plus"></i> ${ _('DistCp') }
+                </a>
+                <p/>
               </div>
               <div class="tab-pane" id="import">
                 <p>

+ 7 - 2
apps/oozie/src/oozie/templates/editor/edit_workflow_action.mako

@@ -66,7 +66,7 @@ ${ layout.menubar(section='workflows') }
       </div>
 
       % for field in action_form:
-        % if field.html_name not in ('name', 'description', 'node_type'):
+        % if field.html_name not in ('name', 'description', 'node_type', 'job_xml'):
           ${ utils.render_field(field) }
         % endif
       % endfor
@@ -144,7 +144,7 @@ ${ layout.menubar(section='workflows') }
             % if node_type in ('pig', 'hive'):
               <button class="btn" data-bind="click: addParam">${ _('Add Param') }</button>
             % endif
-            % if node_type in ('pig', 'shell'):
+            % if node_type in ('pig', 'shell', 'distcp'):
               <button class="btn" data-bind="click: addArgument">${ _('Add Argument') }</button>
             % endif
             % if node_type in ('sqoop', 'ssh'):
@@ -243,6 +243,11 @@ ${ layout.menubar(section='workflows') }
          </div>
       </div>
       % endif
+
+      % if 'job_xml' in action_form.fields:
+        ${ utils.render_field(action_form['job_xml']) }
+      % endif
+
       </fieldset>
 
       <div class="form-actions">

+ 33 - 0
apps/oozie/src/oozie/templates/editor/gen/workflow-distcp.xml.mako

@@ -0,0 +1,33 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%namespace name="common" file="workflow-common.xml.mako" />
+
+    <action name="${ node }">
+        <distcp xmlns="uri:oozie:distcp-action:0.1">
+            <job-tracker>${'${'}jobTracker}</job-tracker>
+            <name-node>${'${'}nameNode}</name-node>
+
+            ${ common.prepares(node.get_prepares()) }
+            ${ common.configuration(node.get_properties()) }
+
+            % for param in node.get_params():
+              <arg>${ param['value'] }</arg>
+            % endfor
+        </distcp>
+        <ok to="${ node.get_child('ok') }"/>
+        <error to="${ node.get_child('error') }"/>
+    </action>

+ 13 - 0
apps/oozie/src/oozie/tests.py

@@ -1202,6 +1202,19 @@ class TestOozieSubmissions(OozieBase):
     assert_equal('SUCCEEDED', job.status)
 
 
+  def test_submit_distcp_action(self):
+    wf = Workflow.objects.get(name='DistCp')
+
+    response = self.c.post(reverse('oozie:submit_workflow', args=[wf.id]),
+                           data={u'form-MAX_NUM_FORMS': [u''],
+                                u'form-0-name': [u'MAP_NUMBER'], u'form-0-value': [u'5'],
+                                u'form-1-name': [u'OUTPUT '], u'form-1-value': [u'${nameNode}/user/test/out/distcp'],
+                                u'form-INITIAL_FORMS': [u'2'], u'form-TOTAL_FORMS': [u'2']},
+                           follow=True)
+    job = OozieServerProvider.wait_until_completion(response.context['oozie_workflow'].id)
+    assert_equal('SUCCEEDED', job.status)
+
+
 class TestDashboard(OozieMockBase):
 
   def test_manage_workflow_dashboard(self):

+ 1 - 1
desktop/libs/liboozie/src/liboozie/oozie_api_test.py

@@ -54,7 +54,7 @@ class OozieServerProvider(object):
 
   @classmethod
 
-  def wait_until_completion(cls, oozie_jobid, timeout=1800.0, step=5):
+  def wait_until_completion(cls, oozie_jobid, timeout=300.0, step=5):
     job = cls.oozie.get_job(oozie_jobid)
     start = time.time()