فهرست منبع

HUE-1452 [oozie] Credentials by action for Oozie workflows

Credentials enabled only when Oozie is using security.
Romain Rigaux 12 سال پیش
والد
کامیت
95a0e14
46فایلهای تغییر یافته به همراه488 افزوده شده و 265 حذف شده
  1. 6 1
      apps/beeswax/src/beeswax/hive_site.py
  2. 8 8
      apps/oozie/src/oozie/migrations/0021_auto__chg_field_java_args__add_field_job_is_trashed.py
  3. 8 8
      apps/oozie/src/oozie/migrations/0022_auto__chg_field_mapreduce_node_ptr__chg_field_start_node_ptr.py
  4. 60 49
      apps/oozie/src/oozie/models.py
  5. 2 2
      apps/oozie/src/oozie/templates/dashboard/list_oozie_coordinator.mako
  6. 2 2
      apps/oozie/src/oozie/templates/dashboard/list_oozie_sla.mako
  7. 2 2
      apps/oozie/src/oozie/templates/dashboard/list_oozie_workflow.mako
  8. 25 6
      apps/oozie/src/oozie/templates/editor/action_utils.mako
  9. 1 1
      apps/oozie/src/oozie/templates/editor/create_coordinator.mako
  10. 1 1
      apps/oozie/src/oozie/templates/editor/create_workflow.mako
  11. 11 8
      apps/oozie/src/oozie/templates/editor/dataset_utils.mako
  12. 1 1
      apps/oozie/src/oozie/templates/editor/edit_bundle.mako
  13. 3 3
      apps/oozie/src/oozie/templates/editor/edit_coordinator.mako
  14. 8 33
      apps/oozie/src/oozie/templates/editor/edit_workflow.mako
  15. 3 0
      apps/oozie/src/oozie/templates/editor/gen/workflow-common.xml.mako
  16. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-distcp.xml.mako
  17. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-email.xml.mako
  18. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-fs.xml.mako
  19. 3 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-generic.xml.mako
  20. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-hive.xml.mako
  21. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-java.xml.mako
  22. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-mapreduce.xml.mako
  23. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-pig.xml.mako
  24. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-shell.xml.mako
  25. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-sqoop.xml.mako
  26. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-ssh.xml.mako
  27. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-streaming.xml.mako
  28. 1 1
      apps/oozie/src/oozie/templates/editor/gen/workflow-subworkflow.xml.mako
  29. 12 9
      apps/oozie/src/oozie/templates/editor/gen/workflow.xml.mako
  30. 1 1
      apps/oozie/src/oozie/templates/editor/import_coordinator.mako
  31. 1 1
      apps/oozie/src/oozie/templates/editor/import_workflow.mako
  32. 1 1
      apps/oozie/src/oozie/templates/editor/list_trashed_coordinators.mako
  33. 34 12
      apps/oozie/src/oozie/tests.py
  34. 1 1
      apps/oozie/src/oozie/urls.py
  35. 1 4
      apps/oozie/src/oozie/utils.py
  36. 0 1
      apps/oozie/src/oozie/views/api.py
  37. 10 10
      apps/oozie/src/oozie/views/dashboard.py
  38. 12 5
      apps/oozie/src/oozie/views/editor.py
  39. 1 41
      apps/oozie/static/js/workflow.js
  40. 29 24
      apps/oozie/static/js/workflow.models.js
  41. 40 2
      apps/oozie/static/js/workflow.node.js
  42. 93 0
      desktop/libs/liboozie/src/liboozie/credentials.py
  43. 88 0
      desktop/libs/liboozie/src/liboozie/credentials_test.py
  44. 2 2
      desktop/libs/liboozie/src/liboozie/oozie_api.py
  45. 6 11
      desktop/libs/liboozie/src/liboozie/submittion.py
  46. 0 2
      desktop/libs/liboozie/src/liboozie/tests.py

+ 6 - 1
apps/beeswax/src/beeswax/hive_site.py

@@ -26,9 +26,9 @@ import re
 import socket
 
 from desktop.lib import security_util
+from hadoop import confparse
 
 import beeswax.conf
-from hadoop import confparse
 
 
 LOG = logging.getLogger(__name__)
@@ -45,6 +45,7 @@ _CNF_HIVESERVER2_KERBEROS_PRINCIPAL = 'hive.server2.authentication.kerberos.prin
 _CNF_HIVESERVER2_AUTHENTICATION = 'hive.server2.authentication'
 _CNF_HIVESERVER2_IMPERSONATION = 'hive.server2.enable.doAs'
 
+
 # Host is whatever up to the colon. Allow and ignore a trailing slash.
 _THRIFT_URI_RE = re.compile("^thrift://([^:]+):(\d+)[/]?$")
 
@@ -118,6 +119,10 @@ def get_hiveserver2_authentication():
 def hiveserver2_impersonation_enabled():
   return get_conf().get(_CNF_HIVESERVER2_IMPERSONATION, 'FALSE').upper() == 'TRUE'
 
+def hiveserver2_jdbc_url():
+  return 'jdbc:hive2://%s:%s' % (beeswax.conf.HIVE_SERVER_HOST.get(), beeswax.conf.HIVE_SERVER_PORT.get())
+
+
 def _parse_hive_site():
   """
   Parse hive-site.xml and store in _HIVE_SITE_DICT

+ 8 - 8
apps/oozie/src/oozie/migrations/0021_auto__chg_field_java_args__add_field_job_is_trashed.py

@@ -5,25 +5,25 @@ from south.v2 import SchemaMigration
 from django.db import models
 
 class Migration(SchemaMigration):
-    
+
     def forwards(self, orm):
-        
+
         # Changing field 'Java.args'
         db.alter_column('oozie_java', 'args', self.gf('django.db.models.fields.TextField')(blank=True))
 
         # Adding field 'Job.is_trashed'
         db.add_column('oozie_job', 'is_trashed', self.gf('django.db.models.fields.BooleanField')(default=False, db_index=True, blank=True), keep_default=False)
-    
-    
+
+
     def backwards(self, orm):
-        
+
         # Changing field 'Java.args'
         db.alter_column('oozie_java', 'args', self.gf('django.db.models.fields.CharField')(max_length=4096, blank=True))
 
         # Deleting field 'Job.is_trashed'
         db.delete_column('oozie_job', 'is_trashed')
-    
-    
+
+
     models = {
         'auth.group': {
             'Meta': {'object_name': 'Group'},
@@ -322,5 +322,5 @@ class Migration(SchemaMigration):
             'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
         }
     }
-    
+
     complete_apps = ['oozie']

+ 8 - 8
apps/oozie/src/oozie/migrations/0022_auto__chg_field_mapreduce_node_ptr__chg_field_start_node_ptr.py

@@ -5,25 +5,25 @@ from south.v2 import SchemaMigration
 from django.db import models
 
 class Migration(SchemaMigration):
-    
+
     def forwards(self, orm):
-        
+
         # Changing field 'Mapreduce.node_ptr'
         db.alter_column('oozie_mapreduce', 'node_ptr_id', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Node'], unique=True, primary_key=True))
 
         # Changing field 'Start.node_ptr'
         db.alter_column('oozie_start', 'node_ptr_id', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Node'], unique=True, primary_key=True))
-    
-    
+
+
     def backwards(self, orm):
-        
+
         # Changing field 'Mapreduce.node_ptr'
         db.alter_column('oozie_mapreduce', 'node_ptr_id', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Node'], unique=True))
 
         # Changing field 'Start.node_ptr'
         db.alter_column('oozie_start', 'node_ptr_id', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['oozie.Node'], unique=True))
-    
-    
+
+
     models = {
         'auth.group': {
             'Meta': {'object_name': 'Group'},
@@ -322,5 +322,5 @@ class Migration(SchemaMigration):
             'start': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'start_workflow'", 'null': 'True', 'to': "orm['oozie.Start']"})
         }
     }
-    
+
     complete_apps = ['oozie']

+ 60 - 49
apps/oozie/src/oozie/models.py

@@ -60,6 +60,7 @@ LOG = logging.getLogger(__name__)
 PATH_MAX = 512
 name_validator = RegexValidator(regex='^[a-zA-Z_][\-_a-zA-Z0-9]{1,39}$',
                                 message=_('Enter a valid value: combination of 2 - 40 letters and digits starting by a letter'))
+# To sync in worklow.models.js
 DEFAULT_SLA = [
     {'key': 'enabled', 'value': False},
     {'key': 'nominal-time', 'value': ''},
@@ -71,8 +72,7 @@ DEFAULT_SLA = [
     {'key': 'notification-msg', 'value': ''},
     {'key': 'upstream-apps', 'value': ''},
 ]
-DEFAULT_GLOBAL_PROPERTIES = []
-DEFAULT_GLOBAL_CONFIG = []
+
 
 class JobManager(models.Manager):
 
@@ -123,10 +123,10 @@ class Job(models.Model):
                                   help_text=_t('Enable other users to have access to this job.'))
   parameters = models.TextField(default='[{"name":"oozie.use.system.libpath","value":"true"}]', verbose_name=_t('Oozie parameters'),
                                 help_text=_t('Parameters used at the submission time (e.g. market=US, oozie.use.system.libpath=true).'))
-  is_trashed = models.BooleanField(default=False, db_index=True, verbose_name=_t('Is trashed'), blank=True,# Deprecated
+  is_trashed = models.BooleanField(default=False, db_index=True, verbose_name=_t('Is trashed'), blank=True, # Deprecated
                                    help_text=_t('If this job is trashed.'))
   doc = generic.GenericRelation(Document, related_name='oozie_doc')
-  data = models.TextField(blank=True, default=json.dumps({}))  # e.g. data=json.dump({'sla': [python data], 'global': [python data], 'credentials': [python data]})
+  data = models.TextField(blank=True, default=json.dumps({}))  # e.g. data=json.dump({'sla': [python data], ...})
 
   objects = JobManager()
   unique_together = ('owner', 'name')
@@ -222,16 +222,16 @@ class Job(models.Model):
     return user.is_superuser or self.owner == user
 
   @property
-  def data_dict(self): 
+  def data_dict(self):
+    if not self.data:
+      self.data = json.dumps({})
     data_python = json.loads(self.data)
     # Backward compatibility
     if 'sla' not in data_python:
       data_python['sla'] = copy.deepcopy(DEFAULT_SLA)
-    if 'globalProperties' not in data_python:
-      data_python['globalProperties'] = DEFAULT_GLOBAL_PROPERTIES
-    if 'globalConfig' not in data_python:
-      data_python['globalConfig'] = DEFAULT_GLOBAL_CONFIG     
-    return data_python 
+    if 'credentials' not in data_python:
+      data_python['credentials'] = []
+    return data_python
 
   @property
   def data_js_escaped(self):
@@ -242,7 +242,7 @@ class Job(models.Model):
     return self.data_dict['sla']
 
   @sla.setter
-  def sla(self, sla): 
+  def sla(self, sla):
     data_ = self.data_dict
     data_['sla'] = sla
     self.data = json.dumps(data_)
@@ -251,31 +251,11 @@ class Job(models.Model):
   def sla_enabled(self):
     return self.sla[0]['value'] # #1 is enabled
 
-  @property
-  def global_properties(self):
-    return self.data_dict['globalProperties']
-
-  @global_properties.setter
-  def global_properties(self, global_properties): 
-    data_ = self.data_dict
-    data_['globalProperties'] = global_properties
-    self.data = json.dumps(data_)
-
-  @property
-  def global_config(self):
-    return self.data_dict['globalConfig']
-
-  @global_config.setter
-  def global_config(self, global_config): 
-    data_ = self.data_dict
-    data_['globalConfig'] = global_config
-    self.data = json.dumps(data_)
-
 
 class WorkflowManager(models.Manager):
   SCHEMA_VERSION = {
-    '0.4': 'url:oozie:workflow:0.4',
-    '0.5': 'url:oozie:workflow:0.5'
+    '0.4': 'uri:oozie:workflow:0.4',
+    '0.5': 'uri:oozie:workflow:0.5'
   }
 
   def new_workflow(self, owner):
@@ -611,6 +591,11 @@ class Workflow(Job):
   def sla_workflow_enabled(self):
     return self.sla_enabled or any([node.sla_enabled for node in self.node_list if hasattr(node, 'sla_enabled')])
 
+  @property
+  def credentials(self):
+    sub_lists = [node.credentials for node in self.node_list if hasattr(node, 'credentials')]
+    return set([item['name'] for l in sub_lists for item in l if item['value']])
+
 
 class Link(models.Model):
   # Links to exclude when using get_children_link(), get_parent_links() in the API
@@ -697,7 +682,7 @@ class Node(models.Model):
 
     return node
 
-  def find_parameters(self):    
+  def find_parameters(self):
     return find_parameters(self, self.PARAM_FIELDS)
 
   def __unicode__(self):
@@ -784,20 +769,42 @@ class Node(models.Model):
       'node_type': self.node_type
     })
 
+  @property
+  def data_dict(self):
+    if not self.data:
+      self.data = json.dumps({})
+    data_python = json.loads(self.data)
+    # Backward compatibility
+    if 'sla' not in data_python:
+      data_python['sla'] = copy.deepcopy(DEFAULT_SLA)
+    if 'credentials' not in data_python:
+      data_python['credentials'] = []
+    return data_python
+
   @property
   def sla(self):
-    return json.loads(self.data).get('sla', copy.deepcopy(DEFAULT_SLA))
+    return self.data_dict['sla']
 
   @sla.setter
   def sla(self, sla):
-    data_json = json.loads(self.data)
-    data_json['sla'] = sla
-    self.data = json.dumps(data_json)
+    data_ = self.data_dict
+    data_['sla'] = sla
+    self.data = json.dumps(data_)
 
   @property
   def sla_enabled(self):
     return self.sla[0]['value'] # #1 is enabled
 
+  @property
+  def credentials(self):
+    return self.data_dict['credentials']
+
+  @credentials.setter
+  def credentials(self, credentials):
+    data_ = self.data_dict
+    data_['credentials'] = credentials
+    self.data = json.dumps(data_)
+
 
 class Action(Node):
   types = ()
@@ -928,7 +935,7 @@ class Java(Action):
 
 
 class Pig(Action):
-  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla')
+  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla', 'credentials')
   node_type = 'pig'
 
   script_path = models.CharField(max_length=256, blank=False, verbose_name=_t('Script name'),
@@ -966,7 +973,7 @@ class Pig(Action):
 
 
 class Hive(Action):
-  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla')
+  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla', 'credentials')
   node_type = 'hive'
 
   script_path = models.CharField(max_length=256, blank=False, verbose_name=_t('Script name'),
@@ -1003,7 +1010,7 @@ class Hive(Action):
 
 
 class Sqoop(Action):
-  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla')
+  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla', 'credentials')
   node_type = 'sqoop'
 
   script_path = models.TextField(blank=True, verbose_name=_t('Command'), default='',
@@ -1044,7 +1051,7 @@ class Sqoop(Action):
 
 
 class Ssh(Action):
-  PARAM_FIELDS = ('user', 'host', 'command', 'params', 'sla')
+  PARAM_FIELDS = ('user', 'host', 'command', 'params', 'sla', 'credentials')
   node_type = 'ssh'
 
   user = models.CharField(max_length=64, verbose_name=_t('User'),
@@ -1066,7 +1073,7 @@ class Ssh(Action):
 
 
 class Shell(Action):
-  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla')
+  PARAM_FIELDS = ('files', 'archives', 'job_properties', 'params', 'prepares', 'sla', 'credentials')
   node_type = 'shell'
 
   command = models.CharField(max_length=256, blank=False, verbose_name=_t('%(type)s command') % {'type': node_type.title()},
@@ -1109,7 +1116,7 @@ class Shell(Action):
 
 
 class DistCp(Action):
-  PARAM_FIELDS = ('job_properties', 'params', 'prepares', 'sla')
+  PARAM_FIELDS = ('job_properties', 'params', 'prepares', 'sla', 'credentials')
   node_type = 'distcp'
 
   params = models.TextField(default="[]", verbose_name=_t('Arguments'),
@@ -1137,7 +1144,7 @@ class DistCp(Action):
 
 
 class Fs(Action):
-  PARAM_FIELDS = ('deletes', 'mkdirs', 'moves', 'chmods', 'touchzs', 'sla')
+  PARAM_FIELDS = ('deletes', 'mkdirs', 'moves', 'chmods', 'touchzs', 'sla', 'credentials')
   node_type = 'fs'
 
   deletes = models.TextField(default="[]", verbose_name=_t('Delete path'), blank=True,
@@ -1172,7 +1179,7 @@ class Fs(Action):
 
 
 class Email(Action):
-  PARAM_FIELDS = ('to', 'cc', 'subject', 'body', 'sla')
+  PARAM_FIELDS = ('to', 'cc', 'subject', 'body', 'sla', 'credentials')
   node_type = 'email'
 
   to = models.TextField(default='', verbose_name=_t('TO addresses'), help_text=_t('Comma-separated values.'))
@@ -1182,7 +1189,7 @@ class Email(Action):
 
 
 class SubWorkflow(Action):
-  PARAM_FIELDS = ('subworkflow', 'propagate_configuration', 'job_properties', 'sla')
+  PARAM_FIELDS = ('subworkflow', 'propagate_configuration', 'job_properties', 'sla', 'credentials')
   node_type = 'subworkflow'
 
   sub_workflow = models.ForeignKey(Workflow, db_index=True, verbose_name=_t('Sub-workflow'),
@@ -1197,7 +1204,7 @@ class SubWorkflow(Action):
 
 
 class Generic(Action):
-  PARAM_FIELDS = ('xml',)
+  PARAM_FIELDS = ('xml', 'credentials')
   node_type = 'generic'
 
   xml = models.TextField(default='', verbose_name=_t('XML of the custom action'),
@@ -1559,6 +1566,10 @@ class Coordinator(Job):
     xml = zfile.read('coordinator.xml')
     return xml, metadata
 
+  @property
+  def sla_jsescaped(self):
+    return json.dumps(self.sla, cls=JSONEncoderForHTML)
+
 
 class DatasetManager(models.Manager):
   def can_read_or_exception(self, request, dataset_id):
@@ -1895,7 +1906,7 @@ def find_json_parameters(fields):
   params = []
 
   for field in fields:
-    for data in field.values():      
+    for data in field.values():
       if isinstance(data, basestring):
         for match in Template.pattern.finditer(data):
           name = match.group('braced')

+ 2 - 2
apps/oozie/src/oozie/templates/dashboard/list_oozie_coordinator.mako

@@ -272,9 +272,9 @@ ${ layout.menubar(section='coordinators', dashboard=True) }
       <div class="tab-pane" id="definition">
         <textarea id="definitionEditor">${ oozie_coordinator.definition.decode('utf-8', 'replace') }</textarea>
       </div>
-      
+
       % if oozie_coordinator.has_sla:
-      <div class="tab-pane" id="sla">          
+      <div class="tab-pane" id="sla">
       </div>
       % endif
     </div>

+ 2 - 2
apps/oozie/src/oozie/templates/dashboard/list_oozie_sla.mako

@@ -82,7 +82,7 @@ ${ layout.menubar(section='sla', dashboard=True) }
     </div>
 
     <div class="tab-pane" id="graphTab">
-      MY GRAPH      
+      MY GRAPH
     </div>
 
     </p>
@@ -116,7 +116,7 @@ ${ layout.menubar(section='sla', dashboard=True) }
           slaTable.fnClearTable();
           if (data['oozie_slas']) {
             slaTable.fnAddData(data['oozie_slas']); ## Should put a class on status, links to oozie job ids...
-          } 
+          }
         });
       }, 300);
     });

+ 2 - 2
apps/oozie/src/oozie/templates/dashboard/list_oozie_workflow.mako

@@ -285,9 +285,9 @@ ${ layout.menubar(section='workflows', dashboard=True) }
         <div class="tab-pane" id="definition">
           <textarea id="definitionEditor">${ oozie_workflow.definition.decode('utf-8', 'replace') }</textarea>
         </div>
-        
+
         % if oozie_workflow.has_sla:
-        <div class="tab-pane" id="sla">          
+        <div class="tab-pane" id="sla">
         </div>
         % endif
       </div>

+ 25 - 6
apps/oozie/src/oozie/templates/editor/action_utils.mako

@@ -61,7 +61,7 @@
 
   ${ utils.render_constant(_('Action type'), node_type) }
 
-  
+
   <div class="control-group ">
     <label class="control-label">
       <a href="javascript:void(0);" id="advanced-btn" onclick="$('#node-advanced-container').toggle('hide')">
@@ -75,10 +75,29 @@
         <div class="control-group">
           <label class="control-label">
               ${ _('SLA') }
-          </label>      
-      
-          ${ utils.slaForm() }  
-    </div>
+          </label>
+
+          ${ utils.slaForm() }
+        </div>
+
+        <div class="control-group" data-bind="visible: credentials().length > 0">
+          <label class="control-label">
+              ${ _('Credentials') }
+          </label>
+
+          <div data-bind="foreach: credentials">
+            <div class="controls">
+              <div class="span3">
+                <span></span>
+              </div>
+              <div class="span9">
+                <span data-bind="text: name" class="span2"/>
+                <input type="checkbox" data-bind="checked: value"/>
+                <span data-bind="visible: name() == 'hbase'">${ _('Requires hbase-site.xml in job-xml field') }</span>
+              </div>
+            </div>
+          </div>
+        </div>
 
     <!-- ko if: $root.context().nodes && $root.context().error_node -->
     <div class="control-group">
@@ -100,7 +119,7 @@
     <!-- /ko -->
 
   </div>
-  
+
 
   <hr/>
 % endif

+ 1 - 1
apps/oozie/src/oozie/templates/editor/create_coordinator.mako

@@ -70,7 +70,7 @@ ${ layout.menubar(section='coordinators') }
             <div class="fieldWrapper">
               ${ utils.render_field_no_popover(coordinator_form['name'], extra_attrs = {'validate':'true'}) }
               ${ utils.render_field_no_popover(coordinator_form['description']) }
-              ${ utils.render_field_no_popover(coordinator_form['workflow'], extra_attrs = {'validate':'true'}) }              
+              ${ utils.render_field_no_popover(coordinator_form['workflow'], extra_attrs = {'validate':'true'}) }
               ${ coordinator_form['parameters'] | n,unicode }
               ${ coordinator_form['job_properties'] | n,unicode }
               <div class="hide">

+ 1 - 1
apps/oozie/src/oozie/templates/editor/create_workflow.mako

@@ -46,7 +46,7 @@ ${ layout.menubar(section='workflows') }
               <form class="form-horizontal" id="workflowForm" action="${ url('oozie:create_workflow') }" method="POST">
               <fieldset>
               ${ utils.render_field(workflow_form['name']) }
-              ${ utils.render_field(workflow_form['description']) }              
+              ${ utils.render_field(workflow_form['description']) }
 
               <div class="control-group ">
                 <label class="control-label">

+ 11 - 8
apps/oozie/src/oozie/templates/editor/dataset_utils.mako

@@ -14,6 +14,9 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 
+<%!
+  from django.utils.translation import ugettext as _
+%>
 
 ## How to use:
 ## <%include file="dataset_utils.mako"/>
@@ -23,33 +26,33 @@
 
 
 <div class="control-group">
-  <label class="control-label">Instance</label>
+  <label class="control-label">${ _('Instance') }</label>
   <div class="controls">
       <div class="btn-group" data-toggle="buttons-radio">
-          <button id="default-btn" type="button" class="btn" data-bind="click: setDefault, css: { active: instance_choice() == 'default' }">Default</button>
-          <button id="single-btn" type="button" class="btn" data-bind="click: setSingle, css: { active: instance_choice() == 'single' }">Single</button>
-          <button id="range-btn" type="button" class="btn" data-bind="click: setRange, css: { active: instance_choice() == 'range' }">Range</button>
+          <button id="default-btn" type="button" class="btn" data-bind="click: setDefault, css: { active: instance_choice() == 'default' }">${ _('Default') }</button>
+          <button id="single-btn" type="button" class="btn" data-bind="click: setSingle, css: { active: instance_choice() == 'single' }">${ _('Single') }</button>
+          <button id="range-btn" type="button" class="btn" data-bind="click: setRange, css: { active: instance_choice() == 'range' }">${ _('Range') }</button>
       </div>
       <span class="help-block">${ dataset_form['instance_choice'].help_text }</span>
 
       <div data-bind="visible: $.inArray(instance_choice(), ['single', 'range']) != -1">
-          <span class="span1">Start</span>
+          <span class="span1">${ _('Start') }</span>
           <input name="instance_start" type="number" data-bind="value: start_instance, enable: ! is_advanced_start_instance()"/>
           <label style="display: inline">
               &nbsp;
               <input type="checkbox" data-bind="checked: is_advanced_start_instance">
-              (advanced)
+              ${ _('(advanced)') }
           </label>
           <input type="text" data-bind="value: advanced_start_instance, visible: is_advanced_start_instance()" class="span4"/>
           <span class="help-block">${ dataset_form['advanced_start_instance'].help_text }</span>
       </div>
       <div data-bind="visible: instance_choice() == 'range'">
-          <span class="span1">End</span>
+          <span class="span1">${ _('End') }</span>
           <input name="instance_end" type="number" data-bind="value: end_instance, enable: ! is_advanced_end_instance()" />
           <label style="display: inline">
               &nbsp;
               <input type="checkbox" data-bind="checked: is_advanced_end_instance">
-              (advanced)
+              ${ _('(advanced)') }
           </label>
           <input type="text" data-bind="value: advanced_end_instance, visible: is_advanced_end_instance()" class="span4"/>
           <span class="help-block">${ dataset_form['advanced_end_instance'].help_text }</span>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/edit_bundle.mako

@@ -113,7 +113,7 @@ ${ layout.menubar(section='bundles') }
                       ${ _('UTC time only. (e.g. if you want 10pm PST (UTC+8) set it 8 hours later to 6am the next day.') }
                     </div>
                   </div>
-                </div>                
+                </div>
                 ${ bundle_form['parameters'] | n,unicode }
                 <div class="hide">
                   ${ utils.render_field_no_popover(bundle_form['is_shared']) }

+ 3 - 3
apps/oozie/src/oozie/templates/editor/edit_coordinator.mako

@@ -764,9 +764,9 @@ ${ layout.menubar(section='coordinators') }
 
       $("a[rel='tooltip']").tooltip();
     });
-    
-    ${ utils.slaGlobal() } 
-   
+
+    ${ utils.slaGlobal() }
+
   </script>
 % endif
 

+ 8 - 33
apps/oozie/src/oozie/templates/editor/edit_workflow.mako

@@ -81,7 +81,7 @@ ${ layout.menubar(section='workflows') }
   <div class="span10">
     <div id="properties" class="section hide">
     <div class="card card-small">
-    
+
       <div class="alert alert-info"><h3 data-bind="text: name()"></h3></div>
       <div class="card-body">
         <p>
@@ -117,41 +117,14 @@ ${ layout.menubar(section='workflows') }
         </div>
 
       <div id="advanced-container" class="hide">
-
-        <div id="globalPropertiesEditord" class="control-group">
-          <label class="control-label">
-            ${ _('Global properites') }
-          </label>
-          <%
-          workflows.key_value_field(_("Global properties"), _("Global properties"), {
-          'name': 'data.global_properties',
-          'remove': '$root.removeGlobalProperty',
-          'add': '$root.addGlobalProperty',
-          })
-          %>
-        </div>
-
-        <div id="globalConfigEditord" class="control-group">
-          <label class="control-label">
-            ${ _('Global config') }
-          </label>
-          <%
-          workflows.key_value_field(_("Global config"), _("Global config"), {
-          'name': 'data.global_config',
-          'remove': '$root.removeGlobalConfig',
-          'add': '$root.addGlobalConfig',
-          })
-          %>
-        </div>
-      
         <div id="slaEditord" class="control-group">
           <label class="control-label">
             ${ _('SLA') }
           </label>
-          
+
           ${ utils.slaForm() }
-        </div>      
-      
+        </div>
+
         % if user_can_edit_job:
           ${ utils.render_field_with_error_js(workflow_form['deployment_dir'], workflow_form['deployment_dir'].name, extra_attrs={'data-bind': 'value: %s' % workflow_form['deployment_dir'].name}) }
         % endif
@@ -674,7 +647,9 @@ function save_workflow() {
   }
 }
 
-// Fetch all nodes from server.
+var OOZIE_CREDENTIALS = ${ credentials | n,unicode };
+
+// Fetch workflow properties from server.
 var workflow_model = new WorkflowModel({
   id: ${ workflow.id },
   name: "${ workflow.name }",
@@ -700,7 +675,7 @@ var import_jobsub_action = new ImportJobsubAction({workflow: workflow});
 var import_workflow_action = new ImportWorkflowAction({workflow: workflow});
 var modal = new Modal($('#node-modal'));
 
-// Load data.
+// Fetch nodes
 import_jobsub_action.fetchWorkflows({ success: import_jobsub_load_success });
 import_workflow_action.fetchWorkflows({ success: import_workflow_load_success });
 {

+ 3 - 0
apps/oozie/src/oozie/templates/editor/gen/workflow-common.xml.mako

@@ -24,6 +24,9 @@
 <%def name="filelink(path)">${ '#' in path and path or path + '#' + posixpath.basename(path) }</%def>
 
 
+<%def name="credentials(credentials)">${ ' cred="%s"' % ' '.join([cred['name'] for cred in credentials if cred['value']]) if credentials else '' | n,unicode }</%def>
+
+
 <%def name="prepares(prepares)">
         % if prepares:
             <prepare>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-distcp.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <distcp xmlns="uri:oozie:distcp-action:0.1">
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-email.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <email xmlns="uri:oozie:email-action:0.1">
             <to>${ node.to }</to>
             % if node.cc:

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-fs.xml.mako

@@ -21,7 +21,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <fs>
             % for param in node.get_deletes():
               <delete path='${ smart_path(param['name'], mapping) }'/>

+ 3 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-generic.xml.mako

@@ -15,7 +15,9 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 
-    <action name="${ node }">
+<%namespace name="common" file="workflow-common.xml.mako" />
+
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         ${ node.xml | n,unicode }
         <ok to="${ node.get_oozie_child('ok') }"/>
         <error to="${ node.get_oozie_child('error') }"/>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-hive.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }"${ " cred='hive_credentials'" if mapping.get('is_kerberized_hive') else '' | n,unicode}>
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <hive xmlns="uri:oozie:hive-action:0.2">
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-java.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <java>
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-mapreduce.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <map-reduce>
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-pig.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <pig>
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-shell.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <shell xmlns="uri:oozie:shell-action:0.1">
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-sqoop.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <sqoop xmlns="uri:oozie:sqoop-action:0.2">
             <job-tracker>${'${'}jobTracker}</job-tracker>
             <name-node>${'${'}nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-ssh.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <ssh xmlns="uri:oozie:ssh-action:0.1">
             <host>${ node.user }@${ node.host }</host>
             <command>${ node.command }</command>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-streaming.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <map-reduce>
             <job-tracker>${ '${' }jobTracker}</job-tracker>
             <name-node>${ '${' }nameNode}</name-node>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-subworkflow.xml.mako

@@ -17,7 +17,7 @@
 
 <%namespace name="common" file="workflow-common.xml.mako" />
 
-    <action name="${ node }">
+    <action name="${ node }"${ common.credentials(node.credentials) }>
         <sub-workflow>
             <app-path>${'${'}nameNode}${ node.sub_workflow.deployment_dir }</app-path>
 

+ 12 - 9
apps/oozie/src/oozie/templates/editor/gen/workflow.xml.mako

@@ -29,19 +29,22 @@
     % endif
   </global>
   % endif
-  % if mapping.get('is_kerberized_hive'):
+  % if workflow.credentials:
   <credentials>
-    <credential name='hive_credentials' type='${ mapping['credential_type'] }'>
+    % for cred_type in workflow.credentials:
+    <%
+      credential = mapping['credentials'][cred_type]
+    %>
+    <credential name="${ credential['xml_name'] }" type="${ cred_type }">
+    % for name, value in credential['properties']:
       <property>
-        <name>hcat.metastore.uri</name>
-        <value>${ mapping['thrift_server'] }</value>
-      </property>
-      <property>
-        <name>hcat.metastore.principal</name>
-        <value>${ mapping['hive_principal'] }</value>
+        <name>${ name }</name>
+        <value>${ value }</value>
       </property>
+    % endfor
     </credential>
-   </credentials>
+    % endfor
+  </credentials>
   % endif
   % for node in workflow.node_list:
       ${ node.to_xml(mapping) | n }

+ 1 - 1
apps/oozie/src/oozie/templates/editor/import_coordinator.mako

@@ -52,7 +52,7 @@ ${ layout.menubar(section='coordinators') }
           </div>
         </div>
 
-        <div class="form-actions" style="padding-left: 20px !important;">        
+        <div class="form-actions" style="padding-left: 20px !important;">
           <a class="btn" onclick="history.back()">${ _('Back') }</a>
           <input class="btn btn-primary" type="submit" value="${ _('Import') }" />
         </div>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/import_workflow.mako

@@ -74,7 +74,7 @@ ${ layout.menubar(section='workflows') }
 
       </div>
 
-      <div class="form-actions" style="padding-left: 20px !important;">        
+      <div class="form-actions" style="padding-left: 20px !important;">
         <a class="btn" onclick="history.back()">${ _('Back') }</a>
         <input class="btn btn-primary" type="submit" value="${ _('Import') }" />
       </div>

+ 1 - 1
apps/oozie/src/oozie/templates/editor/list_trashed_coordinators.mako

@@ -50,7 +50,7 @@ ${ layout.menubar(section='coordinators') }
        &nbsp;&nbsp;
        <a href="${ url('oozie:list_coordinators') }" id="home-btn" class="btn" title="${ _('Got to coordinator manager') }">
          <i class="fa fa-home"></i> ${ _('Back') }
-       </a>        
+       </a>
     </%def>
   </%actionbar:render>
 

+ 34 - 12
apps/oozie/src/oozie/tests.py

@@ -186,6 +186,16 @@ class MockOozieApi:
   def get_oozie_slas(self, **kwargs):
     return MockOozieApi.WORKFLOWS_SLAS
 
+  def get_configuration(self):
+    oozie_credentialclasses = """
+           hbase=org.apache.oozie.action.hadoop.HbaseCredentials,
+           hcat=org.apache.oozie.action.hadoop.HCatCredentials,
+           hive2=org.apache.oozie.action.hadoop.Hive2Credentials
+    """
+    return {
+        'oozie.credentials.credentialclasses': oozie_credentialclasses
+    }
+
 
 class OozieMockBase(object):
 
@@ -1206,12 +1216,22 @@ class TestEditor(OozieMockBase):
       saved = beeswax.conf.HIVE_CONF_DIR
       beeswax.conf.HIVE_CONF_DIR = Getter()
 
+      action1 = Node.objects.get(workflow=self.wf, name='MyHive')
+      action1.credentials = [{'name': 'hcat', 'value': True}, {'name': 'hbase', 'value': False}]
+      action1.save()
+
       xml = self.wf.to_xml(mapping={
-         'is_kerberized_hive': True,
-         'credential_type': 'hcat',
-         'thrift_server': 'thrift://darkside-1234:9999',
-         'hive_principal': 'hive/darkside-1234@test.com'
-      })
+          'credentials': {
+              'hcat': {
+                  'xml_name': 'hcat',
+                  'properties': [
+                      ('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
+                      ('hcat.metastore.principal', 'hive')
+                  ]
+              }
+          }
+        }
+      )
 
       assert_true("""
 <workflow-app name="wf-name-1" xmlns="uri:oozie:workflow:0.4">
@@ -1225,19 +1245,19 @@ class TestEditor(OozieMockBase):
             </configuration>
   </global>
   <credentials>
-    <credential name='hive_credentials' type='hcat'>
+    <credential name="hcat" type="hcat">
       <property>
         <name>hcat.metastore.uri</name>
-        <value>thrift://darkside-1234:9999</value>
+        <value>thrift://hue-koh-chang:9999</value>
       </property>
       <property>
         <name>hcat.metastore.principal</name>
-        <value>hive/darkside-1234@test.com</value>
+        <value>hive</value>
       </property>
     </credential>
-   </credentials>
+  </credentials>
     <start to="MyHive"/>
-    <action name="MyHive" cred='hive_credentials'>
+    <action name="MyHive" cred="hcat">
         <hive xmlns="uri:oozie:hive-action:0.2">
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>
@@ -1422,7 +1442,8 @@ class TestEditor(OozieMockBase):
 """<coordinator-app name="MyCoord"
   frequency="${coord:days(1)}"
   start="2012-07-01T00:00Z" end="2012-07-04T00:00Z" timezone="America/Los_Angeles"
-  xmlns="uri:oozie:coordinator:0.2">
+  xmlns="uri:oozie:coordinator:0.2"
+  >
   <controls>
     <timeout>100</timeout>
     <concurrency>3</concurrency>
@@ -1500,7 +1521,8 @@ class TestEditor(OozieMockBase):
 """<coordinator-app name="MyCoord"
   frequency="${coord:days(1)}"
   start="2012-07-01T00:00Z" end="2012-07-04T00:00Z" timezone="America/Los_Angeles"
-  xmlns="uri:oozie:coordinator:0.2">
+  xmlns="uri:oozie:coordinator:0.2"
+  >
   <controls>
     <timeout>100</timeout>
     <concurrency>3</concurrency>

+ 1 - 1
apps/oozie/src/oozie/urls.py

@@ -100,6 +100,6 @@ urlpatterns += patterns(
   url(r'^submit_external_job/(?P<application_path>.+?)$', 'submit_external_job', name='submit_external_job'),
 
   url(r'^list_oozie_info/$', 'list_oozie_info', name='list_oozie_info'),
-  
+
   url(r'^list_oozie_sla/$', 'list_oozie_sla', name='list_oozie_sla'),
 )

+ 1 - 4
apps/oozie/src/oozie/utils.py

@@ -62,9 +62,6 @@ def model_to_dict(model):
   for field in model._meta.fields:
     try:
       attr = getattr(model, field.name, None)
-#      if field.name == 'data' and issubclass(type(model), Node):
-#        if hasattr(model, 'sla'):
-#          dictionary['sla'] = model.sla
       if isinstance(attr, models.Model):
         dictionary[field.name] = attr.id
       elif isinstance(attr, datetime):
@@ -142,4 +139,4 @@ def oozie_to_hue_frequency(frequency_string):
   # Get frequency number and units from frequency
   # frequency units and number are just different parts of the EL function.
   matches = re.match(FREQUENCY_REGEX, frequency_string)
-  return matches.group('frequency_unit'), matches.group('frequency_number')
+  return matches.group('frequency_unit'), matches.group('frequency_number')

+ 0 - 1
apps/oozie/src/oozie/views/api.py

@@ -32,7 +32,6 @@ from oozie.decorators import check_job_access_permission, check_job_edition_perm
 from oozie.utils import model_to_dict, format_dict_field_values, format_field_value
 
 
-
 LOG = logging.getLogger(__name__)
 
 

+ 10 - 10
apps/oozie/src/oozie/views/dashboard.py

@@ -335,27 +335,27 @@ def list_oozie_sla(request):
     # filter=nominal_start=2013-06-18T00:01Z;nominal_end=2013-06-23T00:01Z;app_name=my-sla-app
     params = {}
     print request.POST
-    
+
     job_name = request.POST.get('job_name')
     if job_name.endswith('-oozie-oozi-W'):
       if 'isParent' in request.POST:
         params['parent_id'] = job_name
-      else:    
+      else:
         params['id'] = job_name
     else:
       params['app_name'] = job_name
-      
+
     if request.POST.get('start'):
       params['nominal_start'] = request.POST.get('start')
     if request.POST.get('end'):
       params['nominal_end'] = request.POST.get('end')
-      
+
     oozie_slas = api.get_oozie_slas(**params)
-    print oozie_slas  
-    
+    print oozie_slas
+
   else:
     oozie_slas = [] # or get latest?
-  
+
   columns = [
     'slaStatus',
     'id',
@@ -366,13 +366,13 @@ def list_oozie_sla(request):
     'expectedStart',
     'actualStart',
     'expectedEnd',
-    'actualEnd',                  
+    'actualEnd',
     'jobStatus',
     #'expectedDuration',
     #'actualDuration',
     'lastModified'
-  ]  
-  
+  ]
+
   if request.REQUEST.get('format') == 'json':
     massaged_slas = []
     for sla in oozie_slas:

+ 12 - 5
apps/oozie/src/oozie/views/editor.py

@@ -35,6 +35,9 @@ from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import smart_str
 from desktop.lib.rest.http_client import RestException
 from desktop.models import Document
+
+from liboozie.credentials import Credentials
+from liboozie.oozie_api import get_oozie
 from liboozie.submittion import Submission
 
 from filebrowser.lib.archives import archive_factory
@@ -78,7 +81,7 @@ def list_trashed_workflows(request):
 
 def list_coordinators(request, workflow_id=None):
   data = Document.objects.available(Coordinator, request.user)
-    
+
   if workflow_id is not None:
     data = [job for job in data if job.workflow.id == workflow_id]
 
@@ -211,7 +214,7 @@ def import_coordinator(request):
         else:
           Coordinator.objects.filter(id=coordinator.id).delete()
           raise PopupException(_('Archive should be a Zip.'))
-      
+
       Document.objects.link(coordinator, owner=request.user, name=coordinator.name, description=coordinator.description)
       request.info(_('Coordinator imported'))
       return redirect(reverse('oozie:edit_coordinator', kwargs={'coordinator': coordinator.id}))
@@ -245,6 +248,9 @@ def edit_workflow(request, workflow):
   workflow_form = WorkflowForm(instance=workflow)
   user_can_access_job = workflow.can_read(request.user)
   user_can_edit_job = workflow.is_editable(request.user)
+  api = get_oozie(request.user)
+  credentials = Credentials()
+  credentials.fetch(api)
 
   return render('editor/edit_workflow.mako', request, {
     'workflow_form': workflow_form,
@@ -257,7 +263,8 @@ def edit_workflow(request, workflow):
     'default_link_form': DefaultLinkForm(action=workflow.start),
     'node_form': NodeForm(),
     'action_forms': [(node_type, design_form_by_type(node_type, request.user, workflow)())
-                     for node_type in ACTION_TYPES.iterkeys()]
+                     for node_type in ACTION_TYPES.iterkeys()],
+    'credentials': json.dumps(credentials.credentials.keys())
   })
 
 
@@ -459,7 +466,7 @@ def edit_coordinator(request, coordinator):
     data_output_formset = DataOutputFormSet(request.POST, request.FILES, instance=coordinator)
     new_data_input_formset = NewDataInputFormSet(request.POST, request.FILES, instance=coordinator, prefix='input')
     new_data_output_formset = NewDataOutputFormSet(request.POST, request.FILES, instance=coordinator, prefix='output')
-    print request.POST
+
     if coordinator_form.is_valid() and dataset_formset.is_valid() and data_input_formset.is_valid() and data_output_formset.is_valid() \
         and new_data_input_formset.is_valid() and new_data_output_formset.is_valid():
       coordinator = coordinator_form.save()
@@ -468,7 +475,7 @@ def edit_coordinator(request, coordinator):
       data_output_formset.save()
       new_data_input_formset.save()
       new_data_output_formset.save()
-      
+
       coordinator.sla = json.loads(request.POST.get('sla'))
       coordinator.save()
 

+ 1 - 41
apps/oozie/static/js/workflow.js

@@ -435,12 +435,7 @@ var WorkflowModule = function($, NodeModelChooser, Node, ForkNode, DecisionNode,
     self.sla = ko.computed(function() {
       return self.data.sla();
     });
-    self.globalProperties = ko.computed(function() {
-      return self.data.globalProperties();
-    });
-    self.globalConfig = ko.computed(function() {
-      return self.data.globalConfig();
-    });
+
 
     self.url = ko.computed(function() {
       return '/oozie/workflows/' + self.id();
@@ -687,41 +682,6 @@ var WorkflowModule = function($, NodeModelChooser, Node, ForkNode, DecisionNode,
       var self = this;
       self.job_properties.remove(data);
     },
-
-    addGlobalProperty: function(data, event) {
-      var self = this;
-      var prop = { name: ko.observable(""), value: ko.observable("") };
-      // force bubble up to containing observable array.
-      prop.name.subscribe(function(){
-        self.data.global_properties.valueHasMutated();
-      });
-      prop.value.subscribe(function(){
-        self.data.global_properties.valueHasMutated();
-      });
-      self.data.global_properties.push(prop);
-    },
-
-    removeGlobalProperty: function(data, event) {
-      var self = this;
-      self.data.global_properties.remove(data);
-    },
-
-    addGlobalConfig: function(data, event) {
-      var self = this;
-      var prop = { name: ko.observable(""), value: ko.observable("") };
-      // force bubble up to containing observable array.
-      prop.name.subscribe(function(){
-        self.data.global_config.valueHasMutated();
-      });
-      prop.value.subscribe(function(){
-        self.data.global_config.valueHasMutated();
-      });
-      self.data.global_config.push(prop);
-    },
-    removeGlobalConfig: function(data, event) {
-      var self = this;
-      self.data.global_config.remove(data);
-    },
     
     // Workflow UI
     // Function to build nodes... recursively.

+ 29 - 24
apps/oozie/static/js/workflow.models.js

@@ -32,10 +32,9 @@ var DEFAULT_SLA = [
 ];
 
 function getDefaultData() {
- return {
-  'sla': DEFAULT_SLA.slice(0),
-  'global_properties': [],
-  'global_config': []
+  return {
+    'sla': DEFAULT_SLA.slice(0),
+    'credentials': []
   };
 }
 
@@ -389,22 +388,28 @@ var ModelModule = function($) {
   return module;
 };
 
-function initializeData() {
+function initializeWorkflowData() {
   var self = this;
 
   self.data = ($.type(self.data) == "string") ? $.parseJSON(self.data) : self.data;
 
-  if (!('sla' in self.data)) {
+  if (! ('sla' in self.data)) {
     self.data['sla'] = DEFAULT_SLA.slice(0);
   }
+}
 
-  if (!('global_properties' in self.data)) {
-    self.data['global_properties'] = [];
-  }
+function initializeNodeData() {
+  var self = this;
 
-  if (!('global_config' in self.data)) {
-    self.data['global_config'] = [];
+  self.data = ($.type(self.data) == "string") ? $.parseJSON(self.data) : self.data;
+
+  if (! ('sla' in self.data)) {
+    self.data['sla'] = DEFAULT_SLA.slice(0);
   }
+  
+  if (! ('credentials' in self.data)) {
+	self.data['credentials'] = getDefaultData()['credentials'].slice(0);
+  }  
 }
 
 var WorkflowModel = ModelModule($);
@@ -420,7 +425,7 @@ $.extend(WorkflowModel.prototype, {
   parameters: '[]',
   job_xml: '',
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeWorkflowData
 });
 
 var NodeModel = ModelModule($);
@@ -466,7 +471,7 @@ $.extend(DistCPModel.prototype, {
   params: '[]',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var MapReduceModel = ModelModule($);
@@ -484,7 +489,7 @@ $.extend(MapReduceModel.prototype, {
   job_xml: '',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var StreamingModel = ModelModule($);
@@ -501,7 +506,7 @@ $.extend(StreamingModel.prototype, {
   reducer: '',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var JavaModel = ModelModule($);
@@ -523,7 +528,7 @@ $.extend(JavaModel.prototype, {
   capture_output: false,
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var PigModel = ModelModule($);
@@ -542,7 +547,7 @@ $.extend(PigModel.prototype, {
   script_path: '',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var HiveModel = ModelModule($);
@@ -561,7 +566,7 @@ $.extend(HiveModel.prototype, {
   script_path: '',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var SqoopModel = ModelModule($);
@@ -580,7 +585,7 @@ $.extend(SqoopModel.prototype, {
   script_path: '',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var ShellModel = ModelModule($);
@@ -600,7 +605,7 @@ $.extend(ShellModel.prototype, {
   capture_output: false,
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var SshModel = ModelModule($);
@@ -617,7 +622,7 @@ $.extend(SshModel.prototype, {
   capture_output: false,
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var FsModel = ModelModule($);
@@ -634,7 +639,7 @@ $.extend(FsModel.prototype, {
   touchzs: '[]',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var EmailModel = ModelModule($);
@@ -650,7 +655,7 @@ $.extend(EmailModel.prototype, {
   body: '',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var SubWorkflowModel = ModelModule($);
@@ -665,7 +670,7 @@ $.extend(SubWorkflowModel.prototype, {
   job_properties: '[]',
   child_links: [],
   data: getDefaultData(),
-  initialize: initializeData
+  initialize: initializeNodeData
 });
 
 var GenericModel = ModelModule($);

+ 40 - 2
apps/oozie/static/js/workflow.node.js

@@ -132,6 +132,46 @@ var NodeModule = function($, IdGeneratorTable, NodeFields) {
       });
     }
 
+    if (self.data && self.data.credentials) {
+      self.credentials = ko.computed(function() {
+        return self.data.credentials();
+      });
+
+      // A bit complicated but just update the available credentials
+      var new_creds = OOZIE_CREDENTIALS.slice(0);
+      var old_creds = [];
+      var to_remove = [];
+
+      $.each(self.credentials(), function(index, credential) {
+        if (credential != null) {
+          if ($.inArray(credential.name(), OOZIE_CREDENTIALS) != -1) {
+            // A new credential was added to the Oozie server
+            new_creds = jQuery.grep(new_creds, function(value) {
+              return value != credential.name();
+            });
+          } else {
+           // A credential was removed from the Oozie server
+           to_remove.push(credential);
+          }
+        }
+      });
+
+      $.each(new_creds, function(index, name) {
+        var prop = { name: ko.observable(name), value: ko.observable(false) };
+        prop.name.subscribe(function(){
+          self.data.credentials.valueHasMutated();
+        });
+        prop.value.subscribe(function(){
+          self.data.credentials.valueHasMutated();
+        });
+        self.data.credentials.push(prop);
+      });
+
+      $.each(to_remove, function(index, name) {
+        self.data.credentials.remove(name);
+      });
+    }
+
     if ('files' in model) {
       //// WARNING: The following order should be preserved!
 
@@ -230,8 +270,6 @@ var NodeModule = function($, IdGeneratorTable, NodeFields) {
 
       var options = {};
 
-      // data = $.extend(true, {}, self.model);
-
       var success = false;
       var request = $.extend({
         url: '/oozie/workflows/' + self._workflow.id() + '/nodes/' + self.node_type() + '/validate',

+ 93 - 0
desktop/libs/liboozie/src/liboozie/credentials.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from django.utils.translation import ugettext as _
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Credentials(object):
+  NAME_TO_CLASS_MAPPING = {
+      "hcat": "org.apache.oozie.action.hadoop.HCatCredentials",
+      "hive2": "org.apache.oozie.action.hadoop.Hive2Credentials",
+      "hbase": "org.apache.oozie.action.hadoop.HbaseCredentials",
+  }
+
+  def __init__(self, credentials=None):
+    self.credentials = credentials
+
+  def fetch(self, oozie_api):
+    configuration = oozie_api.get_configuration()
+    self.credentials = self._parse_oozie(configuration)
+
+  def _parse_oozie(self, configuration_dic):
+    return dict([cred.strip().split('=') for cred in configuration_dic.get('oozie.credentials.credentialclasses', '').strip().split(',') if cred])
+
+  @property
+  def class_to_name_credentials(self):
+    return dict((v,k) for k, v in self.credentials.iteritems())
+
+  def get_properties(self, metastore=None):
+    credentials = {}
+    from beeswax import hive_site
+
+    if metastore is None:
+      metastore = hive_site.get_metastore()
+
+    if not metastore:
+      metastore = {}
+      LOG.warn('Could not get all the Oozie credentials: hive-site.xml required on the Hue host.')
+
+    credentials[self.hive_name] = {
+      'xml_name': self.hive_name,
+      'properties': [
+         ('hcat.metastore.uri', metastore.get('thrift_uri')),
+         ('hcat.metastore.principal', metastore.get('kerberos_principal')),
+      ]
+    }
+
+    credentials[self.hiveserver2_name] = {
+      'xml_name': self.hiveserver2_name,
+      'properties': [
+         ('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()),
+         ('hive2.server.principal', metastore.get('kerberos_principal')),
+      ]
+    }
+
+    credentials[self.hbase_name] = {
+      'xml_name': self.hbase_name,
+      'properties': []
+    }
+
+    LOG.info('Using credentials: %s' % credentials)
+
+    return credentials
+
+  @property
+  def hive_name(self):
+    return self.class_to_name_credentials.get('org.apache.oozie.action.hadoop.HCatCredentials')
+
+  @property
+  def hiveserver2_name(self):
+    return self.class_to_name_credentials.get('org.apache.oozie.action.hadoop.Hive2Credentials')
+
+  @property
+  def hbase_name(self):
+    return self.class_to_name_credentials.get('org.apache.oozie.action.hadoop.HbaseCredentials')

+ 88 - 0
desktop/libs/liboozie/src/liboozie/credentials_test.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from nose.tools import assert_equal, assert_true
+
+import beeswax.conf
+
+from liboozie.credentials import Credentials
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TestCredentials():
+  CREDENTIALS = {
+    "hcat": "org.apache.oozie.action.hadoop.HCatCredentials",
+    "hive2": "org.apache.oozie.action.hadoop.Hive2Credentials",
+    "hbase": "org.apache.oozie.action.hadoop.HbaseCredentials"
+  }
+
+  def test_parse_oozie(self):
+    oozie_credentialclasses = """
+           hbase=org.apache.oozie.action.hadoop.HbaseCredentials,
+           hcat=org.apache.oozie.action.hadoop.HCatCredentials,
+           hive2=org.apache.oozie.action.hadoop.Hive2Credentials
+    """
+    oozie_config = {'oozie.credentials.credentialclasses': oozie_credentialclasses}
+
+    creds = Credentials()
+
+    assert_equal({
+        'hive2': 'org.apache.oozie.action.hadoop.Hive2Credentials',
+        'hbase': 'org.apache.oozie.action.hadoop.HbaseCredentials',
+        'hcat': 'org.apache.oozie.action.hadoop.HCatCredentials'
+      }, creds._parse_oozie(oozie_config)
+    )
+
+  def test_gen_properties(self):
+    creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
+
+    metastore = {
+      'thrift_uri': 'thrift://hue-koh-chang:9999',
+      'kerberos_principal': 'hive',
+    }
+
+    finish = (
+      beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
+      beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
+    )
+
+    try:
+      assert_equal({
+          'hcat': {
+            'xml_name': 'hcat',
+            'properties': [
+                ('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
+                ('hcat.metastore.principal', 'hive')
+            ]},
+         'hive2': {
+            'xml_name': 'hive2',
+            'properties': [
+                ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345'),
+                ('hive2.server.principal', 'hive')
+          ]},
+         'hbase': {
+             'xml_name': 'hbase',
+             'properties': []
+          }
+        }, creds.get_properties(metastore))
+    finally:
+      for f in finish:
+        f()

+ 2 - 2
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -298,6 +298,6 @@ class OozieApi(object):
       nominal_end=2013-06-23T00:01Z
     """
     params = self._get_params()
-    params['filter'] = ';'.join(['%s=%s' % (key, val) for key, val in kwargs.iteritems()])    
+    params['filter'] = ';'.join(['%s=%s' % (key, val) for key, val in kwargs.iteritems()])
     resp = self._root.get('sla', params)
-    return resp['slaSummaryList'] 
+    return resp['slaSummaryList']

+ 6 - 11
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -29,6 +29,7 @@ from hadoop.fs.hadoopfs import Hdfs
 from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR
 from jobsub.parameterization import find_variables
+from liboozie.credentials import Credentials
 
 LOG = logging.getLogger(__name__)
 
@@ -186,17 +187,11 @@ class Submission(object):
         self.job.HUE_ID: self.job.id
       })
 
-    # Even if no Hive action for now
-    from beeswax.hive_site import get_metastore
-    metastore = get_metastore()
-
-    if metastore and metastore.get('use_sasl') and False: # Disabled for now
-      self.properties.update({
-         'is_kerberized_hive': True,
-         'credential_type': 'hcat',
-         'thrift_server': metastore.get('thrift_uri'),
-         'hive_principal': metastore.get('kerberos_principal')
-      })
+    # Generate credentials when using security
+    if self.api.security_enabled:
+      credentials = Credentials()
+      credentials.fetch(self.api)
+      self.properties['credentials'] = credentials.get_properties()
 
   def _create_deployment_dir(self):
     """

+ 0 - 2
desktop/libs/liboozie/src/liboozie/tests.py

@@ -116,8 +116,6 @@ class TestSubmission():
 
   def setUp(self):
     self.c = make_logged_in_client(is_superuser=False)
-#    grant_access("test", "test", "oozie")
-#    add_to_group("test")
     self.user = User.objects.get(username='test')
 
   def test_get_external_parameters(self):