Pārlūkot izejas kodu

[oozie] Credentials can be combined comma separatedly in one action

Tested with hive and hiveserver2.
We should update the properties before deploying.
Romain Rigaux 12 gadi atpakaļ
vecāks
revīzija
e03ea6a

+ 1 - 1
apps/beeswax/src/beeswax/hive_site.py

@@ -120,7 +120,7 @@ def hiveserver2_impersonation_enabled():
   return get_conf().get(_CNF_HIVESERVER2_IMPERSONATION, 'FALSE').upper() == 'TRUE'
 
 def hiveserver2_jdbc_url():
-  return 'jdbc:hive2://%s:%s' % (beeswax.conf.HIVE_SERVER_HOST.get(), beeswax.conf.HIVE_SERVER_PORT.get())
+  return 'jdbc:hive2://%s:%s/default' % (beeswax.conf.HIVE_SERVER_HOST.get(), beeswax.conf.HIVE_SERVER_PORT.get())
 
 
 def _parse_hive_site():

+ 1 - 1
apps/oozie/src/oozie/models.py

@@ -200,7 +200,7 @@ class Job(models.Model):
   def find_all_parameters(self):
     params = self.find_parameters()
 
-    if hasattr(self, 'sla'):
+    if hasattr(self, 'sla') and self.sla_enabled:
       for param in find_json_parameters(self.sla):
         if param not in params:
           params[param] = ''

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-common.xml.mako

@@ -24,7 +24,7 @@
 <%def name="filelink(path)">${ '#' in path and path or path + '#' + posixpath.basename(path) }</%def>
 
 
-<%def name="credentials(credentials)">${ ' cred="%s"' % ' '.join([cred['name'] for cred in credentials if cred['value']]) if credentials else '' | n,unicode }</%def>
+<%def name="credentials(credentials)">${ ' cred="%s"' % ','.join([cred['name'] for cred in credentials if cred['value']]) if credentials else '' | n,unicode }</%def>
 
 
 <%def name="prepares(prepares)">

+ 19 - 2
apps/oozie/src/oozie/tests.py

@@ -1232,7 +1232,7 @@ class TestEditor(OozieMockBase):
       beeswax.conf.HIVE_CONF_DIR = Getter()
 
       action1 = Node.objects.get(workflow=self.wf, name='MyHive')
-      action1.credentials = [{'name': 'hcat', 'value': True}, {'name': 'hbase', 'value': False}]
+      action1.credentials = [{'name': 'hcat', 'value': True}, {'name': 'hbase', 'value': False}, {'name': 'hive2', 'value': True}]
       action1.save()
 
       xml = self.wf.to_xml(mapping={
@@ -1243,6 +1243,13 @@ class TestEditor(OozieMockBase):
                       ('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
                       ('hcat.metastore.principal', 'hive')
                   ]
+              },
+              'hive2': {
+                  'xml_name': 'hive2',
+                  'properties': [
+                      ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:8888'),
+                      ('hive2.server.principal', 'hive')
+                  ]
               }
           }
         }
@@ -1270,9 +1277,19 @@ class TestEditor(OozieMockBase):
         <value>hive</value>
       </property>
     </credential>
+    <credential name="hive2" type="hive2">
+      <property>
+        <name>hive2.jdbc.url</name>
+        <value>jdbc:hive2://hue-koh-chang:8888</value>
+      </property>
+      <property>
+        <name>hive2.server.principal</name>
+        <value>hive</value>
+      </property>
+    </credential>
   </credentials>
     <start to="MyHive"/>
-    <action name="MyHive" cred="hcat">
+    <action name="MyHive" cred="hcat,hive2">
         <hive xmlns="uri:oozie:hive-action:0.2">
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>

+ 1 - 1
desktop/libs/liboozie/src/liboozie/credentials_test.py

@@ -75,7 +75,7 @@ class TestCredentials():
          'hive2': {
             'xml_name': 'hive2',
             'properties': [
-                ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345'),
+                ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'),
                 ('hive2.server.principal', 'hive')
           ]},
          'hbase': {

+ 4 - 2
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -75,6 +75,7 @@ class Submission(object):
     jobtracker = cluster.get_cluster_addr_for_job_submission()
 
     if deployment_dir is None:
+      self._update_properties(jobtracker) # Needed as we need to set some properties like Credentials before
       deployment_dir = self.deploy()
 
     self._update_properties(jobtracker, deployment_dir)
@@ -153,6 +154,7 @@ class Submission(object):
 
     return deployment_dir
 
+
   def get_external_parameters(self, application_path):
     """From XML and job.properties HDFS files"""
     deployment_dir = os.path.dirname(application_path)
@@ -175,7 +177,7 @@ class Submission(object):
                               for line in properties.split('\n') if not line.startswith('#') and len(line.strip().split('=')) == 2]))
     return parameters
 
-  def _update_properties(self, jobtracker_addr, deployment_dir):
+  def _update_properties(self, jobtracker_addr, deployment_dir=None):
     LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
     if self.jt and self.jt.logical_name:
       jobtracker_addr = self.jt.logical_name
@@ -190,7 +192,7 @@ class Submission(object):
       'nameNode': fs_defaultfs,
     })
 
-    if self.job:
+    if self.job and deployment_dir:
       self.properties.update({
         self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
         self.job.HUE_ID: self.job.id