Browse Source

[oozie] Credentials can be combined comma separatedly in one action

Tested with hive and hiveserver2.
We should update the properties before deploying.
Romain Rigaux 12 năm trước cách đây
mục cha
commit
e03ea6a

+ 1 - 1
apps/beeswax/src/beeswax/hive_site.py

@@ -120,7 +120,7 @@ def hiveserver2_impersonation_enabled():
   return get_conf().get(_CNF_HIVESERVER2_IMPERSONATION, 'FALSE').upper() == 'TRUE'
 
 def hiveserver2_jdbc_url():
-  return 'jdbc:hive2://%s:%s' % (beeswax.conf.HIVE_SERVER_HOST.get(), beeswax.conf.HIVE_SERVER_PORT.get())
+  return 'jdbc:hive2://%s:%s/default' % (beeswax.conf.HIVE_SERVER_HOST.get(), beeswax.conf.HIVE_SERVER_PORT.get())
 
 
 def _parse_hive_site():

+ 1 - 1
apps/oozie/src/oozie/models.py

@@ -200,7 +200,7 @@ class Job(models.Model):
   def find_all_parameters(self):
     params = self.find_parameters()
 
-    if hasattr(self, 'sla'):
+    if hasattr(self, 'sla') and self.sla_enabled:
       for param in find_json_parameters(self.sla):
         if param not in params:
           params[param] = ''

+ 1 - 1
apps/oozie/src/oozie/templates/editor/gen/workflow-common.xml.mako

@@ -24,7 +24,7 @@
 <%def name="filelink(path)">${ '#' in path and path or path + '#' + posixpath.basename(path) }</%def>
 
 
-<%def name="credentials(credentials)">${ ' cred="%s"' % ' '.join([cred['name'] for cred in credentials if cred['value']]) if credentials else '' | n,unicode }</%def>
+<%def name="credentials(credentials)">${ ' cred="%s"' % ','.join([cred['name'] for cred in credentials if cred['value']]) if credentials else '' | n,unicode }</%def>
 
 
 <%def name="prepares(prepares)">

+ 19 - 2
apps/oozie/src/oozie/tests.py

@@ -1232,7 +1232,7 @@ class TestEditor(OozieMockBase):
       beeswax.conf.HIVE_CONF_DIR = Getter()
 
       action1 = Node.objects.get(workflow=self.wf, name='MyHive')
-      action1.credentials = [{'name': 'hcat', 'value': True}, {'name': 'hbase', 'value': False}]
+      action1.credentials = [{'name': 'hcat', 'value': True}, {'name': 'hbase', 'value': False}, {'name': 'hive2', 'value': True}]
       action1.save()
 
       xml = self.wf.to_xml(mapping={
@@ -1243,6 +1243,13 @@ class TestEditor(OozieMockBase):
                       ('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
                       ('hcat.metastore.principal', 'hive')
                   ]
+              },
+              'hive2': {
+                  'xml_name': 'hive2',
+                  'properties': [
+                      ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:8888'),
+                      ('hive2.server.principal', 'hive')
+                  ]
               }
           }
         }
@@ -1270,9 +1277,19 @@ class TestEditor(OozieMockBase):
         <value>hive</value>
       </property>
     </credential>
+    <credential name="hive2" type="hive2">
+      <property>
+        <name>hive2.jdbc.url</name>
+        <value>jdbc:hive2://hue-koh-chang:8888</value>
+      </property>
+      <property>
+        <name>hive2.server.principal</name>
+        <value>hive</value>
+      </property>
+    </credential>
   </credentials>
     <start to="MyHive"/>
-    <action name="MyHive" cred="hcat">
+    <action name="MyHive" cred="hcat,hive2">
         <hive xmlns="uri:oozie:hive-action:0.2">
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>

+ 1 - 1
desktop/libs/liboozie/src/liboozie/credentials_test.py

@@ -75,7 +75,7 @@ class TestCredentials():
          'hive2': {
             'xml_name': 'hive2',
             'properties': [
-                ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345'),
+                ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'),
                 ('hive2.server.principal', 'hive')
           ]},
          'hbase': {

+ 4 - 2
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -75,6 +75,7 @@ class Submission(object):
     jobtracker = cluster.get_cluster_addr_for_job_submission()
 
     if deployment_dir is None:
+      self._update_properties(jobtracker) # Needed as we need to set some properties like Credentials before
       deployment_dir = self.deploy()
 
     self._update_properties(jobtracker, deployment_dir)
@@ -153,6 +154,7 @@ class Submission(object):
 
     return deployment_dir
 
+
   def get_external_parameters(self, application_path):
     """From XML and job.properties HDFS files"""
     deployment_dir = os.path.dirname(application_path)
@@ -175,7 +177,7 @@ class Submission(object):
                               for line in properties.split('\n') if not line.startswith('#') and len(line.strip().split('=')) == 2]))
     return parameters
 
-  def _update_properties(self, jobtracker_addr, deployment_dir):
+  def _update_properties(self, jobtracker_addr, deployment_dir=None):
     LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
     if self.jt and self.jt.logical_name:
       jobtracker_addr = self.jt.logical_name
@@ -190,7 +192,7 @@ class Submission(object):
       'nameNode': fs_defaultfs,
     })
 
-    if self.job:
+    if self.job and deployment_dir:
       self.properties.update({
         self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
         self.job.HUE_ID: self.job.id