Преглед изворни кода

HUE-5504 [oozie] JDBC URL specified in Hive2 action should override credential properties

krish пре 8 година
родитељ
комит
e907c94365

+ 31 - 0
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -302,6 +302,37 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
       credentials.fetch(self.api)
       credentials.fetch(self.api)
       self.properties['credentials'] = credentials.get_properties()
       self.properties['credentials'] = credentials.get_properties()
 
 
+      self._update_credentials_from_hive_action(credentials)
+
+
+  def _update_credentials_from_hive_action(self, credentials):
+    """
+    Hive JDBC url from conf should be replaced when URL is set in hive action. Use _HOST from
+    this URL to update the hive2_host in hive principal hive/hive2_host@YOUR-REALM.COM
+    """
+    if hasattr(self.job, 'nodes'):
+      for action in self.job.nodes:
+        if action.data['type'] in ('hive2', 'hive-document') and \
+                        credentials.hiveserver2_name in self.properties['credentials'] and \
+                        action.data['properties']['jdbc_url']:
+          try:
+            hive_jdbc_url = action.data['properties']['jdbc_url']
+            hive_host_from_action = hive_jdbc_url.split('//')[1].split(':')[0]
+
+            hive_principal_from_conf = self.properties['credentials'][credentials.hiveserver2_name]['properties'][1][1]
+            updated_hive_principal = hive_principal_from_conf.split('/')[0] + '/' + hive_host_from_action + '@' + hive_principal_from_conf.split('@')[1]
+
+            self.properties['credentials'][credentials.hiveserver2_name]['properties'] = [
+              ('hive2.jdbc.url', hive_jdbc_url),
+              ('hive2.server.principal', updated_hive_principal)
+            ]
+          except Exception, ex:
+            msg = 'Failed to update the Hive JDBC URL from %s action properties: %s' % (action.data['type'], str(ex))
+            LOG.error(msg)
+            raise PopupException(message=_(msg), detail=str(ex))
+
+
+
   def _create_deployment_dir(self):
   def _create_deployment_dir(self):
     """
     """
     Return the job deployment directory in HDFS, creating it if necessary.
     Return the job deployment directory in HDFS, creating it if necessary.

+ 57 - 1
desktop/libs/liboozie/src/liboozie/submittion2_tests.py

@@ -19,17 +19,22 @@ import logging
 
 
 from django.contrib.auth.models import User
 from django.contrib.auth.models import User
 from nose.plugins.attrib import attr
 from nose.plugins.attrib import attr
-from nose.tools import assert_equal, assert_true, assert_not_equal
+from nose.tools import assert_equal, assert_true, assert_not_equal, assert_raises
+
+import beeswax
 
 
 from hadoop import cluster, pseudo_hdfs4
 from hadoop import cluster, pseudo_hdfs4
 from hadoop.conf import HDFS_CLUSTERS, MR_CLUSTERS, YARN_CLUSTERS
 from hadoop.conf import HDFS_CLUSTERS, MR_CLUSTERS, YARN_CLUSTERS
 
 
 from desktop.lib.test_utils import clear_sys_caches
 from desktop.lib.test_utils import clear_sys_caches
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
+from desktop.lib.exceptions_renderable import PopupException
 from oozie.models2 import Node
 from oozie.models2 import Node
 from oozie.tests import OozieMockBase
 from oozie.tests import OozieMockBase
 
 
 from liboozie.conf import USE_LIBPATH_FOR_JARS
 from liboozie.conf import USE_LIBPATH_FOR_JARS
+from liboozie.credentials import Credentials
+from liboozie.credentials_tests import TestCredentials
 from liboozie.submission2 import Submission
 from liboozie.submission2 import Submission
 
 
 
 
@@ -313,3 +318,54 @@ oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
                    'queueName': 'default'
                    'queueName': 'default'
                   },
                   },
                  parameters)
                  parameters)
+
+  def test_update_credentials_from_hive_action(self):
+
+    class TestJob():
+      XML_FILE_NAME = 'workflow.xml'
+
+      def __init__(self):
+        self.deployment_dir = '/tmp/test'
+        self.nodes = [
+            Node({'id': '1', 'type': 'hive-document', 'properties': {'jdbc_url': u'jdbc:hive2://test-replace-url:12345/default', 'password': u'test'}})
+        ]
+
+    user = User.objects.get(username='test')
+    submission = Submission(user, job=TestJob(), fs=MockFs(logical_name='fsname'), jt=MockJt(logical_name='jtname'))
+
+    finish = (
+      beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
+      beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
+    )
+
+    try:
+      creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
+      hive_properties = {
+        'thrift_uri': 'thrift://first-url:9999',
+        'kerberos_principal': 'hive',
+        'hive2.server.principal': 'hive/hive2_host@test-realm.com',
+      }
+
+      submission.properties['credentials'] = creds.get_properties(hive_properties)
+      submission._update_credentials_from_hive_action(creds)
+
+      assert_equal(submission.properties['credentials'][creds.hiveserver2_name]['properties'], [
+            ('hive2.jdbc.url', u'jdbc:hive2://test-replace-url:12345/default'),
+            ('hive2.server.principal', u'hive/test-replace-url@test-realm.com')
+          ]
+      )
+
+      # Test parsing failure
+      hive_properties = {
+        'thrift_uri': 'thrift://first-url:9999',
+        'kerberos_principal': 'hive',
+        'hive2.server.principal': 'hive',
+      }
+
+      submission.properties['credentials'] = creds.get_properties(hive_properties)
+
+      assert_raises(PopupException,  submission._update_credentials_from_hive_action, creds)
+
+    finally:
+      for f in finish:
+        f()