Browse Source

[beeswax] use hue principal if local metastore with sasl and kerb.

Abraham Elmahrek 12 years ago
parent
commit
9630d98f13
2 changed files with 134 additions and 124 deletions
  1. 13 4
      apps/beeswax/src/beeswax/hive_site.py
  2. 121 120
      apps/beeswax/src/beeswax/tests.py

+ 13 - 4
apps/beeswax/src/beeswax/hive_site.py

@@ -24,10 +24,12 @@ import os.path
 import re
 import socket
 
+from desktop.conf import KERBEROS
 from desktop.lib import security_util
 
 import beeswax.conf
 from hadoop import confparse
+from hadoop import cluster
 
 LOG = logging.getLogger(__name__)
 
@@ -75,14 +77,17 @@ def get_metastore():
   """
   global _METASTORE_LOC_CACHE
   if not _METASTORE_LOC_CACHE:
-    kerberos_principal = security_util.get_kerberos_principal(get_conf().get(_CNF_METASTORE_KERBEROS_PRINCIPAL, None), socket.getfqdn())
-    kerberos_principal_components = security_util.get_components(kerberos_principal)
     thrift_uris = get_conf().get(_CNF_METASTORE_URIS)
     is_local = thrift_uris is None or thrift_uris == ''
+
     if is_local:
+      cluster_conf = cluster.get_cluster_conf_for_job_submission()
+      use_sasl = cluster_conf is not None and cluster_conf.SECURITY_ENABLED.get()
       host = beeswax.conf.BEESWAX_META_SERVER_HOST.get()
       port = beeswax.conf.BEESWAX_META_SERVER_PORT.get()
+      kerberos_principal = security_util.get_kerberos_principal(KERBEROS.HUE_PRINCIPAL.get(), socket.getfqdn())
     else:
+      use_sasl = str(get_conf().get(_CNF_METASTORE_SASL, 'false')).lower() == 'true'
       thrift_uri = thrift_uris.split(",")[0]
       host, port = 'undefined', '0'
       match = _THRIFT_URI_RE.match(thrift_uri)
@@ -90,8 +95,12 @@ def get_metastore():
         LOG.fatal('Cannot understand remote metastore uri "%s"' % thrift_uri)
       else:
         host, port = match.groups()
-      if str(get_conf().get(_CNF_METASTORE_SASL, 'false')).lower() == 'true' and len(kerberos_principal_components) == 3:
-        host = kerberos_principal_components[1]
+      kerberos_principal = security_util.get_kerberos_principal(get_conf().get(_CNF_METASTORE_KERBEROS_PRINCIPAL, None), socket.getfqdn())
+
+    kerberos_principal_components = security_util.get_components(kerberos_principal)
+    if use_sasl and len(kerberos_principal_components) == 3:
+      host = kerberos_principal_components[1]
+
     _METASTORE_LOC_CACHE = (is_local, host, int(port), kerberos_principal)
   return _METASTORE_LOC_CACHE
 

+ 121 - 120
apps/beeswax/src/beeswax/tests.py

@@ -37,6 +37,7 @@ from django.utils.encoding import smart_str
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 
+from desktop.conf import KERBEROS
 from desktop.lib.django_test_util import make_logged_in_client, assert_equal_mod_whitespace
 from desktop.lib.django_test_util import assert_similar_pages
 from desktop.lib.test_utils import grant_access
@@ -1303,49 +1304,20 @@ Starting Job = job_201003191517_0003, Tracking URL = http://localhost:50030/jobd
     beeswax.views._parse_out_hadoop_jobs(sample_log))
   assert_equal([], beeswax.views._parse_out_hadoop_jobs("nothing to see here"))
 
-
 def test_hive_site():
   """Test hive-site parsing"""
-  HIVE_SITE = """
-    <configuration>
-      <property>
-        <name>hive.metastore.local</name>
-        <value>false</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.uris</name>
-        <value>thrift://darkside-1234:9999</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.warehouse.dir</name>
-        <value>/abc</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.kerberos.principal</name>
-        <value>test/test.com@TEST.COM</value>
-      </property>
-
-      <property>
-        <name>hive.server2.authentication.kerberos.principal</name>
-        <value>hs2test/test.com@TEST.COM</value>
-      </property>
-    </configuration>
-  """
-
-  beeswax.hive_site.reset()
   tmpdir = tempfile.mkdtemp()
   saved = None
   try:
-    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(HIVE_SITE)
-
     # We just replace the Beeswax conf variable
     class Getter(object):
       def get(self):
         return tmpdir
 
+    xml = hive_site_xml(is_local=False, use_sasl=False)
+    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
+
+    beeswax.hive_site.reset()
     saved = beeswax.conf.BEESWAX_HIVE_CONF_DIR
     beeswax.conf.BEESWAX_HIVE_CONF_DIR = Getter()
 
@@ -1364,41 +1336,19 @@ def test_hive_site():
 
 def test_hive_site_host_pattern():
   """Test hive-site parsing"""
-  HIVE_SITE = """
-    <configuration>
-      <property>
-        <name>hive.metastore.local</name>
-        <value>false</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.uris</name>
-        <value>thrift://%s:9999</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.warehouse.dir</name>
-        <value>/abc</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.kerberos.principal</name>
-        <value>test/_HOST@TEST.COM</value>
-      </property>
-    </configuration>
-  """ % socket.getfqdn()
-
-  beeswax.hive_site.reset()
   tmpdir = tempfile.mkdtemp()
   saved = None
   try:
-    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(HIVE_SITE)
-
     # We just replace the Beeswax conf variable
     class Getter(object):
       def get(self):
         return tmpdir
 
+    thrift_uris = 'thrift://%s:9999' % socket.getfqdn()
+    xml = hive_site_xml(is_local=False, use_sasl=False, thrift_uris=thrift_uris, kerberos_principal='test/_HOST@TEST.COM')
+    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
+
+    beeswax.hive_site.reset()
     saved = beeswax.conf.BEESWAX_HIVE_CONF_DIR
     beeswax.conf.BEESWAX_HIVE_CONF_DIR = Getter()
 
@@ -1414,54 +1364,36 @@ def test_hive_site_host_pattern():
       beeswax.conf.BEESWAX_HIVE_CONF_DIR = saved
     shutil.rmtree(tmpdir)
 
-def test_hive_site_sasl():
-  """Test hive-site parsing with sasl enabled"""
-  HIVE_SITE = """
-    <configuration>
-      <property>
-        <name>hive.metastore.local</name>
-        <value>false</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.uris</name>
-        <value>thrift://darkside-1234:9999</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.warehouse.dir</name>
-        <value>/abc</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.kerberos.principal</name>
-        <value>test/test.com@TEST.COM</value>
-      </property>
-
-      <property>
-        <name>hive.metastore.sasl.enabled</name>
-        <value>true</value>
-      </property>
-    </configuration>
-  """
-
-  beeswax.hive_site.reset()
+def test_hive_site_external_metastore():
+  """Test hive-site parsing with sasl enabled and external metastore"""
   tmpdir = tempfile.mkdtemp()
   saved = None
   try:
-    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(HIVE_SITE)
-
     # We just replace the Beeswax conf variable
     class Getter(object):
       def get(self):
         return tmpdir
 
+    xml = hive_site_xml(is_local=False, use_sasl=False)
+    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
+
+    beeswax.hive_site.reset()
     saved = beeswax.conf.BEESWAX_HIVE_CONF_DIR
     beeswax.conf.BEESWAX_HIVE_CONF_DIR = Getter()
 
+    # No sasl
+    is_local, host, port, kerberos_principal = beeswax.hive_site.get_metastore()
+    assert_false(is_local)
+    assert_equal(host, 'darkside-1234')
+    assert_equal(port, 9999)
+    assert_equal(beeswax.hive_site.get_conf()['hive.metastore.warehouse.dir'], u'/abc')
+    assert_equal(kerberos_principal, 'test/test.com@TEST.COM')
+
+    # Sasl
+    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(hive_site_xml(is_local=False, use_sasl=True))
+    beeswax.hive_site.reset()
     is_local, host, port, kerberos_principal = beeswax.hive_site.get_metastore()
     assert_false(is_local)
-    # Should look at kerberos instance for host.
     assert_equal(host, 'test.com')
     assert_equal(port, 9999)
     assert_equal(beeswax.hive_site.get_conf()['hive.metastore.warehouse.dir'], u'/abc')
@@ -1472,44 +1404,69 @@ def test_hive_site_sasl():
       beeswax.conf.BEESWAX_HIVE_CONF_DIR = saved
     shutil.rmtree(tmpdir)
 
+def test_hive_site_local_metastore():
+  """Test hive-site parsing with sasl enabled and local metastore"""
+  tmpdir = tempfile.mkdtemp()
+  saved = None
+  try:
+    # We just replace the Beeswax conf variable
+    class Getter(object):
+      def get(self):
+        return tmpdir
 
-def test_hive_site_multi_metastore_uris():
-  """Test hive-site parsing"""
-  HIVE_SITE = """
-    <configuration>
-      <property>
-        <name>hive.metastore.local</name>
-        <value>false</value>
-      </property>
+    xml = hive_site_xml(is_local=True, use_sasl=True, thrift_uris='')
+    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
 
-      <property>
-        <name>hive.metastore.uris</name>
-        <value>thrift://darkside-12345:9998,thrift://darkside-1234:9999</value>
-      </property>
+    beeswax.hive_site.reset()
+    saved = beeswax.conf.BEESWAX_HIVE_CONF_DIR
+    beeswax.conf.BEESWAX_HIVE_CONF_DIR = Getter()
 
-      <property>
-        <name>hive.metastore.warehouse.dir</name>
-        <value>/abc</value>
-      </property>
+    # No sasl
+    reset = []
+    reset.append(KERBEROS.HUE_PRINCIPAL.set_for_testing('hue/test.com@TEST.COM'))
+    is_local, host, port, kerberos_principal = beeswax.hive_site.get_metastore()
+    assert_true(is_local)
+    assert_equal(host, 'localhost')
+    assert_equal(port, 8003)
+    assert_equal(beeswax.hive_site.get_conf()['hive.metastore.warehouse.dir'], u'/abc')
+    assert_equal(kerberos_principal, 'hue/test.com@TEST.COM')
+
+    # Sasl
+    reset.extend([
+      hadoop.conf.HDFS_CLUSTERS['default'].SECURITY_ENABLED.set_for_testing(True),
+      hadoop.conf.MR_CLUSTERS['default'].SECURITY_ENABLED.set_for_testing(True),
+      hadoop.conf.YARN_CLUSTERS['default'].SECURITY_ENABLED.set_for_testing(True)
+    ])
+    beeswax.hive_site.reset()
+    is_local, host, port, kerberos_principal = beeswax.hive_site.get_metastore()
+    assert_true(is_local)
+    assert_equal(host, 'test.com')
+    assert_equal(port, 8003)
+    assert_equal(beeswax.hive_site.get_conf()['hive.metastore.warehouse.dir'], u'/abc')
+    assert_equal(kerberos_principal, 'hue/test.com@TEST.COM')
+  finally:
+    for finish in reset:
+      finish()
+    beeswax.hive_site.reset()
+    if saved is not None:
+      beeswax.conf.BEESWAX_HIVE_CONF_DIR = saved
+    shutil.rmtree(tmpdir)
 
-      <property>
-        <name>hive.metastore.kerberos.principal</name>
-        <value>test/test.com@TEST.COM</value>
-      </property>
-    </configuration>
-  """
 
-  beeswax.hive_site.reset()
+def test_hive_site_multi_metastore_uris():
+  """Test hive-site parsing"""
   tmpdir = tempfile.mkdtemp()
   saved = None
   try:
-    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(HIVE_SITE)
-
     # We just replace the Beeswax conf variable
     class Getter(object):
       def get(self):
         return tmpdir
 
+    xml = hive_site_xml(is_local=False, use_sasl=False, thrift_uris='thrift://darkside-12345:9998,thrift://darkside-1234:9999')
+    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(xml)
+
+    beeswax.hive_site.reset()
     saved = beeswax.conf.BEESWAX_HIVE_CONF_DIR
     beeswax.conf.BEESWAX_HIVE_CONF_DIR = Getter()
 
@@ -1661,3 +1618,47 @@ class TestWithMockedServer(object):
 def search_log_line(component, expected_log, all_logs):
   """Checks if 'expected_log' can be found in one line of 'all_logs' outputed by the logging component 'component'."""
   return re.compile('.+?%(component)s(.+?)%(expected_log)s' % {'component': component, 'expected_log': expected_log}).search(all_logs)
+
+def hive_site_xml(is_local=False, use_sasl=False, thrift_uris='thrift://darkside-1234:9999',
+                  warehouse_dir='/abc', kerberos_principal='test/test.com@TEST.COM',
+                  hs2_kerberos_principal='hs2test/test.com@TEST.COM'):
+  return """
+    <configuration>
+      <property>
+        <name>hive.metastore.local</name>
+        <value>%(is_local)s</value>
+      </property>
+
+      <property>
+        <name>hive.metastore.uris</name>
+        <value>%(thrift_uris)s</value>
+      </property>
+
+      <property>
+        <name>hive.metastore.warehouse.dir</name>
+        <value>%(warehouse_dir)s</value>
+      </property>
+
+      <property>
+        <name>hive.metastore.kerberos.principal</name>
+        <value>%(kerberos_principal)s</value>
+      </property>
+
+      <property>
+        <name>hive.server2.authentication.kerberos.principal</name>
+        <value>%(hs2_kerberos_principal)s</value>
+      </property>
+
+      <property>
+        <name>hive.metastore.sasl.enabled</name>
+        <value>%(use_sasl)s</value>
+      </property>
+    </configuration>
+  """ % {
+    'is_local': str(is_local).lower(),
+    'thrift_uris': thrift_uris,
+    'warehouse_dir': warehouse_dir,
+    'kerberos_principal': kerberos_principal,
+    'hs2_kerberos_principal': hs2_kerberos_principal,
+    'use_sasl': str(use_sasl).lower()
+  }