Pārlūkot izejas kodu

[beeswax] Use first URI of multiple hive.metastore.uris values

Abraham Elmahrek 12 gadi atpakaļ
vecāks
revīzija
f93f8e7

+ 4 - 3
apps/beeswax/src/beeswax/hive_site.py

@@ -75,16 +75,17 @@ def get_metastore():
   if not _METASTORE_LOC_CACHE:
     kerberos_principal = security_util.get_kerberos_principal(get_conf().get(_CNF_METASTORE_KERBEROS_PRINCIPAL, None))
     kerberos_principal_components = security_util.get_components(kerberos_principal)
-    thrift_uri = get_conf().get(_CNF_METASTORE_URIS)
-    is_local = thrift_uri is None or thrift_uri == ''
+    thrift_uris = get_conf().get(_CNF_METASTORE_URIS)
+    is_local = thrift_uris is None or thrift_uris == ''
     if is_local:
       host = beeswax.conf.BEESWAX_META_SERVER_HOST.get()
       port = beeswax.conf.BEESWAX_META_SERVER_PORT.get()
     else:
+      thrift_uri = thrift_uris.split(",")[0]
       host, port = 'undefined', '0'
       match = _THRIFT_URI_RE.match(thrift_uri)
       if not match:
-        LOG.fatal('Cannot understand remote metastore uri "%s"' % (thrift_uri,))
+        LOG.fatal('Cannot understand remote metastore uri "%s"' % thrift_uri)
       else:
         host, port = match.groups()
       if str(get_conf().get(_CNF_METASTORE_SASL, 'false')).lower() == 'true' and len(kerberos_principal_components) == 3:

+ 50 - 0
apps/beeswax/src/beeswax/tests.py

@@ -1371,6 +1371,56 @@ def test_hive_site_sasl():
     shutil.rmtree(tmpdir)
 
 
+def test_hive_site_multi_metastore_uris():
+  """Test hive-site parsing"""
+  HIVE_SITE = """
+    <configuration>
+      <property>
+        <name>hive.metastore.local</name>
+        <value>false</value>
+      </property>
+
+      <property>
+        <name>hive.metastore.uris</name>
+        <value>thrift://darkside-12345:9998,thrift://darkside-1234:9999</value>
+      </property>
+
+      <property>
+        <name>hive.metastore.warehouse.dir</name>
+        <value>/abc</value>
+      </property>
+
+      <property>
+        <name>hive.metastore.kerberos.principal</name>
+        <value>test/test.com@TEST.COM</value>
+      </property>
+    </configuration>
+  """
+
+  beeswax.hive_site.reset()
+  tmpdir = tempfile.mkdtemp()
+  saved = None
+  try:
+    file(os.path.join(tmpdir, 'hive-site.xml'), 'w').write(HIVE_SITE)
+
+    # We just replace the Beeswax conf variable
+    class Getter(object):
+      def get(self):
+        return tmpdir
+
+    saved = beeswax.conf.BEESWAX_HIVE_CONF_DIR
+    beeswax.conf.BEESWAX_HIVE_CONF_DIR = Getter()
+
+    is_local, host, port, kerberos_principal = beeswax.hive_site.get_metastore()
+    assert_false(is_local)
+    assert_equal(host, 'darkside-12345')
+    assert_equal(port, 9998)
+  finally:
+    if saved is not None:
+      beeswax.conf.BEESWAX_HIVE_CONF_DIR = saved
+    shutil.rmtree(tmpdir)
+
+
 def test_collapse_whitespace():
   assert_equal("", collapse_whitespace("\t\n\n  \n\t \n"))
   assert_equal("x", collapse_whitespace("\t\nx\n  \n\t \n"))