Преглед на файлове

[hadoop] Support new sentry.hdfs.integration.path.prefixes hdfs-site.xml property

sentry.authorization-provider.hdfs-path-prefixes is deprecated
Romain Rigaux преди 10 години
родител
ревизия
09bef779db
променени са 3 файла, в които са добавени 74 реда и са изтрити 4 реда
  1. 1 1
      desktop/libs/hadoop/src/hadoop/fs/webhdfs.py
  2. 13 3
      desktop/libs/hadoop/src/hadoop/hdfs_site.py
  3. 60 0
      desktop/libs/hadoop/src/hadoop/test_hdfs_site.py

+ 1 - 1
desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -117,7 +117,7 @@ class WebHdfs(Hdfs):
 
   @classmethod
   def is_sentry_managed(cls, path):
-    prefixes = get_nn_sentry_prefixes().split(',')
+    prefixes = get_nn_sentry_prefixes()
 
     return any([path == p or path.startswith(p + '/') for p in prefixes if p])
 

+ 13 - 3
desktop/libs/hadoop/src/hadoop/hdfs_site.py

@@ -27,9 +27,9 @@ LOG = logging.getLogger(__name__)
 
 _HDFS_SITE_DICT = None
 
-
 _CNF_NN_PERMISSIONS_UMASK_MODE = 'fs.permissions.umask-mode'
-_CNF_NN_SENTRY_PREFIX = 'sentry.authorization-provider.hdfs-path-prefixes'
+_CNF_NN_SENTRY_PREFIXES = 'sentry.authorization-provider.hdfs-path-prefixes' # Deprecated
+_CNF_NN_SENTRY_PATH_PREFIXES = 'sentry.hdfs.integration.path.prefixes'
 _CNF_NN_PERMISSIONS_SUPERGROUP = 'dfs.permissions.superusergroup'
 
 
@@ -51,8 +51,18 @@ def get_umask_mode():
 
   return int(umask, 8)
 
+
 def get_nn_sentry_prefixes():
-  return get_conf().get(_CNF_NN_SENTRY_PREFIX, '')
+  prefixes = set()
+
+  if get_conf().get(_CNF_NN_SENTRY_PREFIXES, ''):
+    prefixes |= set(get_conf().get(_CNF_NN_SENTRY_PREFIXES, '').split(','))
+
+  if get_conf().get(_CNF_NN_SENTRY_PATH_PREFIXES, ''):
+    prefixes |= set(get_conf().get(_CNF_NN_SENTRY_PATH_PREFIXES, '').split(','))
+
+  return list(prefixes)
+
 
 def get_supergroup():
   return get_conf().get(_CNF_NN_PERMISSIONS_SUPERGROUP, 'supergroup')

+ 60 - 0
desktop/libs/hadoop/src/hadoop/test_hdfs_site.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import conf
+import logging
+import os
+import tempfile
+
+from nose.tools import assert_true, assert_equal, assert_false, assert_not_equal, assert_raises
+
+from hadoop import hdfs_site
+
+
+LOG = logging.getLogger(__name__)
+
+
+def test_hdfs_site():
+  hadoop_home = tempfile.mkdtemp()
+  finish = None
+
+  try:
+    xml = """<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>sentry.authorization-provider.hdfs-path-prefixes</name>
+    <value>/path/a,/path/b,/path/1</value>
+  </property>
+  <property>
+    <name>sentry.hdfs.integration.path.prefixes</name>
+    <value>/path/c,/path/d,/path/1</value>
+  </property>
+</configuration>
+    """
+    file(os.path.join(hadoop_home, 'hdfs-site.xml'), 'w').write(xml)
+
+    finish = conf.HDFS_CLUSTERS['default'].HADOOP_CONF_DIR.set_for_testing(hadoop_home)
+    hdfs_site.reset()
+
+    assert_equal(set(hdfs_site.get_nn_sentry_prefixes()), set(['/path/a', '/path/b', '/path/c', '/path/d', '/path/1']))
+    assert_equal(len(hdfs_site.get_nn_sentry_prefixes()), 5)
+  finally:
+    hdfs_site.reset()
+    if finish:
+      finish()