Explorar o código

HUE-5716 [impala] Add impalad_flag file format reader

krish %!s(int64=8) %!d(string=hai) anos
pai
achega
6f705845db

+ 8 - 0
apps/impala/src/impala/conf.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import logging
+import os
 import sys
 import socket
 
@@ -24,6 +25,7 @@ from desktop.conf import default_ssl_cacerts, default_ssl_validate, AUTH_USERNAM
   AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD
 from desktop.lib.conf import ConfigSection, Config, coerce_bool, coerce_csv, coerce_password_from_script
 from desktop.lib.exceptions import StructuredThriftTransportException
+from desktop.lib.paths import get_desktop_root
 
 from impala.settings import NICE_NAME
 
@@ -99,6 +101,12 @@ CONFIG_WHITELIST = Config(
   help=_t('A comma-separated list of white-listed Impala configuration properties that users are authorized to set.')
 )
 
+IMPALA_CONF_DIR = Config(
+  key='impala_conf_dir',
+  help=_t('Impala configuration directory, where impala_flags is located.'),
+  default=os.environ.get("HUE_CONF_DIR", get_desktop_root("conf")) + '/impala-conf'
+)
+
 SSL = ConfigSection(
   key='ssl',
   help=_t('SSL configuration for the server.'),

+ 72 - 0
apps/impala/src/impala/impala_flags.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import logging
+import os.path
+
+from impala import conf
+
+LOG = logging.getLogger(__name__)
+
+_IMPALA_FLAGS = None
+
+_WEBSERVER_CERTIFICATE_FILE = '-webserver_certificate_file'
+_SSL_SERVER_CERTIFICATE = '-ssl_server_certificate'
+_MAX_RESULT_CACHE_SIZE = '-max_result_cache_size'
+_AUTHORIZED_PROXY_USER_CONFIG = '-authorized_proxy_user_config'
+
+
+def reset():
+  global _IMPALA_FLAGS
+  _IMPALA_FLAGS = None
+
+
+def get_conf():
+  if _IMPALA_FLAGS is None:
+    _parse_impala_flags()
+  return _IMPALA_FLAGS
+
+
+def get_webserver_certificate_file():
+  return get_conf().get(_WEBSERVER_CERTIFICATE_FILE)
+
+def get_ssl_server_certificate():
+  return get_conf().get(_SSL_SERVER_CERTIFICATE)
+
+def get_max_result_cache_size():
+  result_size = get_conf().get(_MAX_RESULT_CACHE_SIZE)
+  return int(result_size) if result_size else None
+
+
+def get_authorized_proxy_user_config():
+  return get_conf().get(_AUTHORIZED_PROXY_USER_CONFIG)
+
+
+def _parse_impala_flags():
+  global _IMPALA_FLAGS
+
+  try:
+    impala_flags_path = os.path.join(conf.IMPALA_CONF_DIR.get(), 'impalad_flags')
+    _IMPALA_FLAGS = dict(line.strip().split('=', 1) for line in open(impala_flags_path) if '=' in line)
+  except IOError, err:
+    if err.errno != errno.ENOENT:
+      LOG.error('Cannot read from "%s": %s' % (impala_flags_path, err))
+    _IMPALA_FLAGS = {}
+  except Exception, ex:
+    LOG.error('Failed to parse Impala config from "%s": %s' % (impala_flags_path, ex))
+    _IMPALA_FLAGS = {}

+ 54 - 0
apps/impala/src/impala/test_impala_flags.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import os
+import tempfile
+
+from nose.tools import assert_equal
+
+from impala import conf, impala_flags
+
+
+LOG = logging.getLogger(__name__)
+
+
+def test_impala_flags():
+  test_impala_conf_dir = tempfile.mkdtemp()
+  finish = None
+
+  try:
+    flags = """
+      -webserver_certificate_file=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem
+      -ssl_server_certificate=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem
+      -max_result_cache_size=100000
+      -authorized_proxy_user_config=hue=*
+    """
+    file(os.path.join(test_impala_conf_dir, 'impalad_flags'), 'w').write(flags)
+
+    finish = conf.IMPALA_CONF_DIR.set_for_testing(test_impala_conf_dir)
+    impala_flags.reset()
+
+    assert_equal(impala_flags.get_webserver_certificate_file(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem')
+    assert_equal(impala_flags.get_ssl_server_certificate(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem')
+    assert_equal(impala_flags.get_max_result_cache_size(), 100000)
+    assert_equal(impala_flags.get_authorized_proxy_user_config(), 'hue=*')
+  finally:
+    impala_flags.reset()
+    if finish:
+      finish()

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -983,6 +983,9 @@
   # A comma-separated list of white-listed Impala configuration properties that users are authorized to set.
   # config_whitelist=debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s,request_pool
 
+  # Path to the impala configuration dir which has impalad_flags file
+  ## impala_conf_dir=${HUE_CONF_DIR}/impala-conf
+
   [[ssl]]
     # SSL communication enabled for this server.
     ## enabled=false

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -987,6 +987,9 @@
   # A comma-separated list of white-listed Impala configuration properties that users are authorized to set.
   ## config_whitelist=debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s,request_pool
 
+  # Path to the impala configuration dir which has impalad_flags file
+  ## impala_conf_dir=${HUE_CONF_DIR}/impala-conf
+
   [[ssl]]
     # SSL communication enabled for this server.
     ## enabled=false