Przeglądaj źródła

HUE-4360 [meta] Read configuration from a properties file

Set the Nav API URL automatically.
Romain Rigaux 9 lat temu
rodzic
commit
a22d9ab

+ 10 - 0
desktop/core/src/desktop/lib/paths.py

@@ -85,3 +85,13 @@ def get_run_root(*append):
   Returns the run time root directory
   """
   return __get_root(*append)
+
+
+def get_config_root(*append):
+  """
+  Currently gets it based on the Hadoop configuration location.
+  """
+  from hadoop.cluster import get_default_fscluster_config
+
+  yarn_site_path = get_default_fscluster_config().HADOOP_CONF_DIR.get()
+  return os.path.abspath(os.path.join(yarn_site_path, '..', *append))

+ 9 - 1
desktop/libs/hadoop/src/hadoop/cluster.py

@@ -32,6 +32,7 @@ LOG = logging.getLogger(__name__)
 
 
 FS_CACHE = None
+FS_DEFAULT_NAME = 'default'
 MR_CACHE = None
 MR_NAME_CACHE = 'default'
 DEFAULT_USER = DEFAULT_USER.get()
@@ -136,6 +137,13 @@ def get_default_yarncluster():
     return get_yarn()
 
 
+def get_default_fscluster_config():
+  """
+  Get the default FS config.
+  """
+  return conf.HDFS_CLUSTERS[FS_DEFAULT_NAME]
+
+
 def get_next_ha_mrcluster():
   """
   Return the next available JT instance and cache its name.
@@ -319,4 +327,4 @@ def _make_filesystem(identifier):
 
 def _make_mrcluster(identifier):
   cluster_conf = conf.MR_CLUSTERS[identifier]
-  return LiveJobTracker.from_conf(cluster_conf)
+  return LiveJobTracker.from_conf(cluster_conf)

+ 3 - 0
desktop/libs/hadoop/src/hadoop/conf.py

@@ -20,9 +20,11 @@ import logging
 import os
 
 from django.utils.translation import ugettext_lazy as _t
+
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_bool
 from desktop.conf import default_ssl_validate
 
+
 LOG = logging.getLogger(__name__)
 DEFAULT_NN_HTTP_PORT = 50070
 
@@ -44,6 +46,7 @@ def find_file_recursive(desired_glob, root):
   return f
 
 
+
 UPLOAD_CHUNK_SIZE = Config(
   key="upload_chunk_size",
   help="Size, in bytes, of the 'chunks' Django should store into memory and feed into the handler. Default is 64MB.",

+ 21 - 3
desktop/libs/metadata/src/metadata/conf.py

@@ -20,6 +20,10 @@ from django.utils.translation import ugettext_lazy as _t
 from desktop.conf import AUTH_USERNAME as DEFAULT_AUTH_USERNAME, AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD, \
                          AUTH_PASSWORD_SCRIPT, default_ssl_validate
 from desktop.lib.conf import Config, ConfigSection, coerce_bool, coerce_password_from_script
+from desktop.lib.paths import get_config_root
+from hadoop.cluster import get_default_yarncluster
+
+
 
 
 def get_auth_username():
@@ -34,6 +38,16 @@ def get_auth_password():
     return password
   return DEFAULT_AUTH_PASSWORD.get()
 
+def default_navigator_config_dir():
+  """Get from usual main Hue config directory"""
+  return get_config_root()
+
+
+def default_navigator_url():
+  """Get from usual main Hue config directory"""
+  from metadata.metadata_sites import get_navigator_server_url
+  return get_navigator_server_url() + '/api'
+
 
 def get_optimizer_url():
   return OPTIMIZER.API_URL.get() and OPTIMIZER.API_URL.get().strip('/')
@@ -125,12 +139,11 @@ NAVIGATOR = ConfigSection(
   members=dict(
     API_URL=Config(
       key='api_url',
-      help=_t('Base URL to Navigator API (e.g. - http://localhost:7187/api)'),
-      default=None),
+      help=_t('Base URL to Navigator API.'),
+      dynamic_default=default_navigator_url),
     AUTH_USERNAME=Config(
       key="auth_username",
       help=_t("Auth username of the hue user used for authentications."),
-      private=True,
       dynamic_default=get_auth_username),
     AUTH_PASSWORD=Config(
       key="auth_password",
@@ -143,5 +156,10 @@ NAVIGATOR = ConfigSection(
       private=True,
       type=coerce_password_from_script,
       default=None),
+    CONF_DIR = Config(
+      key='conf_dir',
+      help=_t('Navigator configuration directory, where navigator.client.properties is located.'),
+      dynamic_default=default_navigator_config_dir
+    )
   )
 )

+ 87 - 0
desktop/libs/metadata/src/metadata/metadata_sites.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import logging
+import os
+
+from metadata.conf import NAVIGATOR
+
+
+LOG = logging.getLogger(__name__)
+
+
+_SITE_DICT = None
+
+_CONF_NAVIGATOR_SERVER_URL = 'navigator.server.url'
+_CONF_NAVIGATOR_AUDIT_LOG_DIR = 'audit_event_log_dir'
+_CONF_NAVIGATOR_AUDIT_MAX_FILE_SIZE = 'navigator.audit_log_max_file_size'
+
+
+
+def reset():
+  global _SITE_DICT
+  _SITE_DICT = None
+
+
+def get_conf(name='navigator'):
+  if _SITE_DICT is None:
+    _parse_sites()
+  return _SITE_DICT[name]
+
+
+
+def get_navigator_server_url():
+  return get_conf().get(_CONF_NAVIGATOR_SERVER_URL, 'http://localhost:7187/api')
+
+def get_navigator_audit_log_dir():
+  return get_conf().get(_CONF_NAVIGATOR_AUDIT_LOG_DIR)
+
+def get_navigator_audit_max_file_size():
+  return get_conf().get(_CONF_NAVIGATOR_AUDIT_MAX_FILE_SIZE, '100')
+
+
+def _parse_sites():
+  global _SITE_DICT
+  _SITE_DICT ={}
+
+  paths = [
+    ('navigator', os.path.join(NAVIGATOR.CONF_DIR.get(), 'navigator.client.properties')),
+  ]
+
+  for name, path in paths:
+    _SITE_DICT[name] = _parse_property(path)
+
+
+def _parse_property(file_path):
+  try:
+    return dict(line.strip().rsplit('=', 1) for line in open(file_path) if '=' in line)
+  except IOError, err:
+    if err.errno != errno.ENOENT:
+      LOG.error('Cannot read from "%s": %s' % (file_path, err))
+    return ""
+
+def _parse_site(site_path):
+  try:
+    data = file(site_path, 'r').read()
+  except IOError, err:
+    if err.errno != errno.ENOENT:
+      LOG.error('Cannot read from "%s": %s' % (site_path, err))
+      return
+    data = ""
+
+  return confparse.ConfParse(data)

+ 57 - 0
desktop/libs/metadata/src/metadata/metadata_sites_tests.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import shutil
+import tempfile
+
+from nose.tools import assert_equal
+
+import metadata_sites
+from metadata.conf import NAVIGATOR
+from metadata.metadata_sites import get_navigator_server_url
+
+LOG = logging.getLogger(__name__)
+
+
+class TestReadConfiguration:
+
+  def test_navigator_site(self):
+    tmpdir = tempfile.mkdtemp()
+    resets = [
+        NAVIGATOR.CONF_DIR.set_for_testing(tmpdir)
+    ]
+
+    try:
+      file(os.path.join(tmpdir, 'navigator.client.properties'), 'w').write("""
+navigator.client.serviceType=HUE
+navigator.server.url=http://hue-rocks.com:7186
+navigator.client.roleName=HUE-1-HUE_SERVER-50cf99601c4bf64e9ccded4c8cd96d12
+navigator.client.roleType=HUE_SERVER
+audit_event_log_dir=/var/log/hue/audit
+navigator.audit_log_max_file_size=100
+      """)
+
+      metadata_sites.reset()
+
+      assert_equal(get_navigator_server_url(), 'http://hue-rocks.com:7186')
+    finally:
+      metadata_sites.reset()
+      for reset in resets:
+        reset()
+      shutil.rmtree(tmpdir)

+ 3 - 2
desktop/libs/metadata/src/metadata/navigator_api.py

@@ -25,7 +25,8 @@ from django.views.decorators.http import require_POST
 from desktop.lib.django_util import JsonResponse
 from desktop.lib.i18n import force_unicode
 
-from metadata.navigator_client import NavigatorApi, is_navigator_enabled
+from metadata.conf import has_navigator
+from metadata.navigator_client import NavigatorApi
 
 LOG = logging.getLogger(__name__)
 
@@ -37,7 +38,7 @@ class MetadataApiException(Exception):
 def error_handler(view_fn):
   def decorator(*args, **kwargs):
     try:
-      if is_navigator_enabled():
+      if has_navigator():
         return view_fn(*args, **kwargs)
       else:
         raise MetadataApiException('Navigator API is not configured.')

+ 0 - 4
desktop/libs/metadata/src/metadata/navigator_client.py

@@ -32,10 +32,6 @@ LOG = logging.getLogger(__name__)
 VERSION = 'v3'
 
 
-def is_navigator_enabled():
-  return NAVIGATOR.API_URL.get()
-
-
 def get_filesystem_host():
   host = None
   hadoop_fs = HDFS_CLUSTERS['default'].FS_DEFAULTFS.get()