Prechádzať zdrojové kódy

HUE-7675 [core] Warn on startup for any invalid configurations in hue.ini (Step#2)

Roohi 7 rokov pred
rodič
commit
f10f149ef3

+ 4 - 4
desktop/core/src/desktop/lib/conf.py

@@ -70,7 +70,7 @@ from django.utils.translation import ugettext as _
 
 from desktop.lib.paths import get_desktop_root, get_build_dir
 
-import configobj
+from configobj import ConfigObj, ConfigObjError
 import json
 import logging
 import os
@@ -508,8 +508,8 @@ def _configs_from_dir(conf_dir):
       continue
     LOG.debug("Loading configuration from: %s" % filename)
     try:
-      conf = configobj.ConfigObj(os.path.join(conf_dir, filename))
-    except configobj.ConfigObjError, ex:
+      conf = ConfigObj(os.path.join(conf_dir, filename))
+    except ConfigObjError, ex:
       LOG.error("Error in configuration file '%s': %s" % (os.path.join(conf_dir, filename), ex))
       raise
     conf['DEFAULT'] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir())
@@ -526,7 +526,7 @@ def load_confs(conf_source=None):
   if conf_source is None:
     conf_source = _configs_from_dir(get_desktop_root("conf"))
 
-  conf = configobj.ConfigObj()
+  conf = ConfigObj()
   for in_conf in conf_source:
     conf.merge(in_conf)
   return conf

+ 57 - 0
desktop/core/src/desktop/lib/config_spec_dump.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import desktop.appmanager
+
+from desktop.lib.conf import BoundContainer, is_anonymous
+
+class ConfigSpec():
+  def __init__(self, configspec):
+    self.indent = 0
+    self.level = 0
+    self.file = configspec
+
+  def generate(self, **options):
+    self.recurse(desktop.lib.conf.GLOBAL_CONFIG)
+    self.file.close()
+
+  def p(self, s):
+    self.file.write("\n" + " " * self.indent + s + "\n")
+
+  def recurse(self, config_obj):
+    if isinstance(config_obj, BoundContainer):
+      if is_anonymous(config_obj.config.key):
+        key = "__many__"
+        if 'notebook.interpreters.' in config_obj.prefix:
+          key = config_obj.prefix.split('notebook.interpreters.')[1]
+      else:
+        key = config_obj.config.key
+      if self.level != 0:
+        self.p("%s" % "[" * self.level + key + "]" * self.level)
+      self.indent += 2
+      self.level += 1
+      sections = []
+      for v in config_obj.get().values():
+        if isinstance(v, BoundContainer):
+          sections.append(v)
+        else:
+          self.p("%s=%s" % (v.config.key, v.get()))
+
+      for sec in sections:
+        self.recurse(sec)
+
+      self.indent -= 2
+      self.level -= 1

+ 53 - 1
desktop/core/src/desktop/tests.py

@@ -36,6 +36,8 @@ from django.core.urlresolvers import reverse
 from django.http import HttpResponse
 from django.db.models import query, CharField, SmallIntegerField
 
+from configobj import ConfigObj
+
 from settings import HUE_DESKTOP_VERSION
 
 from beeswax.conf import HIVE_SERVER_HOST
@@ -60,7 +62,7 @@ from desktop.models import Directory, Document, Document2, get_data_link, _versi
   ClusterConfig
 from desktop.redaction import logfilter
 from desktop.redaction.engine import RedactionPolicy, RedactionRule
-from desktop.views import check_config, home
+from desktop.views import check_config, home, generate_configspec, load_confs, collect_validation_messages
 from desktop.auth.backend import rewrite_user
 from dashboard.conf import HAS_SQL_ENABLED
 
@@ -1388,3 +1390,53 @@ def test_get_dn():
   assert_equal(['.hue.com'], desktop.conf.get_dn('sql.hue.com'))
   assert_equal(['.hue.com'], desktop.conf.get_dn('finance.sql.hue.com'))
   assert_equal(['.hue.com'], desktop.conf.get_dn('bank.finance.sql.hue.com'))
+
+def test_collect_validation_messages_default():
+  try:
+    # Generate the spec file
+    configspec = generate_configspec()
+    # Load the .ini files
+    conf = load_confs(configspec.name)
+    # This is for the hue.ini file only
+    error_list = []
+    collect_validation_messages(conf, error_list)
+    assert_equal(len(error_list), 0)
+  finally:
+    os.remove(configspec.name)
+
+def test_collect_validation_messages_extras():
+  try:
+    # Generate the spec file
+    configspec = generate_configspec()
+    # Load the .ini files
+    conf = load_confs(configspec.name)
+
+    test_conf = ConfigObj()
+    test_conf['extrasection'] = {
+      'key1': 'value1',
+      'key2': 'value1'
+    }
+    extrasubsection = {
+      'key1': 'value1',
+      'key2': 'value1'
+    }
+    # Test with extrasections as well as existing subsection, keyvalues in existing section [desktop]
+    test_conf['desktop'] = {
+      'extrasubsection': extrasubsection,
+      'extrakey': 'value1',
+      'auth': {
+        'ignore_username_case': 'true',
+        'extrasubsubsection': {
+          'extrakey': 'value1'
+        }
+      }
+    }
+    conf.merge(test_conf)
+    error_list = []
+    collect_validation_messages(conf, error_list)
+  finally:
+    os.remove(configspec.name)
+  assert_equal(len(error_list), 1)
+  assert_equal(u'Extra section, extrasection in the section: top level, Extra keyvalue, extrakey in the section: [desktop] , Extra section, extrasubsection in the section: [desktop] , Extra section, extrasubsubsection in the section: [desktop] [[auth]] ', error_list[0]['message'])
+
+

+ 76 - 1
desktop/core/src/desktop/views.py

@@ -26,6 +26,7 @@ import tempfile
 import time
 import traceback
 import zipfile
+import validate
 
 from django.conf import settings
 from django.shortcuts import render_to_response
@@ -35,6 +36,7 @@ from django.core.servers.basehttp import FileWrapper
 from django.shortcuts import redirect
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_POST
+from configobj import ConfigObj, get_extra_values, ConfigObjError
 
 import django.views.debug
 
@@ -49,7 +51,8 @@ from desktop.api import massaged_tags_for_json, massaged_documents_for_json, _ge
 
 from desktop.conf import USE_NEW_EDITOR, IS_HUE_4, HUE_LOAD_BALANCER, get_clusters, DISABLE_HUE_3
 from desktop.lib import django_mako
-from desktop.lib.conf import GLOBAL_CONFIG, BoundConfig
+from desktop.lib.conf import GLOBAL_CONFIG, BoundConfig, _configs_from_dir
+from desktop.lib.config_spec_dump import ConfigSpec
 from desktop.lib.django_util import JsonResponse, login_notrequired, render
 from desktop.lib.i18n import smart_str
 from desktop.lib.paths import get_desktop_root
@@ -59,6 +62,7 @@ from desktop.log import set_all_debug as _set_all_debug, reset_all_debug as _res
 from desktop.models import Settings, hue_version, _get_apps, UserPreferences, Cluster
 
 
+
 LOG = logging.getLogger(__name__)
 
 
@@ -597,9 +601,80 @@ def _get_config_errors(request, cache=True):
           error_list.append(error)
       except Exception, ex:
         LOG.exception("Error in config validation by %s: %s" % (module.nice_name, ex))
+
+    validate_by_spec(error_list)
+
     _CONFIG_ERROR_LIST = error_list
+
+  if _CONFIG_ERROR_LIST:
+    LOG.warn("Errors in config : %s" % _CONFIG_ERROR_LIST)
+
   return _CONFIG_ERROR_LIST
 
+def validate_by_spec(error_list):
+  try:
+    # Generate the spec file
+    configspec = generate_configspec()
+    # Load the .ini files
+    conf = load_confs(configspec.name)
+    # Validate after merging all the confs
+    collect_validation_messages(conf, error_list)
+  finally:
+    os.remove(configspec.name)
+
+
+def load_confs(configspecpath):
+  conf_source = _configs_from_dir(get_desktop_root("conf"))
+  conf = ConfigObj(configspec=configspecpath)
+  for in_conf in conf_source:
+    conf.merge(in_conf)
+  return conf
+
+
+def generate_configspec():
+  configspec = tempfile.NamedTemporaryFile(delete=False)
+  cs = ConfigSpec(configspec)
+  cs.generate()
+  return configspec
+
+
+def collect_validation_messages(conf, error_list):
+  validator = validate.Validator()
+  conf.validate(validator, preserve_errors=True)
+  message = []
+  for sections, name in get_extra_values(conf):
+    the_section = conf
+    hierarchy_sections_string = ''
+    try:
+      parent = conf
+      for section in sections:
+        the_section = parent[section]
+        hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " "
+        parent = the_section
+    except KeyError, ex:
+      LOG.warn("Section %s not found: %s" % (section, str(ex)))
+
+    the_value = ''
+    try:
+      # the_value may be a section or a value
+      the_value = the_section[name]
+    except KeyError, ex:
+      LOG.warn("Error in accessing Section or Value %s: %s" % (name, str(ex)))
+
+    section_or_value = 'keyvalue'
+    if isinstance(the_value, dict):
+      # Sections are subclasses of dict
+      section_or_value = 'section'
+
+    section_string = hierarchy_sections_string or "top level"
+    message.append('Extra %s, %s in the section: %s' % (section_or_value, name, section_string))
+  if message:
+    error = {
+      'name': 'Desktop',
+      'message': ', '.join(message),
+    }
+    error_list.append(error)
+
 
 def check_config(request):
   """Check config and view for the list of errors"""