Эх сурвалжийг харах

HUE-116: Auto detect config errors and alert admin

- Record prefix of config variables
- Added comments to deskop/lib/conf.py
- Added errno to some errors raised by hadoopfs
- Modified `dump_config' view to show conf file location
- Added new config_validator() feature in sdk docs
bc Wong 15 жил өмнө
parent
commit
c51bf8c5a4

+ 1 - 0
apps/about/src/about/templates/index.mako

@@ -29,6 +29,7 @@
 		<p>${version}</p>
 		<p>${version}</p>
                 <p>&nbsp;</p>
                 <p>&nbsp;</p>
                 <p><a target="_blank" href="${url("desktop.views.dump_config")}">Configuration</a></p>
                 <p><a target="_blank" href="${url("desktop.views.dump_config")}">Configuration</a></p>
+                <p><a target="_blank" href="${url("desktop.views.check_config")}">Check for Misconfiguration</a></p>
                 <p><a target="_blank" href="${url("desktop.views.log_view")}">Server Logs</a></p>
                 <p><a target="_blank" href="${url("desktop.views.log_view")}">Server Logs</a></p>
 	</center>
 	</center>
 </body>
 </body>

+ 2 - 1
apps/filebrowser/src/filebrowser/views.py

@@ -20,6 +20,7 @@
 # Useful resources:
 # Useful resources:
 #   django/views/static.py manages django's internal directory index
 #   django/views/static.py manages django's internal directory index
 
 
+import errno
 import logging
 import logging
 import mimetypes
 import mimetypes
 import posixpath
 import posixpath
@@ -124,7 +125,7 @@ def edit(request, path, form=None):
     stats = request.fs.stats(path)
     stats = request.fs.stats(path)
   except IOError, ioe:
   except IOError, ioe:
     # A file not found is OK, otherwise re-raise
     # A file not found is OK, otherwise re-raise
-    if "not found" in ioe.args[0]:
+    if ioe.errno == errno.ENOENT:
       stats = None
       stats = None
     else:
     else:
       raise
       raise

+ 5 - 0
desktop/core/src/desktop/auth/views_test.py

@@ -17,10 +17,15 @@
 
 
 from nose.tools import assert_true, assert_false, assert_equal
 from nose.tools import assert_true, assert_false, assert_equal
 
 
+from django.contrib.auth.models import User
 from django.test.client import Client
 from django.test.client import Client
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.django_test_util import make_logged_in_client
 
 
 def test_jframe_login():
 def test_jframe_login():
+  # Simulate first login ever
+  for user in User.objects.all():
+    user.delete()
+
   c = Client()
   c = Client()
 
 
   response = c.get('/accounts/login_form')
   response = c.get('/accounts/login_form')

+ 25 - 1
desktop/core/src/desktop/conf.py

@@ -16,8 +16,10 @@
 # limitations under the License.
 # limitations under the License.
 """General configuration for core Desktop features (authentication, etc)"""
 """General configuration for core Desktop features (authentication, etc)"""
 
 
-from desktop.lib.conf import Config, ConfigSection, UnspecifiedConfigSection, coerce_bool
+from desktop.lib.conf import Config, ConfigSection, UnspecifiedConfigSection
+from desktop.lib.conf import coerce_bool, validate_path
 from desktop.lib.paths import get_desktop_root
 from desktop.lib.paths import get_desktop_root
+from desktop.lib import i18n
 import os
 import os
 import socket
 import socket
 
 
@@ -261,3 +263,25 @@ HTTP_500_DEBUG_MODE = Config(
   type=coerce_bool,
   type=coerce_bool,
   default=True
   default=True
 )
 )
+
+
+def config_validator():
+  """
+  config_validator() -> [ (config_variable, error_message) ]
+
+  Called by core check_config() view.
+  """
+  res = [ ]
+  if not SECRET_KEY.get():
+    res.append((SECRET_KEY, "Secret key should be configured as a random string."))
+
+  if SSL_CERTIFICATE.get():
+    res.extend(validate_path(SSL_CERTIFICATE, is_dir=False))
+    if not SSL_PRIVATE_KEY.get():
+      res.append((SSL_PRIVATE_KEY, "SSL private key file should be set to enable HTTPS."))
+    else:
+      res.extend(validate_path(SSL_PRIVATE_KEY, is_dir=False))
+  if not i18n.validate_encoding(DEFAULT_SITE_ENCODING.get()):
+    res.append((DEFAULT_SITE_ENCODING, "Encoding not supported."))
+
+  return res

+ 98 - 33
desktop/core/src/desktop/lib/conf.py

@@ -70,7 +70,7 @@ variables.
 """
 """
 
 
 # Magical object for use as a "symbol"
 # Magical object for use as a "symbol"
-_ANONYMOUS=["_ANONYMOUS"]
+_ANONYMOUS = ("_ANONYMOUS")
 
 
 # a BoundContainer(BoundConfig) object which has all of the application's configs as members
 # a BoundContainer(BoundConfig) object which has all of the application's configs as members
 GLOBAL_CONFIG = None
 GLOBAL_CONFIG = None
@@ -78,18 +78,34 @@ GLOBAL_CONFIG = None
 __all__ = ["UnspecifiedConfigSection", "ConfigSection", "Config", "load_confs", "coerce_bool"]
 __all__ = ["UnspecifiedConfigSection", "ConfigSection", "Config", "load_confs", "coerce_bool"]
 
 
 class BoundConfig(object):
 class BoundConfig(object):
-  def __init__(self, config, bind_to, grab_key=_ANONYMOUS):
+  def __init__(self, config, bind_to, grab_key=_ANONYMOUS, prefix=''):
     """
     """
     A Config object that has been bound to specific data.
     A Config object that has been bound to specific data.
 
 
     @param config   The config that is bound - must handle get_value
     @param config   The config that is bound - must handle get_value
     @param bind_to  The data it is bound to - must support subscripting
     @param bind_to  The data it is bound to - must support subscripting
     @param grab_key The key in bind_to in which to expect this configuration
     @param grab_key The key in bind_to in which to expect this configuration
+    @param prefix   The prefix in the config tree leading to this configuration
     """
     """
     self.config = config
     self.config = config
     self.bind_to = bind_to
     self.bind_to = bind_to
     self.grab_key = grab_key
     self.grab_key = grab_key
 
 
+    # The prefix of a config is the path leading to the config, including section and subsections
+    # along the way, e.g. hadoop.filesystems.cluster_1.
+    #
+    # The prefix is recorded in BoundConfig only, because section names can change dynamically for
+    # UnspecifiedConfigSection's, which have children with _ANONYMOUS keys. If our config is
+    # _ANONYMOUS, this `prefix' includes the prefix plus the actual key name.
+    self.prefix = prefix
+
+  def get_fully_qualifying_key(self):
+    """Returns the full key name, in the form of section[.subsection[...]].key"""
+    res = self.prefix
+    if self.config.key is not _ANONYMOUS:
+      res += self.prefix and '.' + self.config.key or self.config.key
+    return res
+
   def _get_data_and_presence(self):
   def _get_data_and_presence(self):
     """
     """
     Returns a tuple (data, present).
     Returns a tuple (data, present).
@@ -108,7 +124,7 @@ class BoundConfig(object):
   def get(self):
   def get(self):
     """Get the data, or its default value."""
     """Get the data, or its default value."""
     data, present = self._get_data_and_presence()
     data, present = self._get_data_and_presence()
-    return self.config.get_value(data, present=present)
+    return self.config.get_value(data, present=present, prefix=self.prefix)
 
 
   def set_for_testing(self, data=None, present=True):
   def set_for_testing(self, data=None, present=True):
     """
     """
@@ -178,7 +194,7 @@ class Config(object):
     assert not (self.required and self.default), \
     assert not (self.required and self.default), \
            "Config cannot be required if it has a default."
            "Config cannot be required if it has a default."
 
 
-  def bind(self, conf):
+  def bind(self, conf, prefix):
     """Rather than doing the lookup now and assigning self.value or something,
     """Rather than doing the lookup now and assigning self.value or something,
     this binding creates a new object. This is because, for a given Config
     this binding creates a new object. This is because, for a given Config
     object, it might need to be bound to different parts of a configuration
     object, it might need to be bound to different parts of a configuration
@@ -188,9 +204,9 @@ class Config(object):
     it will be end up applying to each subsection. We therefore bind it
     it will be end up applying to each subsection. We therefore bind it
     multiple times, once to each subsection.
     multiple times, once to each subsection.
     """
     """
-    return BoundConfig(config=self, bind_to=conf, grab_key=self.key)
+    return BoundConfig(config=self, bind_to=conf, grab_key=self.key, prefix=prefix)
 
 
-  def get_value(self, val, present):
+  def get_value(self, val, present, prefix=None):
     """
     """
     Return the value for this configuration variable from the
     Return the value for this configuration variable from the
     currently loaded configuration.
     currently loaded configuration.
@@ -205,7 +221,7 @@ class Config(object):
       raw_val = val
       raw_val = val
     else:
     else:
       raw_val = self.default
       raw_val = self.default
-    return self._coerce_type(raw_val)
+    return self._coerce_type(raw_val, prefix)
 
 
   def validate(self, source):
   def validate(self, source):
     """
     """
@@ -213,12 +229,13 @@ class Config(object):
     or of the incorrect type.
     or of the incorrect type.
     """
     """
     # Getting the value will raise an exception if it's in bad form.
     # Getting the value will raise an exception if it's in bad form.
-    _ = self.get_value()
+    val = source.get(self.key, None)
+    present = val is not None
+    _ = self.get_value(val, present)
 
 
-  def _coerce_type(self, raw):
+  def _coerce_type(self, raw, _):
     """
     """
-    Coerces the value in 'raw' to the correct type, based on
-    self.type
+    Coerces the value in 'raw' to the correct type, based on self.type
     """
     """
     if raw is None:
     if raw is None:
       return raw
       return raw
@@ -300,7 +317,7 @@ class BoundContainerWithGetAttr(BoundContainer):
   This is used by ConfigSection
   This is used by ConfigSection
   """
   """
   def __getattr__(self, attr):
   def __getattr__(self, attr):
-    return self.config.get_member(self.get_data_dict(), attr)
+    return self.config.get_member(self.get_data_dict(), attr, self.prefix)
 
 
 class BoundContainerWithGetItem(BoundContainer):
 class BoundContainerWithGetItem(BoundContainer):
   """
   """
@@ -312,7 +329,7 @@ class BoundContainerWithGetItem(BoundContainer):
   def __getitem__(self, attr):
   def __getitem__(self, attr):
     if attr in self.__dict__:
     if attr in self.__dict__:
       return self.__dict__[attr]
       return self.__dict__[attr]
-    return self.config.get_member(self.get_data_dict(), attr)
+    return self.config.get_member(self.get_data_dict(), attr, self.prefix)
 
 
 
 
 class ConfigSection(Config):
 class ConfigSection(Config):
@@ -353,21 +370,23 @@ class ConfigSection(Config):
     self.members.update(new_members)
     self.members.update(new_members)
 
 
 
 
-  def bind(self, config):
-    return BoundContainerWithGetAttr(self, bind_to=config, grab_key=self.key)
+  def bind(self, config, prefix):
+    return BoundContainerWithGetAttr(self, bind_to=config, grab_key=self.key, prefix=prefix)
 
 
-  def _coerce_type(self, raw):
+  def _coerce_type(self, raw, prefix=''):
     """
     """
     Materialize this section as a dictionary.
     Materialize this section as a dictionary.
 
 
     The keys are those specified in the members dict, and the values
     The keys are those specified in the members dict, and the values
     are bound configuration parameters.
     are bound configuration parameters.
     """
     """
-    return dict([(key, self.get_member(raw, key))
+    return dict([(key, self.get_member(raw, key, prefix))
                  for key in self.members.iterkeys()])
                  for key in self.members.iterkeys()])
 
 
-  def get_member(self, data, attr):
-    return self.members[attr].bind(data)
+  def get_member(self, data, attr, prefix):
+    if self.key is not _ANONYMOUS:
+      prefix += prefix and '.' + self.key or self.key
+    return self.members[attr].bind(data, prefix)
 
 
   def print_help(self, out=sys.stdout, indent=0, skip_header=False):
   def print_help(self, out=sys.stdout, indent=0, skip_header=False):
     if self.private:
     if self.private:
@@ -387,31 +406,41 @@ class ConfigSection(Config):
 
 
 class UnspecifiedConfigSection(Config):
 class UnspecifiedConfigSection(Config):
   """
   """
-  A section of configuration variables whose subsections are unknown
-  but of some given type.
+  A special Config that maps a section name to a list of anonymous subsections.
+  The subsections are anonymous in the sense that their names are unknown beforehand,
+  but all have the same structure.
 
 
   For example, this can be used for a [clusters] section which
   For example, this can be used for a [clusters] section which
   expects some number of [[cluster]] sections underneath it.
   expects some number of [[cluster]] sections underneath it.
+
+  This class is NOT a ConfigSection, although it supports get_member(). The key
+  difference is that its get_member() returns a BoundConfig with:
+  (1) an anonymous ConfigSection,
+  (2) an anonymous grab_key, and
+  (3) a `prefix' containing the prefix plus the actual key name.
   """
   """
   def __init__(self, key=_ANONYMOUS, each=None, **kwargs):
   def __init__(self, key=_ANONYMOUS, each=None, **kwargs):
     super(UnspecifiedConfigSection, self).__init__(key, default={}, **kwargs)
     super(UnspecifiedConfigSection, self).__init__(key, default={}, **kwargs)
     assert each.key is _ANONYMOUS
     assert each.key is _ANONYMOUS
-    self.each = each
+    self.each = each    # `each' is a ConfigSection
 
 
-  def bind(self, config):
-    return BoundContainerWithGetItem(self, bind_to=config, grab_key=self.key)
+  def bind(self, config, prefix):
+    return BoundContainerWithGetItem(self, bind_to=config, grab_key=self.key, prefix=prefix)
 
 
-  def _coerce_type(self, raw):
+  def _coerce_type(self, raw, prefix=''):
     """
     """
     Materialize this section as a dictionary.
     Materialize this section as a dictionary.
 
 
     The keys are the keys specified by the user in the config file.
     The keys are the keys specified by the user in the config file.
     """
     """
-    return dict([(key, self.get_member(raw, key))
+    return dict([(key, self.get_member(raw, key, prefix))
                  for key in raw.iterkeys()])
                  for key in raw.iterkeys()])
 
 
-  def get_member(self, data, attr):
-    return self.each.bind(data[attr])
+  def get_member(self, data, attr, prefix=''):
+    tail = self.key + '.' + attr
+    child_prefix = prefix
+    child_prefix += prefix and '.' + tail or tail
+    return self.each.bind(data[attr], child_prefix)
 
 
   def print_help(self, out=sys.stdout, indent=0):
   def print_help(self, out=sys.stdout, indent=0):
     indent_str = " " * indent
     indent_str = " " * indent
@@ -452,7 +481,7 @@ def load_confs(conf_source=None):
     conf.merge(in_conf)
     conf.merge(in_conf)
   return conf
   return conf
 
 
-def _bind_module_members(module, data):
+def _bind_module_members(module, data, section):
   """
   """
   Bind all Config instances found inside the given module
   Bind all Config instances found inside the given module
   to the given data.
   to the given data.
@@ -465,7 +494,7 @@ def _bind_module_members(module, data):
       continue
       continue
 
 
     members[key] = val
     members[key] = val
-    module.__dict__[key] = val.bind(data)
+    module.__dict__[key] = val.bind(data, prefix=section)
   return members
   return members
 
 
 
 
@@ -496,7 +525,7 @@ def bind_module_config(mod, conf_data):
     section = mod.__name__
     section = mod.__name__
 
 
   bind_data = conf_data.get(section, {})
   bind_data = conf_data.get(section, {})
-  members = _bind_module_members(mod, bind_data)
+  members = _bind_module_members(mod, bind_data, section)
   return ConfigSection(section,
   return ConfigSection(section,
                        members=members,
                        members=members,
                        help=mod.__doc__)
                        help=mod.__doc__)
@@ -518,12 +547,12 @@ def initialize(modules):
 
 
   GLOBAL_HELP = "(root of all configuration)"
   GLOBAL_HELP = "(root of all configuration)"
   if GLOBAL_CONFIG is None:
   if GLOBAL_CONFIG is None:
-    GLOBAL_CONFIG = ConfigSection(members=sections, help=GLOBAL_HELP).bind(conf_data)
+    GLOBAL_CONFIG = ConfigSection(members=sections, help=GLOBAL_HELP).bind(conf_data, prefix='')
   else:
   else:
     new_config = ConfigSection(members=sections, help=GLOBAL_HELP)
     new_config = ConfigSection(members=sections, help=GLOBAL_HELP)
     new_config.update_members(GLOBAL_CONFIG.config.members, overwrite=False)
     new_config.update_members(GLOBAL_CONFIG.config.members, overwrite=False)
     conf_data.merge(GLOBAL_CONFIG.bind_to)
     conf_data.merge(GLOBAL_CONFIG.bind_to)
-    GLOBAL_CONFIG = new_config.bind(conf_data)
+    GLOBAL_CONFIG = new_config.bind(conf_data, prefix='')
   return
   return
 
 
 def is_anonymous(key):
 def is_anonymous(key):
@@ -537,3 +566,39 @@ def coerce_bool(value):
   if value in ("true", "True", "1", "yes", "on"):
   if value in ("true", "True", "1", "yes", "on"):
     return True
     return True
   raise Exception("Could not coerce boolean value: " + str(value))
   raise Exception("Could not coerce boolean value: " + str(value))
+
+
+def validate_path(confvar, is_dir=None):
+  """
+  Validate that the value of confvar is an existent local path.
+
+  @param confvar  The configuration variable.
+  @param is_dir  True/False would verify that the path is/isn't a directory.
+                 None to disable check.
+  @return [(confvar, error_msg)] or []
+  """
+  path = confvar.get()
+  if not os.path.exists(path):
+    return [(confvar, 'Path does not exist on local filesystem.')]
+  if is_dir is not None:
+    if is_dir:
+      if not os.path.isdir(path):
+        return [(confvar, 'Not a directory.')]
+    elif not os.path.isfile(path):
+      return [(confvar, 'Not a file.')]
+  return [ ]
+
+def validate_port(confvar):
+  """
+  Validate that the value of confvar is between [0, 65535].
+  Returns [(confvar, error_msg)] or []
+  """
+  port_val = confvar.get()
+  error_res = [(confvar, 'Port should be an integer between 0 and 65535 (inclusive).')]
+  try:
+    port = int(port_val)
+    if port < 0 or port > 65535:
+      return error_res
+  except ValueError:
+    return error_res
+  return [ ]

+ 10 - 1
desktop/core/src/desktop/lib/conf_test.py

@@ -83,7 +83,8 @@ class ConfigTest(unittest.TestCase):
                                        type=int, default=9090))))))
                                        type=int, default=9090))))))
     self.conf = self.conf.bind(
     self.conf = self.conf.bind(
       load_confs([configobj.ConfigObj(infile=StringIO(self.CONF_ONE)),
       load_confs([configobj.ConfigObj(infile=StringIO(self.CONF_ONE)),
-                  configobj.ConfigObj(infile=StringIO(self.CONF_TWO))]))
+                  configobj.ConfigObj(infile=StringIO(self.CONF_TWO))]),
+      prefix='')
 
 
   def testDynamicDefault(self):
   def testDynamicDefault(self):
     self.assertEquals(7, self.conf.DYNAMIC_DEF.get())
     self.assertEquals(7, self.conf.DYNAMIC_DEF.get())
@@ -106,6 +107,14 @@ class ConfigTest(unittest.TestCase):
     self.assertEquals("localhost", self.conf.CLUSTERS['clustera'].HOST.get())
     self.assertEquals("localhost", self.conf.CLUSTERS['clustera'].HOST.get())
     self.assertEquals(9090, self.conf.CLUSTERS['clustera'].PORT.get())
     self.assertEquals(9090, self.conf.CLUSTERS['clustera'].PORT.get())
 
 
+  def testFullKeyName(self):
+    self.assertEquals(self.conf.REQ.get_fully_qualifying_key(), 'req')
+    self.assertEquals(self.conf.CLUSTERS.get_fully_qualifying_key(), 'clusters')
+    self.assertEquals(self.conf.CLUSTERS['clustera'].get_fully_qualifying_key(),
+                      'clusters.clustera')
+    self.assertEquals(self.conf.CLUSTERS['clustera'].HOST.get_fully_qualifying_key(),
+                      'clusters.clustera.host')
+
   def testSetForTesting(self):
   def testSetForTesting(self):
     # Test base case
     # Test base case
     self.assertEquals(123, self.conf.FOO.get())
     self.assertEquals(123, self.conf.FOO.get())

+ 54 - 0
desktop/core/src/desktop/templates/check_config.mako

@@ -0,0 +1,54 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+<%!
+from desktop.lib.conf import BoundContainer
+%>
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+        <title>Hue Configuration Check</title>
+</head>
+
+<body>
+  <dl>
+    <dt><a target="_blank" href="${url("desktop.views.dump_config")}">View configuration</a></dt>
+    <dd>Configuration files located in <code>${conf_dir}</code></dd>
+  </dl>
+  <dl>
+    % if error_list:
+      <h3>Potential misconfiguration detected. Please fix and restart HUE.</h3>
+      <dl>
+      % for confvar, error in error_list:
+        <dt><code>${confvar.get_fully_qualifying_key()}</code></dt>
+        <dd>
+          ## Doesn't make sense to print the value of a BoundContainer
+          % if not isinstance(confvar, BoundContainer):
+            Current value: <code>${confvar.get()}</code><br/>
+          % endif
+          ${error | n}
+        </dd>
+      % endfor
+      </dl>
+    % else:
+      All ok. Configuration check passed!
+    % endif
+  </dl>
+</body>

+ 22 - 0
desktop/core/src/desktop/templates/config_alert_dock.mako

@@ -0,0 +1,22 @@
+## Licensed to Cloudera, Inc. under one
+## or more contributor license agreements.  See the NOTICE file
+## distributed with this work for additional information
+## regarding copyright ownership.  Cloudera, Inc. licenses this file
+## to you under the Apache License, Version 2.0 (the
+## "License"); you may not use this file except in compliance
+## with the License.  You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+% if error_list:
+  <a target="_blank" href="${url('desktop.views.check_config')}"
+      title="Misconfiguration detected" alt="Misconfiguration detected">
+    <img src="/static/art/error.png"/>
+  </a>
+% endif

+ 1 - 0
desktop/core/src/desktop/templates/dump_config.mako

@@ -23,6 +23,7 @@ from desktop.lib.conf import BoundContainer, is_anonymous
 <html><head><title>Hue Configuration</title></head>
 <html><head><title>Hue Configuration</title></head>
 <body>
 <body>
 <h1>Hue Configuration</h1>
 <h1>Hue Configuration</h1>
+<ul><li>Configuration files located in <code>${conf_dir}</code></li></ul>
 <h2>Installed applications</h2>
 <h2>Installed applications</h2>
 <ul>
 <ul>
 % for app in sorted(apps, key=lambda app: app.name.lower()):
 % for app in sorted(apps, key=lambda app: app.name.lower()):

+ 26 - 0
desktop/core/src/desktop/tests.py

@@ -275,3 +275,29 @@ def test_log_event():
     handler.records[-1].message)
     handler.records[-1].message)
 
 
   root.removeHandler(handler)
   root.removeHandler(handler)
+
+
+def test_config_check():
+  reset = (
+    desktop.conf.SECRET_KEY.set_for_testing(''),
+    desktop.conf.SSL_CERTIFICATE.set_for_testing('foobar'),
+    desktop.conf.SSL_PRIVATE_KEY.set_for_testing(''),
+    desktop.conf.DEFAULT_SITE_ENCODING.set_for_testing('klingon')
+  )
+
+  try:
+    cli = make_logged_in_client()
+    resp = cli.get('/debug/check_config')
+    assert_true('Secret key should be configured' in resp.content)
+    assert_true('desktop.ssl_certificate' in resp.content)
+    assert_true('Path does not exist' in resp.content)
+    assert_true('SSL private key file should be set' in resp.content)
+    assert_true('klingon' in resp.content)
+    assert_true('Encoding not supported' in resp.content)
+
+    # Alert present in the status bar
+    resp = cli.get('/status_bar/')
+    assert_true('Misconfiguration' in resp.content)
+  finally:
+    for old_conf in reset:
+      old_conf()

+ 4 - 3
desktop/core/src/desktop/urls.py

@@ -19,7 +19,7 @@ import logging
 import os
 import os
 import re
 import re
 
 
-from django.conf.urls.defaults import include, patterns, url
+from django.conf.urls.defaults import include, patterns
 from django.contrib import admin
 from django.contrib import admin
 
 
 from desktop import appmanager
 from desktop import appmanager
@@ -61,6 +61,7 @@ dynamic_patterns = patterns('',
   (r'^depender/', include(depender.urls)),
   (r'^depender/', include(depender.urls)),
   (r'^debug/threads$', 'desktop.views.threads'),
   (r'^debug/threads$', 'desktop.views.threads'),
   (r'^debug/who_am_i$', 'desktop.views.who_am_i'),
   (r'^debug/who_am_i$', 'desktop.views.who_am_i'),
+  (r'^debug/check_config$', 'desktop.views.check_config'),
   (r'^log_frontend_event$', 'desktop.views.log_frontend_event'),
   (r'^log_frontend_event$', 'desktop.views.log_frontend_event'),
   # Top level web page!
   # Top level web page!
   (r'^$', 'desktop.views.index'),
   (r'^$', 'desktop.views.index'),
@@ -89,6 +90,6 @@ static_patterns.append(static_pattern("static", buildpath("core/static")))
 urlpatterns = patterns('', *static_patterns) + dynamic_patterns
 urlpatterns = patterns('', *static_patterns) + dynamic_patterns
 
 
 for x in dynamic_patterns:
 for x in dynamic_patterns:
-  logging.debug("Dynamic pattern: %s" % x)
+  logging.debug("Dynamic pattern: %s" % (x,))
 for x in static_patterns:
 for x in static_patterns:
-  logging.debug("Static pattern: %s" % str(x))
+  logging.debug("Static pattern: %s" % (x,))

+ 75 - 8
desktop/core/src/desktop/views.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 # limitations under the License.
 
 
 import logging
 import logging
+import os
 import sys
 import sys
 import tempfile
 import tempfile
 import time
 import time
@@ -28,6 +29,7 @@ from django.core.servers.basehttp import FileWrapper
 import django.views.debug
 import django.views.debug
 
 
 from desktop.lib.django_util import login_notrequired, render_json, render
 from desktop.lib.django_util import login_notrequired, render_json, render
+from desktop.lib.paths import get_desktop_root
 from desktop.log.access import access_log_level, access_warn
 from desktop.log.access import access_log_level, access_warn
 from desktop.models import UserPreferences
 from desktop.models import UserPreferences
 from desktop import appmanager
 from desktop import appmanager
@@ -144,7 +146,6 @@ def status_bar(request):
   Concatenates multiple views together to build up a "status bar"/"status_bar".
   Concatenates multiple views together to build up a "status bar"/"status_bar".
   These views are registered using register_status_bar_view above.
   These views are registered using register_status_bar_view above.
   """
   """
-  global _status_bar_views
   resp = ""
   resp = ""
   for view in _status_bar_views:
   for view in _status_bar_views:
     try:
     try:
@@ -152,14 +153,15 @@ def status_bar(request):
       if r.status_code == 200:
       if r.status_code == 200:
         resp += r.content
         resp += r.content
       else:
       else:
-        LOG.warning("Failed to execute status_bar view %s" % view)
+        LOG.warning("Failed to execute status_bar view %s" % (view,))
     except:
     except:
-      LOG.exception("Failed to execute status_bar view %s" % view)
+      LOG.exception("Failed to execute status_bar view %s" % (view,))
   return HttpResponse(resp)
   return HttpResponse(resp)
 
 
 def dump_config(request):
 def dump_config(request):
   # Note that this requires login (as do most apps).
   # Note that this requires login (as do most apps).
   show_private = False
   show_private = False
+  conf_dir = os.path.realpath(get_desktop_root('conf'))
 
 
   if not request.user.is_superuser:
   if not request.user.is_superuser:
     return HttpResponse("You must be a superuser.")
     return HttpResponse("You must be a superuser.")
@@ -167,8 +169,11 @@ def dump_config(request):
   if request.GET.get("private"):
   if request.GET.get("private"):
     show_private = True
     show_private = True
 
 
-  return render("dump_config.mako", request, dict(show_private=show_private,
-    top_level=desktop.lib.conf.GLOBAL_CONFIG, apps=appmanager.DESKTOP_MODULES))
+  return render("dump_config.mako", request, dict(
+    show_private=show_private,
+    top_level=desktop.lib.conf.GLOBAL_CONFIG,
+    conf_dir=conf_dir,
+    apps=appmanager.DESKTOP_MODULES))
 
 
 if sys.version_info[0:2] <= (2,4):
 if sys.version_info[0:2] <= (2,4):
   def _threads():
   def _threads():
@@ -204,8 +209,6 @@ def serve_404_error(request, *args, **kwargs):
   """Registered handler for 404. We just return a simple error"""
   """Registered handler for 404. We just return a simple error"""
   access_warn(request, "404 not found")
   access_warn(request, "404 not found")
   return render_to_response("404.html", dict(uri=request.build_absolute_uri()))
   return render_to_response("404.html", dict(uri=request.build_absolute_uri()))
-  return HttpResponse('Page not found. You are trying to access %s' % (request.build_absolute_uri(),),
-                      content_type="text/plain")
 
 
 def serve_500_error(request, *args, **kwargs):
 def serve_500_error(request, *args, **kwargs):
   """Registered handler for 500. We use the debug view to make debugging easier."""
   """Registered handler for 500. We use the debug view to make debugging easier."""
@@ -240,7 +243,7 @@ def log_frontend_event(request):
 
 
   level = _LOG_LEVELS.get(get("level"), logging.INFO)
   level = _LOG_LEVELS.get(get("level"), logging.INFO)
   msg = "Untrusted log event from user %s: %s" % (
   msg = "Untrusted log event from user %s: %s" % (
-    request.user, 
+    request.user,
     get("message", "")[:_MAX_LOG_FRONTEND_EVENT_LENGTH])
     get("message", "")[:_MAX_LOG_FRONTEND_EVENT_LENGTH])
   _LOG_FRONTEND_LOGGER.log(level, msg)
   _LOG_FRONTEND_LOGGER.log(level, msg)
   return HttpResponse("")
   return HttpResponse("")
@@ -255,3 +258,67 @@ def who_am_i(request):
     sleep = 0.0
     sleep = 0.0
   time.sleep(sleep)
   time.sleep(sleep)
   return HttpResponse(request.user.username + "\t" + request.fs.user + "\n")
   return HttpResponse(request.user.username + "\t" + request.fs.user + "\n")
+
+
+# If the app's conf.py has a config_validator() method, call it.
+CONFIG_VALIDATOR = 'config_validator'
+
+#
+# Cache config errors because (1) they mostly don't go away until restart,
+# and (2) they can be costly to compute. So don't stress the system just because
+# the dock bar wants to refresh every n seconds.
+#
+# The actual viewing of all errors may choose to disregard the cache.
+#
+_CONFIG_ERROR_LIST = None
+
+def _get_config_errors(cache=True):
+  """Returns a list of (confvar, err_msg) tuples."""
+  global _CONFIG_ERROR_LIST
+
+  if not cache or _CONFIG_ERROR_LIST is None:
+    error_list = [ ]
+    for module in appmanager.DESKTOP_MODULES:
+      # Get the config_validator() function
+      try:
+        validator = getattr(module.conf, CONFIG_VALIDATOR)
+      except AttributeError:
+        continue
+
+      if not callable(validator):
+        LOG.warn("Auto config validation: %s.%s is not a function" %
+                 (module.conf.__name__, CONFIG_VALIDATOR))
+        continue
+
+      try:
+        error_list.extend(validator())
+      except Exception, ex:
+        LOG.exception("Error in config validation by %s: %s" % (module.nice_name, ex))
+    _CONFIG_ERROR_LIST = error_list
+  return _CONFIG_ERROR_LIST
+
+
+def check_config(request):
+  """Check config and view for the list of errors"""
+  if not request.user.is_superuser:
+    return HttpResponse("You must be a superuser.")
+  conf_dir = os.path.realpath(get_desktop_root('conf'))
+  return render('check_config.mako', request, dict(
+                    error_list=_get_config_errors(cache=False),
+                    conf_dir=conf_dir))
+
+def status_bar_config_check(request):
+  """Alert administrators about configuration problems."""
+  if not request.user.is_superuser:
+    return HttpResponse('')
+
+  error_list = _get_config_errors()
+  if not error_list:
+    # Return an empty response, rather than using the mako template, for performance.
+    return HttpResponse('')
+  return render('config_alert_dock.mako',
+                request,
+                dict(error_list=error_list),
+                force_template=True)
+
+register_status_bar_view(status_bar_config_check)

+ 3 - 1
desktop/core/static/js/Source/CCS/CCS.Desktop.js

@@ -215,7 +215,9 @@ CCS.Desktop = {
 	launch: function(component, args, callback){
 	launch: function(component, args, callback){
 		callback = callback || $empty;
 		callback = callback || $empty;
 		if (!this.hasApp(component)) {
 		if (!this.hasApp(component)) {
-			CCS.error('Could Not Launch App', 'Sorry, we couldn\'t find an app named ' + component + '. Launching a new window instead.');
+			if (component != '_blank') {
+				CCS.error('Could Not Launch App', 'Sorry, we couldn\'t find an app named ' + component + '. Launching a new window instead.');
+			}
 			callback();
 			callback();
 			return window.open(args[0], component);
 			return window.open(args[0], component);
 		}
 		}

+ 34 - 2
desktop/libs/hadoop/src/hadoop/conf.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
 """Settings to configure your Hadoop cluster."""
 """Settings to configure your Hadoop cluster."""
-from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection
+from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, validate_path
 import glob
 import glob
 import os
 import os
 import logging
 import logging
@@ -80,7 +80,7 @@ HADOOP_EXAMPLES_JAR = Config(
   private=True)
   private=True)
 
 
 HADOOP_STREAMING_JAR = Config(
 HADOOP_STREAMING_JAR = Config(
-  key="hadoop_examples_jar",
+  key="hadoop_streaming_jar",
   dynamic_default=find_jar(os.path.join("contrib", "streaming", "hadoop-*streaming*.jar")),
   dynamic_default=find_jar(os.path.join("contrib", "streaming", "hadoop-*streaming*.jar")),
   help="Path to the hadoop-streaming.jar (used by jobdesigner)",
   help="Path to the hadoop-streaming.jar (used by jobdesigner)",
   type=str,
   type=str,
@@ -122,3 +122,35 @@ MR_CLUSTERS = UnspecifiedConfigSection(
       JT_HOST=Config("jobtracker_host", help="IP for JobTracker"),
       JT_HOST=Config("jobtracker_host", help="IP for JobTracker"),
       JT_THRIFT_PORT=Config("thrift_port", help="Thrift port for JobTracker", default=9290,
       JT_THRIFT_PORT=Config("thrift_port", help="Thrift port for JobTracker", default=9290,
                             type=int))))
                             type=int))))
+
+
+def config_validator():
+  """
+  config_validator() -> [ (config_variable, error_message) ]
+
+  Called by core check_config() view.
+  """
+  from hadoop.fs import hadoopfs
+  from hadoop import job_tracker
+  res = [ ]
+
+  # HADOOP_HOME
+  res.extend(validate_path(HADOOP_HOME, is_dir=True))
+  # HADOOP_BIN
+  res.extend(validate_path(HADOOP_BIN, is_dir=False))
+
+  # JARs: even though these are private, we need them to run jobsub
+  res.extend(validate_path(HADOOP_EXAMPLES_JAR, is_dir=False))
+  res.extend(validate_path(HADOOP_STREAMING_JAR, is_dir=False))
+
+  # HDFS_CLUSTERS
+  for name in HDFS_CLUSTERS.keys():
+    cluster = HDFS_CLUSTERS[name]
+    res.extend(hadoopfs.test_fs_configuration(cluster, HADOOP_BIN))
+
+  # MR_CLUSTERS
+  for name in MR_CLUSTERS.keys():
+    cluster = MR_CLUSTERS[name]
+    res.extend(job_tracker.test_jt_configuration(cluster))
+
+  return res

+ 107 - 37
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -22,6 +22,7 @@ import errno
 import logging
 import logging
 import os
 import os
 import posixpath
 import posixpath
+import random
 import stat as statconsts
 import stat as statconsts
 import subprocess
 import subprocess
 import sys
 import sys
@@ -34,6 +35,7 @@ from thrift.protocol import TBinaryProtocol
 
 
 from django.utils.encoding import smart_str, force_unicode
 from django.utils.encoding import smart_str, force_unicode
 from desktop.lib import thrift_util, i18n
 from desktop.lib import thrift_util, i18n
+from desktop.lib.conf import validate_port
 from hadoop.api.hdfs import Namenode, Datanode
 from hadoop.api.hdfs import Namenode, Datanode
 from hadoop.api.hdfs.constants import QUOTA_DONT_SET, QUOTA_RESET
 from hadoop.api.hdfs.constants import QUOTA_DONT_SET, QUOTA_RESET
 from hadoop.api.common.ttypes import RequestContext, IOException
 from hadoop.api.common.ttypes import RequestContext, IOException
@@ -79,6 +81,96 @@ def decode_fs_path(path):
   return force_unicode(path, HDFS_ENCODING, errors='strict')
   return force_unicode(path, HDFS_ENCODING, errors='strict')
 
 
 
 
+def test_fs_configuration(fs_config, hadoop_bin_conf):
+  """Test FS configuration. Returns list of (confvar, error)."""
+  TEST_FILE = '/tmp/.hue_config_test.%s' % (random.randint(0, 9999999999))
+  res = [ ]
+
+  res.extend(validate_port(fs_config.NN_THRIFT_PORT))
+  res.extend(validate_port(fs_config.NN_HDFS_PORT))
+  if res:
+    return res
+
+  # Check thrift plugin
+  try:
+    fs = HadoopFileSystem(fs_config.NN_HOST.get(),
+                          fs_config.NN_THRIFT_PORT.get(),
+                          fs_config.NN_HDFS_PORT.get(),
+                          hadoop_bin_conf.get())
+
+    fs.setuser(fs.superuser)
+    ls = fs.listdir('/')
+  except TTransport.TTransportException:
+    msg = 'Failed to contact Namenode plugin at %s:%s.' % \
+            (fs_config.NN_HOST.get(), fs_config.NN_THRIFT_PORT.get())
+    LOG.exception(msg)
+    res.append((fs_config, msg))
+    return res
+  except (IOError, IOException):
+    msg = 'Failed to see HDFS root directory at %s. Please check HDFS configuration.' % (fs.uri,)
+    LOG.exception(msg)
+    res.append((fs_config, msg))
+    return res
+
+  if 'tmp' not in ls:
+    return res
+
+  # Check nn port (via upload)
+  try:
+    w_file = fs.open(TEST_FILE, 'w')
+  except OSError, ex:
+    msg = 'Failed to execute Hadoop (%s)' % (hadoop_bin_conf.get(),)
+    LOG.exception(msg)
+    res.append((hadoop_bin_conf, msg))
+    return res
+
+  try:
+    try:
+      w_file.write('hello world')
+      w_file.close()
+    except IOError:
+      msg = 'Failed to upload files using %s' % (fs.uri,)
+      LOG.exception(msg)
+      res.append((fs_config.NN_HDFS_PORT, msg))
+      return res
+
+    # Check dn plugin (via read)
+    try:
+      r_file = fs.open(TEST_FILE, 'r')
+      r_file.read()
+    except Exception:
+      msg = 'Failed to read file. Are all datanodes configured with the HUE plugin?'
+      res.append((fs_config, msg))
+  finally:
+    # Cleanup. Ignore if file not found.
+    try:
+      if fs.exists(TEST_FILE):
+        fs.remove(TEST_FILE)
+    except Exception, ex:
+      LOG.error('Failed to cleanup test file "%s:%s": %s' % (fs.uri, TEST_FILE, ex))
+  return res
+
+
+def _coerce_exceptions(function):
+  """
+  Decorator that causes exceptions thrown by the decorated function
+  to be coerced into generic exceptions from the hadoop.fs.exceptions
+  module.
+  """
+  def wrapper(*args, **kwargs):
+    try:
+      return function(*args, **kwargs)
+    except IOException, e:
+      e.msg = force_unicode(e.msg, errors='replace')
+      e.stack = force_unicode(e.stack, errors='replace')
+      LOG.exception("Exception in Hadoop FS call " + function.__name__)
+      if e.clazz == HADOOP_ACCESSCONTROLEXCEPTION:
+        raise PermissionDeniedException(e.msg, e)
+      else:
+        raise
+  return wrapper
+
+
 class HadoopFileSystem(object):
 class HadoopFileSystem(object):
   """
   """
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.
   Implementation of Filesystem APIs through Thrift to a Hadoop cluster.
@@ -113,26 +205,6 @@ class HadoopFileSystem(object):
   def _get_hdfs_base(self):
   def _get_hdfs_base(self):
     return "hdfs://%s:%d" % (self.host, self.hdfs_port) # TODO(todd) fetch the port from the NN thrift
     return "hdfs://%s:%d" % (self.host, self.hdfs_port) # TODO(todd) fetch the port from the NN thrift
 
 
-
-  def _coerce_exceptions(function):
-    """
-    Decorator that causes exceptions thrown by the decorated function
-    to be coerced into generic exceptions from the hadoop.fs.exceptions
-    module.
-    """
-    def wrapper(*args, **kwargs):
-      try:
-        return function(*args, **kwargs)
-      except IOException, e:
-        e.msg = force_unicode(e.msg, errors='replace')
-        e.stack = force_unicode(e.stack, errors='replace')
-        LOG.exception("Exception in Hadoop FS call " + function.__name__)
-        if e.clazz == HADOOP_ACCESSCONTROLEXCEPTION:
-          raise PermissionDeniedException(e.msg, e)
-        else:
-          raise
-    return wrapper
-
   def _resolve_hadoop_path(self):
   def _resolve_hadoop_path(self):
     """The hadoop_bin_path configuration may be a non-absolute path, in which case
     """The hadoop_bin_path configuration may be a non-absolute path, in which case
     it's checked against $PATH.
     it's checked against $PATH.
@@ -144,8 +216,7 @@ class HadoopFileSystem(object):
       if os.path.exists(path):
       if os.path.exists(path):
         self.hadoop_bin_path = os.path.abspath(path)
         self.hadoop_bin_path = os.path.abspath(path)
         return
         return
-
-    raise Exception("Hadoop binary (%s) does not exist." % self.hadoop_bin_path)
+    raise OSError(errno.ENOENT, "Hadoop binary (%s) does not exist." % (self.hadoop_bin_path,))
 
 
   @property
   @property
   def uri(self):
   def uri(self):
@@ -200,9 +271,9 @@ class HadoopFileSystem(object):
     path = encode_fs_path(path)
     path = encode_fs_path(path)
     stat = self._hadoop_stat(path)
     stat = self._hadoop_stat(path)
     if not stat:
     if not stat:
-      raise IOError("File not found: %s" % path)
+      raise IOError(errno.ENOENT, "File not found: %s" % path)
     if stat.isDir:
     if stat.isDir:
-      raise IOError("Is a directory: %s" % path)
+      raise IOError(errno.EISDIR, "Is a directory: %s" % path)
 
 
     success = self.nn_client.unlink(
     success = self.nn_client.unlink(
       self.request_context, normpath(path), recursive=False)
       self.request_context, normpath(path), recursive=False)
@@ -222,9 +293,9 @@ class HadoopFileSystem(object):
     path = encode_fs_path(path)
     path = encode_fs_path(path)
     stat = self._hadoop_stat(path)
     stat = self._hadoop_stat(path)
     if not stat:
     if not stat:
-      raise IOError("Directory not found: %s" % (path,))
+      raise IOError(errno.ENOENT, "Directory not found: %s" % (path,))
     if not stat.isDir:
     if not stat.isDir:
-      raise IOError("Is not a directory: %s" % (path,))
+      raise IOError(errno.EISDIR, "Is not a directory: %s" % (path,))
 
 
     success = self.nn_client.unlink(
     success = self.nn_client.unlink(
       self.request_context, normpath(path), recursive=recursive)
       self.request_context, normpath(path), recursive=recursive)
@@ -253,9 +324,8 @@ class HadoopFileSystem(object):
 
 
   @_coerce_exceptions
   @_coerce_exceptions
   def get_content_summaries(self, paths):
   def get_content_summaries(self, paths):
-    path = encode_fs_path(path)
-    summaries = self.nn_client.multiGetContentSummary(self.request_context,
-                                                      [normpath(path) for path in paths])
+    paths = [ normpath(encode_fs_path(path)) for path in paths ]
+    summaries = self.nn_client.multiGetContentSummary(self.request_context, paths)
     def _fix_summary(summary):
     def _fix_summary(summary):
       summary.path = decode_fs_path(summary.path)
       summary.path = decode_fs_path(summary.path)
       return summary
       return summary
@@ -274,11 +344,11 @@ class HadoopFileSystem(object):
   def rename_star(self, old_dir, new_dir):
   def rename_star(self, old_dir, new_dir):
     """Equivalent to `mv old_dir/* new"""
     """Equivalent to `mv old_dir/* new"""
     if not self.isdir(old_dir):
     if not self.isdir(old_dir):
-      raise IOError("'%s' is not a directory" % (old_dir,))
+      raise IOError(errno.ENOTDIR, "'%s' is not a directory" % (old_dir,))
     if not self.exists(new_dir):
     if not self.exists(new_dir):
       self.mkdir(new_dir)
       self.mkdir(new_dir)
     elif not self.isdir(new_dir):
     elif not self.isdir(new_dir):
-      raise IOError("'%s' is not a directory" % (new_dir,))
+      raise IOError(errno.ENOTDIR, "'%s' is not a directory" % (new_dir,))
     ls = self.listdir(old_dir)
     ls = self.listdir(old_dir)
     for dirent in ls:
     for dirent in ls:
       self.rename(HadoopFileSystem.join(old_dir, dirent),
       self.rename(HadoopFileSystem.join(old_dir, dirent),
@@ -308,7 +378,7 @@ class HadoopFileSystem(object):
     stat = self._hadoop_stat(path)
     stat = self._hadoop_stat(path)
     if not stat:
     if not stat:
       if raise_on_fnf:
       if raise_on_fnf:
-        raise IOError("File %s not found" % path)
+        raise IOError(errno.ENOENT, "File %s not found" % (path,))
       else:
       else:
         return None
         return None
     ret = self._unpack_stat(stat)
     ret = self._unpack_stat(stat)
@@ -560,7 +630,7 @@ def require_open(func):
   """
   """
   def wrapper(self, *args, **kwargs):
   def wrapper(self, *args, **kwargs):
     if self.closed:
     if self.closed:
-      raise IOError("I/O operation on closed file")
+      raise IOError(errno.EBADF, "I/O operation on closed file")
     return func(self, *args, **kwargs)
     return func(self, *args, **kwargs)
   return wrapper
   return wrapper
 
 
@@ -583,9 +653,9 @@ class File(object):
     stat = self._stat()
     stat = self._stat()
 
 
     if stat is None:
     if stat is None:
-      raise IOError("No such file or directory: '%s'" % path)
+      raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path)
     if stat.isDir:
     if stat.isDir:
-      raise IOError("Is a directory: '%s'" % path)
+      raise IOError(errno.EISDIR, "Is a directory: '%s'" % path)
     #TODO(todd) somehow we need to check permissions here - maybe we need an access() call?
     #TODO(todd) somehow we need to check permissions here - maybe we need an access() call?
 
 
   # Minimal context manager implementation.
   # Minimal context manager implementation.
@@ -607,7 +677,7 @@ class File(object):
     elif whence == SEEK_END:
     elif whence == SEEK_END:
       self.pos = self._stat().length + offset
       self.pos = self._stat().length + offset
     else:
     else:
-      raise IOError("Invalid argument to seek for whence")
+      raise IOError(errno.EINVAL, "Invalid argument to seek for whence")
 
 
   @require_open
   @require_open
   def tell(self):
   def tell(self):
@@ -691,7 +761,7 @@ class FileUpload(object):
       extra_confs.append("-Ddfs.block.size=%d" % block_size)
       extra_confs.append("-Ddfs.block.size=%d" % block_size)
     self.subprocess_cmd = [self.fs.hadoop_bin_path,
     self.subprocess_cmd = [self.fs.hadoop_bin_path,
                            "dfs",
                            "dfs",
-                           "-Dfs.default.name=" + self.fs._get_hdfs_base(),
+                           "-Dfs.default.name=" + self.fs.uri,
                            "-Dhadoop.job.ugi=" + self.fs.ugi] + \
                            "-Dhadoop.job.ugi=" + self.fs.ugi] + \
                            extra_confs + \
                            extra_confs + \
                            ["-put", "-", encode_fs_path(path)]
                            ["-put", "-", encode_fs_path(path)]

+ 19 - 1
desktop/libs/hadoop/src/hadoop/job_tracker.py

@@ -18,6 +18,7 @@
 # Django-side implementation of the JobTracker plugin interface
 # Django-side implementation of the JobTracker plugin interface
 
 
 from desktop.lib import thrift_util
 from desktop.lib import thrift_util
+from desktop.lib.conf import validate_port
 from desktop.lib.thrift_util import fixup_enums
 from desktop.lib.thrift_util import fixup_enums
 
 
 from hadoop.api.jobtracker import Jobtracker
 from hadoop.api.jobtracker import Jobtracker
@@ -26,6 +27,7 @@ from hadoop.api.jobtracker.ttypes import ThriftJobID, ThriftTaskAttemptID, \
     ThriftTaskState, ThriftJobState, ThriftJobPriority, TaskNotFoundException, \
     ThriftTaskState, ThriftJobState, ThriftJobPriority, TaskNotFoundException, \
     JobTrackerState, JobNotFoundException, ThriftTaskQueryState
     JobTrackerState, JobNotFoundException, ThriftTaskQueryState
 from hadoop.api.common.ttypes import RequestContext
 from hadoop.api.common.ttypes import RequestContext
+from thrift.transport import TTransport
 
 
 import threading
 import threading
 
 
@@ -33,11 +35,27 @@ VALID_TASK_STATES = set(["succeeded", "failed", "running", "pending", "killed"])
 VALID_TASK_TYPES = set(["map", "reduce", "job_cleanup", "job_setup"])
 VALID_TASK_TYPES = set(["map", "reduce", "job_cleanup", "job_setup"])
 
 
 # timeout (seconds) for thrift calls to jobtracker
 # timeout (seconds) for thrift calls to jobtracker
-JT_THRIFT_TIMEOUT=15
+JT_THRIFT_TIMEOUT = 15
 
 
 DEFAULT_USER = "webui"
 DEFAULT_USER = "webui"
 DEFAULT_GROUPS = ["webui"]
 DEFAULT_GROUPS = ["webui"]
 
 
+def test_jt_configuration(cluster):
+  """Test FS configuration. Returns list of (confvar, error)."""
+  err = validate_port(cluster.JT_THRIFT_PORT)
+  if err:
+    return err
+
+  try:
+    jt = LiveJobTracker(cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get())
+    jt.runtime_info()
+  except TTransport.TTransportException:
+    msg = 'Failed to contact JobTracker plugin at %s:%s.' % \
+          (cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get())
+    return [ (cluster, msg) ]
+  return []
+
+
 class LiveJobTracker(object):
 class LiveJobTracker(object):
   """
   """
   Connects to a JobTracker over our Thrift interface.
   Connects to a JobTracker over our Thrift interface.

+ 61 - 2
desktop/libs/hadoop/src/hadoop/tests.py

@@ -20,9 +20,13 @@ Tests for libs/hadoop
 import cStringIO
 import cStringIO
 import os
 import os
 
 
-from nose.tools import assert_true, assert_equal
+from nose.tools import assert_true, assert_equal, assert_false
 from nose.plugins.attrib import attr
 from nose.plugins.attrib import attr
 
 
+import desktop.views
+
+from desktop.lib.django_test_util import make_logged_in_client
+from hadoop import conf
 from hadoop import confparse
 from hadoop import confparse
 from hadoop import mini_cluster
 from hadoop import mini_cluster
 
 
@@ -103,5 +107,60 @@ def test_tricky_confparse():
   We found (experimentally) that dealing with a file
   We found (experimentally) that dealing with a file
   sometimes triggered the wrong results here.
   sometimes triggered the wrong results here.
   """
   """
-  cp_data = confparse.ConfParse(file(os.path.join(os.path.dirname(__file__), "test_data", "sample_conf.xml")))
+  cp_data = confparse.ConfParse(file(os.path.join(os.path.dirname(__file__),
+                                                  "test_data",
+                                                  "sample_conf.xml")))
   assert_equal("org.apache.hadoop.examples.SleepJob", cp_data["mapred.mapper.class"])
   assert_equal("org.apache.hadoop.examples.SleepJob", cp_data["mapred.mapper.class"])
+
+
+def test_config_validator_basic():
+  reset = (
+    conf.HADOOP_STREAMING_JAR.set_for_testing('/tmp'),
+    conf.HDFS_CLUSTERS['default'].NN_THRIFT_PORT.set_for_testing(1),
+    conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(70000),
+  )
+  try:
+    cli = make_logged_in_client()
+    resp = cli.get('/debug/check_config')
+    assert_true('hadoop.hadoop_streaming_jar' in resp.content)
+    assert_true('Not a file' in resp.content)
+    assert_true('hadoop.hdfs_clusters.default' in resp.content)
+    assert_true('Failed to contact Namenode plugin' in resp.content)
+    assert_true('hadoop.mapred_clusters.default.thrift_port' in resp.content)
+    assert_true('Port should be' in resp.content)
+  finally:
+    for old_conf in reset:
+      old_conf()
+
+
+@attr('requires_hadoop')
+def test_config_validator_more():
+  # TODO: Setup DN to not load the plugin, which is a common user error.
+
+  # We don't actually use the mini_cluster. But the cluster sets up the correct
+  # configuration that forms the test basis.
+  cluster = mini_cluster.shared_cluster()
+  if not cluster.fs.exists('/tmp'):
+    cluster.fs.setuser(cluster.fs.superuser)
+    cluster.fs.mkdir('/tmp', 0777)
+  cli = make_logged_in_client()
+
+  reset = (
+    conf.HADOOP_BIN.set_for_testing(cluster.fs.hadoop_bin_path),
+    conf.HDFS_CLUSTERS['default'].NN_HOST.set_for_testing('localhost'),
+    conf.HDFS_CLUSTERS['default'].NN_HDFS_PORT.set_for_testing(22),
+    conf.HDFS_CLUSTERS["default"].NN_THRIFT_PORT.set_for_testing(cluster.fs.thrift_port),
+    conf.MR_CLUSTERS["default"].JT_HOST.set_for_testing("localhost"),
+    conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(23),
+  )
+  try:
+    resp = cli.get('/debug/check_config')
+
+    assert_false('Failed to contact Namenode plugin' in resp.content)
+    assert_false('Failed to see HDFS root' in resp.content)
+    assert_true('Failed to upload files' in resp.content)
+    assert_true('Failed to contact JobTracker plugin' in resp.content)
+  finally:
+    for old_conf in reset:
+      old_conf()
+    cluster.shutdown()

+ 39 - 20
docs/sdk/sdk.md

@@ -395,15 +395,15 @@ ini-style format).	By default, HUE loads all `*.ini` files in the `build/desktop
 directory.	The configuration files have the following format:
 directory.	The configuration files have the following format:
 
 
 		# This is a comment
 		# This is a comment
-		[ app_name ]								# Same as your app's name
+		[ app_name ]					# Same as your app's name
 		app_property = "Pink Floyd"
 		app_property = "Pink Floyd"
 
 
-		[[ section_a ]]							# The double brackets start a section under [ app_name ]
-		a_weight = 80								# that is useful for grouping
+		[[ section_a ]]					# The double brackets start a section under [ app_name ]
+		a_weight = 80					# that is useful for grouping
 		a_height = 180
 		a_height = 180
 
 
-		[[ filesystems ]]						# Sections are also useful for making a list
-		[[[ cluster_1 ]]]						# All list members are sub-sections of the same type
+		[[ filesystems ]]				# Sections are also useful for making a list
+		[[[ cluster_1 ]]]				# All list members are sub-sections of the same type
 		namenode_host = localhost
 		namenode_host = localhost
 		# User may define more:
 		# User may define more:
 		# [[[ cluster_2 ]]]
 		# [[[ cluster_2 ]]]
@@ -418,43 +418,62 @@ define the following:
 
 
 * A `desktop.lib.conf.Config` object for `app_property`, such as:
 * A `desktop.lib.conf.Config` object for `app_property`, such as:
 <pre>
 <pre>
-		MY_PROPERTY = Config(key='app_property', default='Beatles', help='blah')
+	MY_PROPERTY = Config(key='app_property', default='Beatles', help='blah')
 </pre>
 </pre>
 	You can access its value by `MY_PROPERTY.get()`.
 	You can access its value by `MY_PROPERTY.get()`.
 
 
 * A `desktop.lib.conf.ConfigSection` object for `section_a`, such as:
 * A `desktop.lib.conf.ConfigSection` object for `section_a`, such as:
 <pre>
 <pre>
-		SECTION_A = ConfigSection(key='section_a',
-															help='blah',
-															members=dict(
-																AWEIGHT=Config(key='a_weight', type=int, default=0),
-																AHEIGHT=Config(key='a_height', type=int, default=0)))
+	SECTION_A = ConfigSection(key='section_a',
+				help='blah',
+				members=dict(
+					AWEIGHT=Config(key='a_weight', type=int, default=0),
+					AHEIGHT=Config(key='a_height', type=int, default=0)))
 </pre>
 </pre>
 	You can access the values by `SECTION_A.AWEIGHT.get()`.
 	You can access the values by `SECTION_A.AWEIGHT.get()`.
 
 
 * A `desktop.lib.conf.UnspecifiedConfigSection` object for `filesystems`, such as:
 * A `desktop.lib.conf.UnspecifiedConfigSection` object for `filesystems`, such as:
 <pre>
 <pre>
-		FS = UnspecifiedConfigSection(
-														 key='filesystems',
-														 each=ConfigSection(members=dict(
-																nn_host=Config(key='namenode_host', required=True))
+	FS = UnspecifiedConfigSection(
+			key='filesystems',
+			each=ConfigSection(members=dict(
+					nn_host=Config(key='namenode_host', required=True))
 </pre>
 </pre>
 	An `UnspecifiedConfigSection` is useful when the children of the section are not known.
 	An `UnspecifiedConfigSection` is useful when the children of the section are not known.
 	When HUE loads your application's configuration, it binds all sub-sections. You can
 	When HUE loads your application's configuration, it binds all sub-sections. You can
 	access the values by:
 	access the values by:
 <pre>
 <pre>
-		cluster1_val = FS['cluster_1'].nn_host.get()
-		all_clusters = FS.keys()
-		for cluster in all_clusters:
-				val = FS[cluster].nn_host.get()
+	cluster1_val = FS['cluster_1'].nn_host.get()
+	all_clusters = FS.keys()
+	for cluster in all_clusters:
+			val = FS[cluster].nn_host.get()
 </pre>
 </pre>
 
 
+Your HUE application can automatically detect configuration problems and alert
+the admin. To take advantage of this feature, create a `config_validator`
+function in your `conf.py`:
+
+<pre>
+	def config_validator():
+		"""
+		config_validator() -> [(config_variable, error_msg)] or None
+		Called by core check_config() view.
+		"""
+		res = [ ]
+		if not REQUIRED_PROPERTY.get():
+			res.append((REQUIRED_PROPERTY, "This variable must be set"))
+		if MY_INT_PROPERTY.get() < 0:
+			res.append((MY_INT_PROPERTY, "This must be a non-negative number"))
+		return res
+</pre>
+
+
 <div class="note">
 <div class="note">
 You should specify the <code>help="..."</code> argument to all configuration related objects in
 You should specify the <code>help="..."</code> argument to all configuration related objects in
 your <code>conf.py</code>. The examples omit some for the sake of space. But you and your
 your <code>conf.py</code>. The examples omit some for the sake of space. But you and your
 application's users can view all the configuration variables by doing:
 application's users can view all the configuration variables by doing:
 <pre>
 <pre>
-		$ build/env/bin/hue config_help
+	$ build/env/bin/hue config_help
 </pre>
 </pre>
 </div>
 </div>