Browse Source

HUE-662. [config] Each Hadoop hdfs/mr/yarn cluster to define its own env

* Have Beeswax and Shell define the appropriate HADOOP_MAPRED_HOME for the
  child process.
* Renamed JT_HOST & RM_HOST to just "HOST", and JT_PORT & RM_PORT to just
  "PORT" for consistent reference.
* Fixed README to drop the requirement on having a HADOOP_HOME. The build
  does not depend on one because we're using maven.
bc Wong 13 years ago
parent
commit
c7e29b51f7

+ 1 - 1
README.testing

@@ -84,7 +84,7 @@ DESKTOP_DB_CONFIG=<db engine:db name:test db name:username:password:host:port>
 Writing tests that depend on Hadoop
 Writing tests that depend on Hadoop
 ===================================
 ===================================
 
 
-Use mini_cluster.py!  You should tag such tests with "requires_hadoop", as follows:
+Use pseudo_hdfs4.py!  You should tag such tests with "requires_hadoop", as follows:
 
 
   from nose.plugins.attrib import attr
   from nose.plugins.attrib import attr
 
 

+ 4 - 7
apps/beeswax/beeswax_server.sh

@@ -19,8 +19,8 @@
 
 
 set -o errexit
 set -o errexit
 
 
-if [ -z "$HADOOP_HOME" ]; then
-  echo "\$HADOOP_HOME must be specified" 1>&2
+if [ -z "$HADOOP_CONF_DIR" ]; then
+  echo "\$HADOOP_CONF_DIR must be specified" 1>&2
   exit 1
   exit 1
 fi
 fi
 echo \$HADOOP_HOME=$HADOOP_HOME
 echo \$HADOOP_HOME=$HADOOP_HOME
@@ -46,7 +46,6 @@ fi
 
 
 echo \$HIVE_HOME=$HIVE_HOME
 echo \$HIVE_HOME=$HIVE_HOME
 
 
-
 BEESWAX_ROOT=$(dirname $0)
 BEESWAX_ROOT=$(dirname $0)
 BEESWAX_JAR=$BEESWAX_ROOT/java-lib/BeeswaxServer.jar
 BEESWAX_JAR=$BEESWAX_ROOT/java-lib/BeeswaxServer.jar
 HIVE_LIB=$HIVE_HOME/lib
 HIVE_LIB=$HIVE_HOME/lib
@@ -65,15 +64,13 @@ echo \$HADOOP_OPTS=$HADOOP_OPTS
 # and to force hive-default to correspond to the Hive version we have.
 # and to force hive-default to correspond to the Hive version we have.
 # Because we are abusing HADOOP_CONF_DIR, we have to emulate its default
 # Because we are abusing HADOOP_CONF_DIR, we have to emulate its default
 # behavior here as well.
 # behavior here as well.
-if [ -z "$HADOOP_CONF_DIR" ]; then
-  HADOOP_CONF_DIR="$HADOOP_HOME/conf"
-fi
 if [ -f $HADOOP_CONF_DIR/hadoop-env.sh ]; then
 if [ -f $HADOOP_CONF_DIR/hadoop-env.sh ]; then
   . $HADOOP_CONF_DIR/hadoop-env.sh
   . $HADOOP_CONF_DIR/hadoop-env.sh
 fi
 fi
 
 
-export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$BEESWAX_ROOT/../../desktop/conf:$HADOOP_CONF_DIR
+export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$HADOOP_CONF_DIR
 echo \$HADOOP_CONF_DIR=$HADOOP_CONF_DIR
 echo \$HADOOP_CONF_DIR=$HADOOP_CONF_DIR
+echo \$HADOOP_MAPRED_HOME=$HADOOP_MAPRED_HOME
 
 
 # Note: I've had trouble running this with just "java -jar" with the classpath
 # Note: I've had trouble running this with just "java -jar" with the classpath
 # determined with a seemingly appropriate find command.
 # determined with a seemingly appropriate find command.

+ 22 - 12
apps/beeswax/src/beeswax/management/commands/beeswax_server.py

@@ -22,26 +22,36 @@ from django.core.management.base import NoArgsCommand
 import beeswax.conf
 import beeswax.conf
 import beeswax.hive_site
 import beeswax.hive_site
 import desktop.conf
 import desktop.conf
-import hadoop.conf
-import os
+import hadoop.cluster
+
 import logging
 import logging
+import os
+import sys
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 
 
 class Command(NoArgsCommand):
 class Command(NoArgsCommand):
   """ Starts beeswax daemon.  """
   """ Starts beeswax daemon.  """
+
   def handle_noargs(self, **options):
   def handle_noargs(self, **options):
+    cluster_conf = hadoop.cluster.get_cluster_conf_for_job_submission()
+    if cluster_conf is None:
+      LOG.error("Configuration does not contain any MR/Yarn clusters with "
+                "`submit_to' enabled. Cannot start BeeswaxServer.")
+      sys.exit(1)
+
     env = os.environ.copy()
     env = os.environ.copy()
-    env['HADOOP_HOME'] = hadoop.conf.HADOOP_HOME.get()
-    env['HADOOP_BIN'] = hadoop.conf.HADOOP_BIN.get()
-    if hadoop.conf.HADOOP_CONF_DIR.get():
-      env['HADOOP_CONF_DIR'] = hadoop.conf.HADOOP_CONF_DIR.get()
-    if beeswax.conf.BEESWAX_HIVE_HOME_DIR.get():
-      env['HIVE_HOME'] = beeswax.conf.BEESWAX_HIVE_HOME_DIR.get()
-    if beeswax.conf.BEESWAX_HIVE_CONF_DIR.get():
-      env['HIVE_CONF_DIR'] = beeswax.conf.BEESWAX_HIVE_CONF_DIR.get()
-    if beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get():
-      env['HADOOP_HEAPSIZE'] = beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get()
+    def set_if_present(name, val):
+      if val:
+        env[name] = val
+
+    env['HADOOP_BIN'] = cluster_conf.HADOOP_BIN.get()
+    set_if_present('HADOOP_MAPRED_HOME', cluster_conf.HADOOP_MAPRED_HOME.get())
+    set_if_present('HADOOP_CONF_DIR', cluster_conf.HADOOP_CONF_DIR.get())
+    set_if_present('HADOOP_HEAPSIZE', beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get())
+    set_if_present('HIVE_HOME', beeswax.conf.BEESWAX_HIVE_HOME_DIR.get())
+    set_if_present('HIVE_CONF_DIR', beeswax.conf.BEESWAX_HIVE_CONF_DIR.get())
+
     bin = beeswax.conf.BEESWAX_SERVER_BIN.get()
     bin = beeswax.conf.BEESWAX_SERVER_BIN.get()
 
 
     # Host that desktop is running on
     # Host that desktop is running on

+ 1 - 1
apps/jobsub/src/jobsub/submit.py

@@ -69,7 +69,7 @@ class Submission(object):
       raise Exception("Job design already submitted (Oozie job id %s)" % (self.job_id,))
       raise Exception("Job design already submitted (Oozie job id %s)" % (self.job_id,))
 
 
     fs_defaultfs = self._fs.fs_defaultfs
     fs_defaultfs = self._fs.fs_defaultfs
-    jobtracker = hadoop.cluster.get_cluster_for_job_submission()
+    jobtracker = hadoop.cluster.get_cluster_addr_for_job_submission()
 
 
     try:
     try:
       wf_dir = self._get_and_create_deployment_dir()
       wf_dir = self._get_and_create_deployment_dir()

+ 1 - 0
apps/shell/src/shell/constants.py

@@ -49,6 +49,7 @@ NO_SUCH_USER = "noSuchUser"
 SHELL_NOT_ALLOWED = "shellNotAllowed"
 SHELL_NOT_ALLOWED = "shellNotAllowed"
 HOME = "HOME"
 HOME = "HOME"
 HADOOP_HOME = "HADOOP_HOME"
 HADOOP_HOME = "HADOOP_HOME"
+HADOOP_MAPRED_HOME = "HADOOP_MAPRED_HOME"
 HADOOP_TOKEN_FILE_LOCATION = 'HADOOP_TOKEN_FILE_LOCATION'
 HADOOP_TOKEN_FILE_LOCATION = 'HADOOP_TOKEN_FILE_LOCATION'
 EXISTS = "exists"
 EXISTS = "exists"
 
 

+ 10 - 3
apps/shell/src/shell/shellmanager.py

@@ -21,7 +21,6 @@ Shell class itself, but a lot also happens in ShellManager.
 
 
 
 
 import cStringIO
 import cStringIO
-import desktop.lib.i18n
 import errno
 import errno
 import eventlet
 import eventlet
 import hadoop.conf
 import hadoop.conf
@@ -39,7 +38,8 @@ import tty
 from eventlet.green import os
 from eventlet.green import os
 from eventlet.green import select
 from eventlet.green import select
 from eventlet.green import time
 from eventlet.green import time
-from hadoop.cluster import all_mrclusters, get_all_hdfs
+from hadoop.cluster import all_mrclusters, get_all_hdfs, \
+                           get_cluster_conf_for_job_submission
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
 SHELL_OUTPUT_LOGGER = logging.getLogger("shell_output")
 SHELL_OUTPUT_LOGGER = logging.getLogger("shell_output")
@@ -388,6 +388,7 @@ class ShellManager(object):
     self._parse_configs()
     self._parse_configs()
     eventlet.spawn_after(1, self._handle_periodic)
     eventlet.spawn_after(1, self._handle_periodic)
 
 
+
   @classmethod
   @classmethod
   def global_instance(cls):
   def global_instance(cls):
     if not hasattr(cls, "_global_instance"):
     if not hasattr(cls, "_global_instance"):
@@ -395,9 +396,15 @@ class ShellManager(object):
     return cls._global_instance
     return cls._global_instance
 
 
   def _parse_configs(self):
   def _parse_configs(self):
+    mr_cluster = get_cluster_conf_for_job_submission()
+    if mr_cluster is None:
+      mapred_home = None
+    else:
+      mapred_home = mr_cluster.HADOOP_MAPRED_HOME.get()
+
     shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
     shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
     for item in shell.conf.SHELL_TYPES.keys():
     for item in shell.conf.SHELL_TYPES.keys():
-      env_for_shell = { constants.HADOOP_HOME: hadoop.conf.HADOOP_HOME.get() }
+      env_for_shell = { constants.HADOOP_MAPRED_HOME: mapred_home }
       command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
       command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
       nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
       nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
       executable_exists = utils.executable_exists(command)
       executable_exists = utils.executable_exists(command)

+ 39 - 13
desktop/conf.dist/hue.ini

@@ -165,15 +165,6 @@
 
 
 [hadoop]
 [hadoop]
 
 
-  # If you installed Hadoop in a different location, you need to set
-  # hadoop_home, in which bin/hadoop, the Hadoop wrapper script, is found.
-  #
-  # NOTE: Hue depends on Cloudera's Distribution of Hadoop version 4 (CDH4)
-  # or later.
-  hadoop_home=/usr/lib/hadoop
-  hadoop_bin=/usr/bin/hadoop
-  # hadoop_conf_dir=/etc/hadoop/conf
-
   # Configuration for HDFS NameNode
   # Configuration for HDFS NameNode
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[hdfs_clusters]]
   [[hdfs_clusters]]
@@ -185,11 +176,23 @@
       # Change this if your HDFS cluster is Kerberos-secured
       # Change this if your HDFS cluster is Kerberos-secured
       ## security_enabled=false
       ## security_enabled=false
 
 
-      # Use WebHdfs/HttpFs as the communication mechanism. To fallback to
-      # using the Thrift plugin (used in Hue 1.x), this must be uncommented
-      # and explicitly set to the empty value.
+      # Use WebHdfs/HttpFs as the communication mechanism.
+      # This should be the web service root URL, such as
+      # http://namenode:50070/webhdfs/v1
       ## webhdfs_url=
       ## webhdfs_url=
 
 
+      # Settings about this HDFS cluster. If you install HDFS in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_HDFS_HOME or /usr/lib/hadoop-hdfs
+      ## hadoop_hdfs_home=/usr/lib/hadoop-hdfs
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
   # Configuration for MapReduce 0.20 JobTracker (MR1)
   # Configuration for MapReduce 0.20 JobTracker (MR1)
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[mapred_clusters]]
   [[mapred_clusters]]
@@ -202,11 +205,22 @@
       # Thrift plug-in port for the JobTracker
       # Thrift plug-in port for the JobTracker
       ## thrift_port=9290
       ## thrift_port=9290
       # Whether to submit jobs to this cluster
       # Whether to submit jobs to this cluster
-      ## submit_to=False
+      ## submit_to=True
 
 
       # Change this if your MapReduce cluster is Kerberos-secured
       # Change this if your MapReduce cluster is Kerberos-secured
       ## security_enabled=false
       ## security_enabled=false
 
 
+      # Settings about this MR1 cluster. If you install MR1 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR1_HOME or /usr/lib/hadoop-0.20-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-0.20-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
 
 
   # Configuration for Yarn (MR2)
   # Configuration for Yarn (MR2)
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
@@ -220,6 +234,18 @@
       # Whether to submit jobs to this cluster
       # Whether to submit jobs to this cluster
       ## submit_to=False
       ## submit_to=False
 
 
+      # Settings about this MR2 cluster. If you install MR2 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR2_HOME or /usr/lib/hadoop-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure Beeswax
 # Settings to configure Beeswax

+ 36 - 11
desktop/conf/pseudo-distributed.ini.tmpl

@@ -171,17 +171,6 @@
 
 
 [hadoop]
 [hadoop]
 
 
-  # If you installed Hadoop in a different location, you need to set
-  # hadoop_home, in which bin/hadoop, the Hadoop wrapper script, is found.
-  #
-  # NOTE: Hue depends on Cloudera's Distribution of Hadoop version 4 (CDH4)
-  # or later.
-  hadoop_home=$HADOOP_HOME
-  hadoop_bin=$HADOOP_HOME/bin/hadoop
-  hadoop_conf_dir=$HADOOP_HOME/conf
-
-  ## hadoop_mr1_home=$HADOOP_MR1_HOME
-
   # Configuration for HDFS NameNode
   # Configuration for HDFS NameNode
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[hdfs_clusters]]
   [[hdfs_clusters]]
@@ -195,6 +184,18 @@
       # and explicitly set to the empty value.
       # and explicitly set to the empty value.
       ## webhdfs_url=
       ## webhdfs_url=
 
 
+      # Settings about this HDFS cluster. If you install HDFS in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_HDFS_HOME or /usr/lib/hadoop-hdfs
+      ## hadoop_hdfs_home=/usr/lib/hadoop-hdfs
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
   # Configuration for MapReduce JobTracker
   # Configuration for MapReduce JobTracker
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[mapred_clusters]]
   [[mapred_clusters]]
@@ -209,6 +210,18 @@
       # Whether to submit jobs to this cluster
       # Whether to submit jobs to this cluster
       ## submit_to=False
       ## submit_to=False
 
 
+      # Settings about this MR1 cluster. If you install MR1 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR1_HOME or /usr/lib/hadoop-0.20-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-0.20-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
   # Configuration for Yarn
   # Configuration for Yarn
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[yarn_clusters]]
   [[yarn_clusters]]
@@ -221,6 +234,18 @@
       # Whether to submit jobs to this cluster
       # Whether to submit jobs to this cluster
       ## submit_to=False
       ## submit_to=False
 
 
+      # Settings about this MR2 cluster. If you install MR2 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR2_HOME or /usr/lib/hadoop-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
 
 
 ###########################################################################
 ###########################################################################
 # Settings to configure Beeswax
 # Settings to configure Beeswax

+ 15 - 4
desktop/libs/hadoop/src/hadoop/cluster.py

@@ -91,21 +91,32 @@ def all_mrclusters():
     MR_CACHE[identifier] = _make_mrcluster(identifier)
     MR_CACHE[identifier] = _make_mrcluster(identifier)
   return MR_CACHE
   return MR_CACHE
 
 
-def get_cluster_for_job_submission():
+def get_cluster_conf_for_job_submission():
   """
   """
   Check the `submit_to' for each MR/Yarn cluster, and return the
   Check the `submit_to' for each MR/Yarn cluster, and return the
-  host:port of first one that enables submission.
+  config section of first one that enables submission.
   """
   """
   for name in conf.YARN_CLUSTERS.keys():
   for name in conf.YARN_CLUSTERS.keys():
     yarn = conf.YARN_CLUSTERS[name]
     yarn = conf.YARN_CLUSTERS[name]
     if yarn.SUBMIT_TO.get():
     if yarn.SUBMIT_TO.get():
-      return "%s:%s" % (yarn.RM_HOST.get(), yarn.RM_PORT.get())
+      return yarn
   for name in conf.MR_CLUSTERS.keys():
   for name in conf.MR_CLUSTERS.keys():
     mr = conf.MR_CLUSTERS[name]
     mr = conf.MR_CLUSTERS[name]
     if mr.SUBMIT_TO.get():
     if mr.SUBMIT_TO.get():
-      return "%s:%s" % (mr.JT_HOST.get(), mr.JT_PORT.get())
+      return mr
   return None
   return None
 
 
+def get_cluster_addr_for_job_submission():
+  """
+  Check the `submit_to' for each MR/Yarn cluster, and return the
+  host:port of first one that enables submission.
+  """
+  conf = get_cluster_conf_for_job_submission()
+  if conf is None:
+    return None
+  return "%s:%s" % (conf.HOST.get(), conf.PORT.get())
+
+
 def clear_caches():
 def clear_caches():
   """
   """
   Clears cluster's internal caches.  Returns
   Clears cluster's internal caches.  Returns

+ 96 - 88
desktop/libs/hadoop/src/hadoop/conf.py

@@ -22,46 +22,9 @@ import os
 
 
 DEFAULT_NN_HTTP_PORT = 50070
 DEFAULT_NN_HTTP_PORT = 50070
 
 
-HADOOP_HOME = Config(
-  key="hadoop_home",
-  default=os.environ.get("HADOOP_HOME", "/usr/lib/hadoop"),
-  help=("Path to directory holding hadoop libs - HADOOP_HOME in " +
-        "hadoop parlance; defaults to environment variable, when" +
-        "set.")
-)
-
-HADOOP_MR1_HOME = Config(
-  key="hadoop_mr1_home",
-  default=os.environ.get("HADOOP_MR1_HOME", "/usr/lib/hadoop-0.20-mapreduce"),
-  help=("Path to directory holding hadoop libs - HADOOP_HOME in " +
-        "hadoop parlance; defaults to environment variable, when" +
-        "set.")
-)
-
-def hadoop_bin_from_hadoop_home():
-  """Returns $HADOOP_HOME/bin/hadoop"""
-  return os.path.join(HADOOP_HOME.get(), "bin/hadoop")
-
-HADOOP_BIN = Config("hadoop_bin",
-  help="Path to your Hadoop binary",
-  dynamic_default=hadoop_bin_from_hadoop_home,
-  type=str)
-
-# TODO(philip): This will need more love for dealing with multiple clusters.
-HADOOP_CONF_DIR = Config(
-  key="hadoop_conf_dir",
-  default="/etc/hadoop/conf",
-  help="Directory to pass to hadoop_bin (from Hadoop configuration) as the --config flag.",
-)
-
-def find_file_recursive(desired_glob, root=None):
-  if root is None:
-    root_f = lambda: HADOOP_HOME.get()
-  else:
-    root_f = lambda: not callable(root) and root or root()
-
+def find_file_recursive(desired_glob, root):
   def f():
   def f():
-    for dirpath, dirnames, filenames in os.walk(root_f()):
+    for dirpath, dirnames, filenames in os.walk(root):
       matches = fnmatch.filter(filenames, desired_glob)
       matches = fnmatch.filter(filenames, desired_glob)
       if matches:
       if matches:
         if len(matches) != 1:
         if len(matches) != 1:
@@ -72,33 +35,9 @@ def find_file_recursive(desired_glob, root=None):
     logging.error("Trouble finding jars matching %s" % (desired_glob,))
     logging.error("Trouble finding jars matching %s" % (desired_glob,))
     return None
     return None
 
 
-  if root is None:
-    root_str = "$HADOOP_HOME"
-  else:
-    root_str = root
-  f.__doc__ = "Finds %s/%s" % (root_str, desired_glob)
+  f.__doc__ = "Finds %s/%s" % (root, desired_glob)
   return f
   return f
 
 
-HADOOP_EXAMPLES_JAR = Config(
-  key="hadoop_examples_jar",
-  dynamic_default=find_file_recursive("hadoop-*examples*.jar", lambda: HADOOP_MR1_HOME.get()),
-  help="Path to the hadoop-examples.jar (used for tests and jobdesigner setup)",
-  type=str,
-  private=True)
-
-HADOOP_STREAMING_JAR = Config(
-  key="hadoop_streaming_jar",
-  dynamic_default=find_file_recursive("hadoop-*streaming*.jar", lambda: HADOOP_MR1_HOME.get()),
-  help="Path to the hadoop-streaming.jar (used by jobdesigner)",
-  type=str,
-  private=True)
-
-HADOOP_TEST_JAR = Config("hadoop_test_jar",
-  help="[Used by testing code.] Path to hadoop-test.jar",
-  dynamic_default=find_file_recursive("hadoop-*test*.jar", lambda: HADOOP_MR1_HOME.get()),
-  type=str,
-  private=True)
-
 HADOOP_PLUGIN_CLASSPATH = Config("hadoop_plugin_classpath",
 HADOOP_PLUGIN_CLASSPATH = Config("hadoop_plugin_classpath",
   help="[Used only in testing code.] Path to the Hadoop plugin jar.",
   help="[Used only in testing code.] Path to the Hadoop plugin jar.",
   type=str,
   type=str,
@@ -149,6 +88,31 @@ HDFS_CLUSTERS = UnspecifiedConfigSection(
                               default=False, type=coerce_bool),
                               default=False, type=coerce_bool),
       TEMP_DIR=Config("temp_dir", help="HDFS directory for temporary files",
       TEMP_DIR=Config("temp_dir", help="HDFS directory for temporary files",
                       default='/tmp', type=str),
                       default='/tmp', type=str),
+
+      HADOOP_HDFS_HOME = Config(
+        key="hadoop_hdfs_home",
+        default=os.environ.get("HADOOP_HDFS_HOME", "/usr/lib/hadoop-hdfs"),
+        help=("Path to Hadoop HDFS home - HADOOP_HOME or HADOOP_HDFS_HOME in " +
+              "hadoop parlance. For tarball installations, it is the root of " +
+              "the untarred directory. For packages, " +
+              "it is /usr/lib/hadoop-hdfs." +
+              "Defaults to the environment varible HADOOP_BIN when set, " +
+              "or '/usr/bin/hadoop'."),
+      ),
+      HADOOP_BIN = Config(
+        key="hadoop_bin",
+        default=os.environ.get("HADOOP_BIN", "/usr/bin/hadoop"),
+        help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+              "Defaults to the environment varible HADOOP_BIN when set, " +
+              "or '/usr/bin/hadoop'.")
+      ),
+      HADOOP_CONF_DIR = Config(
+        key="hadoop_conf_dir",
+        default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+        help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+              "as the --config flag. Defaults to the environment variable " +
+              "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+      ),
     )
     )
   )
   )
 )
 )
@@ -159,11 +123,11 @@ MR_CLUSTERS = UnspecifiedConfigSection(
   each=ConfigSection(
   each=ConfigSection(
     help="Information about a single MapReduce cluster",
     help="Information about a single MapReduce cluster",
     members=dict(
     members=dict(
-      JT_HOST=Config("jobtracker_host", help="Host/IP for JobTracker"),
-      JT_PORT=Config("jobtracker_port",
-                     default=8021,
-                     help="Service port for the JobTracker",
-                     type=int),
+      HOST=Config("jobtracker_host", help="Host/IP for JobTracker"),
+      PORT=Config("jobtracker_port",
+                  default=8021,
+                  help="Service port for the JobTracker",
+                  type=int),
       JT_THRIFT_PORT=Config("thrift_port", help="Thrift port for JobTracker", default=9290,
       JT_THRIFT_PORT=Config("thrift_port", help="Thrift port for JobTracker", default=9290,
                             type=int),
                             type=int),
       JT_KERBEROS_PRINCIPAL=Config("jt_kerberos_principal", help="Kerberos principal for JobTracker",
       JT_KERBEROS_PRINCIPAL=Config("jt_kerberos_principal", help="Kerberos principal for JobTracker",
@@ -171,7 +135,29 @@ MR_CLUSTERS = UnspecifiedConfigSection(
       SECURITY_ENABLED=Config("security_enabled", help="Is running with Kerberos authentication",
       SECURITY_ENABLED=Config("security_enabled", help="Is running with Kerberos authentication",
                               default=False, type=coerce_bool),
                               default=False, type=coerce_bool),
       SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
       SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
-                       default=False, type=coerce_bool),
+                       default=True, type=coerce_bool),
+
+      HADOOP_MAPRED_HOME = Config(
+        key="hadoop_mapred_home",
+        default=os.environ.get("HADOOP_MR1_HOME", "/usr/lib/hadoop-0.20-mapreduce"),
+        help=("Path to directory holding Hadoop MR1 libs. " +
+              "E.g. /usr/lib/hadoop. Defaults to the environment variable " +
+              "HADOOP_MR1_HOME when set, or '/usr/lib/hadoop'.")
+      ),
+      HADOOP_BIN = Config(
+        key="hadoop_bin",
+        default=os.environ.get("HADOOP_MR1_BIN", "/usr/bin/hadoop"),
+        help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+              "Defaults to the environment varible HADOOP_MR1_BIN when set, " +
+              "or '/usr/bin/hadoop'.")
+      ),
+      HADOOP_CONF_DIR = Config(
+        key="hadoop_conf_dir",
+        default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+        help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+              "as the --config flag. Defaults to the environment variable " +
+              "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+      ),
     )
     )
   )
   )
 )
 )
@@ -183,15 +169,37 @@ YARN_CLUSTERS = UnspecifiedConfigSection(
   each=ConfigSection(
   each=ConfigSection(
     help="Information about a single Yarn cluster",
     help="Information about a single Yarn cluster",
     members=dict(
     members=dict(
-      RM_HOST=Config("resourcemanager_host",
-                     default='localhost',
-                     help="Host/IP for the ResourceManager"),
-      RM_PORT=Config("resourcemanager_port",
-                     default=8032,
-                     type=int,
-                     help="Service port for the ResourceManager"),
+      HOST=Config("resourcemanager_host",
+                  default='localhost',
+                  help="Host/IP for the ResourceManager"),
+      PORT=Config("resourcemanager_port",
+                  default=8032,
+                  type=int,
+                  help="Service port for the ResourceManager"),
       SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
       SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
                        default=False, type=coerce_bool),
                        default=False, type=coerce_bool),
+
+      HADOOP_MAPRED_HOME = Config(
+        key="hadoop_mapred_home",
+        default=os.environ.get("HADOOP_MR2_HOME", "/usr/lib/hadoop-mapreduce"),
+        help=("Path to directory holding Hadoop MR2 libs. " +
+              "E.g. /usr/lib/hadoop. Defaults to the environment " +
+              "variable HADOOP_MR2_HOME when set, or '/usr/lib/hadoop'.")
+      ),
+      HADOOP_BIN = Config(
+        key="hadoop_bin",
+        default=os.environ.get("HADOOP_MR2_BIN", "/usr/bin/hadoop"),
+        help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+              "Defaults to the environment varible HADOOP_MR2_BIN when set, " +
+              "or '/usr/bin/hadoop'.")
+      ),
+      HADOOP_CONF_DIR = Config(
+        key="hadoop_conf_dir",
+        default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+        help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+              "as the --config flag. Defaults to the environment variable " +
+              "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+      ),
     )
     )
   )
   )
 )
 )
@@ -206,22 +214,15 @@ def config_validator():
   from hadoop.fs import webhdfs
   from hadoop.fs import webhdfs
   from hadoop import job_tracker
   from hadoop import job_tracker
   res = [ ]
   res = [ ]
-
-  # HADOOP_HOME
-  res.extend(validate_path(HADOOP_HOME, is_dir=True))
-  # HADOOP_BIN
-  res.extend(validate_path(HADOOP_BIN, is_dir=False))
-
-  # JARs: even though these are private, we need them to run jobsub
-  res.extend(validate_path(HADOOP_EXAMPLES_JAR, is_dir=False))
-  res.extend(validate_path(HADOOP_STREAMING_JAR, is_dir=False))
-
   submit_to = [ ]
   submit_to = [ ]
 
 
   # HDFS_CLUSTERS
   # HDFS_CLUSTERS
   has_default = False
   has_default = False
   for name in HDFS_CLUSTERS.keys():
   for name in HDFS_CLUSTERS.keys():
     cluster = HDFS_CLUSTERS[name]
     cluster = HDFS_CLUSTERS[name]
+    res.extend(validate_path(cluster.HADOOP_HDFS_HOME, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
     res.extend(webhdfs.test_fs_configuration(cluster))
     res.extend(webhdfs.test_fs_configuration(cluster))
     if name == 'default':
     if name == 'default':
       has_default = True
       has_default = True
@@ -231,16 +232,23 @@ def config_validator():
   # MR_CLUSTERS
   # MR_CLUSTERS
   for name in MR_CLUSTERS.keys():
   for name in MR_CLUSTERS.keys():
     cluster = MR_CLUSTERS[name]
     cluster = MR_CLUSTERS[name]
+    res.extend(validate_path(cluster.HADOOP_MAPRED_HOME, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
     res.extend(job_tracker.test_jt_configuration(cluster))
     res.extend(job_tracker.test_jt_configuration(cluster))
     if cluster.SUBMIT_TO.get():
     if cluster.SUBMIT_TO.get():
       submit_to.append('mapred_clusters.' + name)
       submit_to.append('mapred_clusters.' + name)
 
 
-  # Only one cluster should have submit_to
+  # YARN_CLUSTERS
   for name in YARN_CLUSTERS.keys():
   for name in YARN_CLUSTERS.keys():
     cluster = YARN_CLUSTERS[name]
     cluster = YARN_CLUSTERS[name]
+    res.extend(validate_path(cluster.HADOOP_MAPRED_HOME, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
     if cluster.SUBMIT_TO.get():
     if cluster.SUBMIT_TO.get():
       submit_to.append('yarn_clusters.' + name)
       submit_to.append('yarn_clusters.' + name)
 
 
+  # Only one cluster should have submit_to
   if len(submit_to) > 1:
   if len(submit_to) > 1:
     res.append(("hadoop", "Only one cluster may enable 'submit_to'. "
     res.append(("hadoop", "Only one cluster may enable 'submit_to'. "
                 "But it is enabled in the following clusters: " + 
                 "But it is enabled in the following clusters: " + 

+ 2 - 2
desktop/libs/hadoop/src/hadoop/job_tracker.py

@@ -50,7 +50,7 @@ def test_jt_configuration(cluster):
     jt.runtime_info()
     jt.runtime_info()
   except TTransport.TTransportException:
   except TTransport.TTransportException:
     msg = 'Failed to contact JobTracker plugin at %s:%s.' % \
     msg = 'Failed to contact JobTracker plugin at %s:%s.' % \
-          (cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get())
+          (cluster.HOST.get(), cluster.JT_THRIFT_PORT.get())
     return [ (cluster, msg) ]
     return [ (cluster, msg) ]
   return []
   return []
 
 
@@ -84,7 +84,7 @@ class LiveJobTracker(object):
   @classmethod
   @classmethod
   def from_conf(cls, conf):
   def from_conf(cls, conf):
     return cls(
     return cls(
-      conf.JT_HOST.get(),
+      conf.HOST.get(),
       conf.JT_THRIFT_PORT.get(),
       conf.JT_THRIFT_PORT.get(),
       security_enabled=conf.SECURITY_ENABLED.get(),
       security_enabled=conf.SECURITY_ENABLED.get(),
       kerberos_principal=conf.JT_KERBEROS_PRINCIPAL.get())
       kerberos_principal=conf.JT_KERBEROS_PRINCIPAL.get())

+ 11 - 2
desktop/libs/hadoop/src/hadoop/mini_cluster.py

@@ -14,7 +14,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # limitations under the License.
-#
+
+
+#######################################################
+##               WARNING!!!                          ##
+##   This file is stale. Hadoop 0.23 and CDH4        ##
+##   do not support minicluster. This is replaced    ##
+##   by webhdfs.py, to set up a running cluster.     ##
+#######################################################
+
+
 # A Python-side driver for MiniHadoopClusterManager
 # A Python-side driver for MiniHadoopClusterManager
 # 
 # 
 # See README.testing for hints on how to use this,
 # See README.testing for hints on how to use this,
@@ -413,7 +422,7 @@ def shared_cluster(conf=False):
     closers.extend([
     closers.extend([
       hadoop.conf.HDFS_CLUSTERS["default"].NN_HOST.set_for_testing("localhost"),
       hadoop.conf.HDFS_CLUSTERS["default"].NN_HOST.set_for_testing("localhost"),
       hadoop.conf.HDFS_CLUSTERS["default"].NN_HDFS_PORT.set_for_testing(cluster.namenode_port),
       hadoop.conf.HDFS_CLUSTERS["default"].NN_HDFS_PORT.set_for_testing(cluster.namenode_port),
-      hadoop.conf.MR_CLUSTERS["default"].JT_HOST.set_for_testing("localhost"),
+      hadoop.conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
       hadoop.conf.MR_CLUSTERS["default"].JT_THRIFT_PORT.set_for_testing(cluster.jt.thrift_port),
       hadoop.conf.MR_CLUSTERS["default"].JT_THRIFT_PORT.set_for_testing(cluster.jt.thrift_port),
     ])
     ])
     # Clear the caches
     # Clear the caches

+ 10 - 18
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -199,7 +199,8 @@ class PseudoHdfs4(object):
 
 
     # More stuff to setup in the environment
     # More stuff to setup in the environment
     env = dict(
     env = dict(
-      HADOOP_HOME = hadoop.conf.HADOOP_HOME.get(),
+      HADOOP_HOME = hadoop.conf.HDFS_CLUSTERS['default'].HADOOP_HDFS_HOME.get(),
+      HADOOP_BIN = hadoop.conf.HDFS_CLUSTERS['default'].HADOOP_BIN.get(),
       HADOOP_CONF_DIR = conf_dir,
       HADOOP_CONF_DIR = conf_dir,
       HADOOP_HEAPSIZE = "128",
       HADOOP_HEAPSIZE = "128",
       HADOOP_LOG_DIR = self._log_dir,
       HADOOP_LOG_DIR = self._log_dir,
@@ -237,21 +238,9 @@ class PseudoHdfs4(object):
 
 
     # We need a different env because it's a different hadoop
     # We need a different env because it's a different hadoop
     env = env.copy()
     env = env.copy()
-    env['HADOOP_HOME'] = hadoop.conf.HADOOP_MR1_HOME.get()
+    env['HADOOP_HOME'] = hadoop.conf.MR_CLUSTERS['default'].HADOOP_MAPRED_HOME.get()
+    env['HADOOP_BIN'] = hadoop.conf.MR_CLUSTERS['default'].HADOOP_BIN.get()
     env["HADOOP_CLASSPATH"] = ':'.join([
     env["HADOOP_CLASSPATH"] = ':'.join([
-        # -- BEGIN JAVA TRIVIA --
-        # Add the -test- jar to the classpath to work around a subtle issue
-        # involving Java classloaders. In brief, hadoop's RunJar class creates
-        # a child classloader with the test jar on it, but the core classes
-        # are loaded by the system classloader. This is fine except that
-        # some classes in the test jar extend package-protected classes in the
-        # core jar. Even though the classes are in the same package name, they
-        # are thus loaded by different classloaders and therefore an IllegalAccessError
-        # prevents the MiniMRCluster from starting. Adding the test jar to the system
-        # classpath prevents this error since then both the MiniMRCluster and the
-        # core classes are loaded by the system classloader.
-        hadoop.conf.HADOOP_TEST_JAR.get(),
-        # -- END JAVA TRIVIA --
         hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get(),
         hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get(),
         # Due to CDH-4537, we need to add test dependencies to run minicluster
         # Due to CDH-4537, we need to add test dependencies to run minicluster
         os.path.join(os.path.dirname(__file__), 'test_jars', '*'),
         os.path.join(os.path.dirname(__file__), 'test_jars', '*'),
@@ -276,7 +265,7 @@ class PseudoHdfs4(object):
 
 
   def _format(self, conf_dir, env):
   def _format(self, conf_dir, env):
     """Format HDFS"""
     """Format HDFS"""
-    args = (hadoop.conf.HADOOP_BIN.get(), 
+    args = (self._get_hadoop_bin(env),
             '--config', conf_dir,
             '--config', conf_dir,
             'namenode', '-format')
             'namenode', '-format')
     LOG.info('Formatting HDFS: %s' % (args,))
     LOG.info('Formatting HDFS: %s' % (args,))
@@ -377,7 +366,10 @@ class PseudoHdfs4(object):
       env=env)
       env=env)
 
 
   def _get_hadoop_bin(self, env):
   def _get_hadoop_bin(self, env):
-    return os.path.join(env['HADOOP_HOME'], 'bin', 'hadoop')
+    try:
+      return env['HADOOP_BIN']
+    except KeyError:
+      return os.path.join(env['HADOOP_HOME'], 'bin', 'hadoop')
 
 
   def _write_hdfs_site(self):
   def _write_hdfs_site(self):
     self._dfs_http_port = find_unused_port()
     self._dfs_http_port = find_unused_port()
@@ -466,7 +458,7 @@ def shared_cluster():
     closers = [
     closers = [
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
-      hadoop.conf.MR_CLUSTERS['default'].JT_HOST.set_for_testing('localhost'),
+      hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
     ]
     ]
 
 

+ 1 - 4
desktop/libs/hadoop/src/hadoop/tests.py

@@ -112,7 +112,6 @@ def test_tricky_confparse():
 
 
 def test_config_validator_basic():
 def test_config_validator_basic():
   reset = (
   reset = (
-    conf.HADOOP_STREAMING_JAR.set_for_testing('/tmp'),
     conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing('http://not.the.re:50070/'),
     conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing('http://not.the.re:50070/'),
     conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(70000),
     conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(70000),
   )
   )
@@ -120,8 +119,6 @@ def test_config_validator_basic():
   try:
   try:
     cli = make_logged_in_client()
     cli = make_logged_in_client()
     resp = cli.get('/debug/check_config')
     resp = cli.get('/debug/check_config')
-    assert_true('hadoop.hadoop_streaming_jar' in resp.content)
-    assert_true('Not a file' in resp.content)
     assert_true('hadoop.hdfs_clusters.default.webhdfs_url' in resp.content)
     assert_true('hadoop.hdfs_clusters.default.webhdfs_url' in resp.content)
     assert_true('hadoop.mapred_clusters.default.thrift_port' in resp.content)
     assert_true('hadoop.mapred_clusters.default.thrift_port' in resp.content)
     assert_true('Port should be' in resp.content)
     assert_true('Port should be' in resp.content)
@@ -141,7 +138,7 @@ def test_config_validator_more():
   cli = make_logged_in_client()
   cli = make_logged_in_client()
 
 
   reset = (
   reset = (
-    conf.MR_CLUSTERS["default"].JT_HOST.set_for_testing("localhost"),
+    conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
     conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(23),
     conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(23),
   )
   )
   old = cluster.clear_caches()
   old = cluster.clear_caches()

+ 6 - 14
dist/README

@@ -5,26 +5,18 @@ If you're impatient, these are the key steps.  Please check the full manual
 for more details.
 for more details.
 
 
 ## Install
 ## Install
-$ HADOOP_HOME=/usr/lib/hadoop-0.20 PREFIX=/usr/local make install
+$ PREFIX=/usr/share make install
 
 
 ## Install plug-ins
 ## Install plug-ins
-$ cd /usr/lib/hadoop/lib
-$ ln -s /usr/local/hue/desktop/libs/hadoop/java-lib/hue*jar
+$ cd /usr/lib/hadoop-0.20-mapreduce/lib
+$ ln -s /usr/share/hue/desktop/libs/hadoop/java-lib/hue*jar
 
 
 ## Configure Hadoop
 ## Configure Hadoop
 Edit hdfs-site.xml:
 Edit hdfs-site.xml:
 
 
 <property>
 <property>
-  <name>dfs.namenode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
-  <description>Comma-separated list of namenode plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>dfs.datanode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
-  <description>Comma-separated list of datanode plug-ins to be activated.
-  </description>
+  <name>dfs.webhdfs.enable</name>
+  <value>true</value>
 </property>
 </property>
 
 
 Edit mapred-site.xml:
 Edit mapred-site.xml:
@@ -37,4 +29,4 @@ Edit mapred-site.xml:
 </property>
 </property>
 
 
 ## Run!
 ## Run!
-$ /usr/local/hue/build/env/bin/supervisor
+$ /usr/share/hue/build/env/bin/supervisor

+ 46 - 14
docs/manual.txt

@@ -412,19 +412,6 @@ Hue Configuration for Hadoop
 These configuration variables are under the `[hadoop]` section in
 These configuration variables are under the `[hadoop]` section in
 the `/etc/hue/hue.ini` configuration file.
 the `/etc/hue/hue.ini` configuration file.
 
 
-hadoop_home::
-  This becomes the value of `$HADOOP_HOME` for any Shell processes
-  and the Beeswax Server. If you use MR1, set this to the MR1 home.
-
-hadoop_bin::
-  Use this as the hadoop binary. If you use MR1, set this to
-  `<hadoop_home>/bin/hadoop` instead of the default `/usr/bin/hadoop`.
-
-hadoop_conf_dir::
-  This is the configuration directory for any processes to
-  configure their Hadoop client. If you use MR1, set this to the
-  directory containing your MR1 configuration.
-
 
 
 HDFS Cluster
 HDFS Cluster
 ^^^^^^^^^^^^
 ^^^^^^^^^^^^
@@ -433,13 +420,26 @@ Hue only support one HDFS cluster currently. That cluster should be defined
 under the `[[[default]]]` sub-section.
 under the `[[[default]]]` sub-section.
 
 
 fs_defaultfs::
 fs_defaultfs::
-  This is the equivalence of ``fs.defaultFS`` (aka ``fs.default.name``) in
+  This is the equivalence of `fs.defaultFS` (aka `fs.default.name`) in
   Hadoop configuration.
   Hadoop configuration.
 
 
 webhdfs_url::
 webhdfs_url::
   You can also set this to be the HttpFS url. The default value is the HTTP
   You can also set this to be the HttpFS url. The default value is the HTTP
   port on the NameNode.
   port on the NameNode.
 
 
+hadoop_hdfs_home::
+  This is the home of your Hadoop HDFS installation. It is the
+  root of the Hadoop untarred directory, or usually
+  `/usr/lib/hadoop`.
+
+hadoop_bin::
+  Use this as the HDFS Hadoop launcher script, which is usually
+  `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+  This is the configuration directory of the HDFS, typically
+  `/etc/hadoop/conf`.
+
 
 
 MapReduce (MR1) Cluster
 MapReduce (MR1) Cluster
 ^^^^^^^^^^^^^^^^^^^^^^^
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -457,6 +457,22 @@ submit_to::
   If your Oozie is configured with to talk to a 0.20 MapReduce service, then
   If your Oozie is configured with to talk to a 0.20 MapReduce service, then
   set this to `true`. Hue will be submitting jobs to this MapReduce cluster.
   set this to `true`. Hue will be submitting jobs to this MapReduce cluster.
 
 
+hadoop_mapred_home::
+  This is the home of your Hadoop MapReduce installation. It is the
+  root of the Hadoop MR1 untarred directory, or the root of the
+  Hadoop 0.20 untarred directory, or `/usr/lib/hadoop-0.20-mapreduce` for
+  CDH packages. If `submit_to` is true for this cluster, this
+  config value becomes the `$HADOOP_MAPRED_HOME` for
+  BeeswaxServer and child shell processes.
+
+hadoop_bin::
+  Use this as the MR1 Hadoop launcher script, which is usually
+  `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+  This is the configuration directory of the MR1 service,
+  typically `/etc/hadoop/conf`.
+
 
 
 Yarn (MR2) Cluster
 Yarn (MR2) Cluster
 ^^^^^^^^^^^^^^^^^^
 ^^^^^^^^^^^^^^^^^^
@@ -475,6 +491,22 @@ submit_to::
   set this to `true`. Hue will be submitting jobs to this Yarn cluster.
   set this to `true`. Hue will be submitting jobs to this Yarn cluster.
   But note that JobBrowser will not be able to show MR2 jobs.
   But note that JobBrowser will not be able to show MR2 jobs.
 
 
+hadoop_mapred_home::
+  This is the home of your Hadoop MapReduce installation. It is the
+  root of the Hadoop 0.23 untarred directory, or
+  `/usr/lib/hadoop-mapreduce` for CDH packages. If `submit_to` is
+  true for this cluster, this config value becomes the
+  `$HADOOP_MAPRED_HOME` for BeeswaxServer and child shell
+  processes.
+
+hadoop_bin::
+  Use this as the Yarn/MR2 Hadoop launcher script, which is usually
+  `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+  This is the configuration directory of the Yarn/MR2 service,
+  typically `/etc/hadoop/conf`.
+
 
 
 Beeswax Configuration
 Beeswax Configuration
 ~~~~~~~~~~~~~~~~~~~~~
 ~~~~~~~~~~~~~~~~~~~~~

+ 3 - 1
tools/jenkins/build-functions

@@ -35,7 +35,8 @@ build_hadoop() {
   fi
   fi
 
 
   HADOOP_DIR=$HUE_ROOT/ext/hadoop
   HADOOP_DIR=$HUE_ROOT/ext/hadoop
-  export HADOOP_HOME="$HADOOP_DIR/${CDH_VERSION}"
+  export HADOOP_HDFS_HOME="$HADOOP_DIR/${CDH_VERSION}"
+  export HADOOP_BIN=$HADOOP_HDFS_HOME/bin/hadoop
 
 
   mkdir -p $HADOOP_DIR
   mkdir -p $HADOOP_DIR
   rm -rf $HADOOP_HOME
   rm -rf $HADOOP_HOME
@@ -61,6 +62,7 @@ build_mr1() {
 
 
   MR1_DIR=$HUE_ROOT/ext/mr1
   MR1_DIR=$HUE_ROOT/ext/mr1
   export HADOOP_MR1_HOME="$MR1_DIR/${MR1_VERSION}"
   export HADOOP_MR1_HOME="$MR1_DIR/${MR1_VERSION}"
+  export HADOOP_MR1_BIN="$HADOOP_MR1_HOME/bin/hadoop"
 
 
   mkdir -p $MR1_DIR
   mkdir -p $MR1_DIR
   rm -rf $HADOOP_MR1_HOME
   rm -rf $HADOOP_MR1_HOME

+ 0 - 267
tools/scripts/configure-hadoop.sh

@@ -1,267 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-
-if test -z $HADOOP_HOME ; then
-  echo "Please define your HADOOP_HOME environment variable."
-  exit
-fi
-
-SCRIPT_DIR=$(dirname $0)
-GIT_ROOT=$(cd $SCRIPT_DIR && pwd)/$(cd $SCRIPT_DIR && git rev-parse --show-cdup)
-BIND_IP=${BIND_IP:-localhost}
-HADOOP_TMP_DIR=$HADOOP_HOME/data
-
-if [ $(uname) == "Darwin" ]; then
-  export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/1.6.0/Home
-fi
-SLAVE_IPS=${SLAVE_IPS:-127.0.0.1}
-
-echo -e ".======================== Parameters ========================\n"\
-        "HADOOP_HOME    : $HADOOP_HOME\n"\
-	"GIT_ROOT       : $GIT_ROOT\n"\
-	"HADOOP_TMP_DIR : $HADOOP_TMP_DIR\n"\
-	"============================================================\n"
-set -x
-
-
-# Configure a slave
-# Used by the sandboxer
-function write_hadoop_config() {
-  SLAVE_IP=$1
-  MASTER_IP=$2
-  TARGET_DIR=$3
-  TMP_DIR=$4
-  if [ -z "$SLAVE_IP" -o -z "$MASTER_IP" -o -z "$TARGET_DIR" -o -z "$TMP_DIR" ]; then
-    echo usage: $0 slave_ip master_ip target_conf_dir tmp_dir
-    exit 1
-  fi
-
-  mkdir -p $TARGET_DIR
-  cp $HADOOP_HOME/example-confs/conf.pseudo/hadoop-metrics.properties $TARGET_DIR
-  cat > $TARGET_DIR/core-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>fs.default.name</name>
-  <value>hdfs://${MASTER_IP}:8020</value>
-</property>
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>$TMP_DIR</value>
-  <description>A base for other temporary directories.</description>
-</property>
-<property>
-  <name>webinterface.private.actions</name>
-  <value>true</value>
-</property>
-<property>
-  <name>slave.host.name</name>
-  <value>${SLAVE_IP}</value>
-</property>
-<property>
-  <name>dfs.thrift.address</name>
-  <value>${MASTER_IP}:10090</value>
-</property>
-<property>
-  <name>jobtracker.thrift.address</name>
-  <value>${MASTER_IP}:9290</value>
-</property>
-</configuration>
-END
-  cat > $TARGET_DIR/hdfs-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>dfs.datanode.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>dfs.datanode.ipc.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>dfs.datanode.http.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>dfs.http.address</name>
-  <value>${MASTER_IP}:50070</value>
-</property>
-<property>
-  <name>dfs.namenode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
-  <description>Comma-separated list of namenode plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>dfs.datanode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
-  <description>Comma-separated list of datanode plug-ins to be activated.
-  </description>
-</property>
-<!-- we dont really care about being super safe -->
-<property>
-  <name>dfs.safemode.min.datanodes</name>
-  <value>1</value>
-</property>
-<property>
-  <name>dfs.safemode.extension</name>
-  <value>5000</value>
-  <description>
-    Determines extension of safe mode in milliseconds
-    after the threshold level is reached.
-  </description>
-</property>
-</configuration>
-END
-  cat > $TARGET_DIR/mapred-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>mapred.job.tracker</name>
-  <value>${MASTER_IP}:8021</value>
-</property>
-<property>
-  <name>mapred.job.tracker.http.address</name>
-  <value>${MASTER_IP}:50030</value>
-</property>
-<property>
-  <name>mapred.task.tracker.http.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>mapred.jobtracker.plugins</name>
-  <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
-  <description>Comma-separated list of jobtracker plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>mapred.system.dir</name>
-  <value>/hadoop/mapred/system</value>
-</property>
-<property>
-  <name>mapred.local.dir</name>
-  <value>$TMP_DIR/mapred/local</value>
-</property>
-</configuration>
-END
-
-}
-
-# Configure
-function configure() {
-  perl -p -i -e "s,localhost,$BIND_IP,g" desktop/conf/pseudo-distributed.ini
-  mkdir -p $HADOOP_TMP_DIR
-  write_hadoop_config $BIND_IP $BIND_IP $HADOOP_HOME/conf $HADOOP_TMP_DIR
-  idx=0
-  for slave in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    datadir=$HADOOP_TMP_DIR-slave-$idx
-    write_hadoop_config $slave $BIND_IP $HADOOP_HOME/conf-slave-$idx $datadir
-  done
-}
-
-function start() {
-  pushd $HADOOP_HOME
-  export HADOOP_CLASSPATH=$GIT_ROOT/desktop/libs/hadoop/java-lib/\*
-  if [ ! -d $HADOOP_TMP_DIR/dfs/name ]; then
-    bin/hadoop namenode -format
-  fi
-  # Pass HADOOP_OPTS=$JDB_ON to any hadoop-daemon.sh to enable jdb
-  JDB_ON="-Xdebug -Xrunjdwp:transport=dt_socket,address=8901,server=y,suspend=n"
-
-  HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-    bin/hadoop-daemon.sh start namenode
-  idx=0
-  for ip in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-      HADOOP_LOG_DIR="$HADOOP_HOME/logs-slave-$idx" \
-      bin/hadoop-daemon.sh start datanode || true
-  done
-  bin/hadoop dfsadmin -safemode wait
-  HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-    bin/hadoop-daemon.sh start secondarynamenode
-  HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-    bin/hadoop-daemon.sh start jobtracker
-  idx=0
-  for ip in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-      HADOOP_LOG_DIR="$HADOOP_HOME/logs-slave-$idx" \
-      bin/hadoop-daemon.sh start tasktracker || true
-  done
-  popd
-}
-
-function stop() {
-  pushd $HADOOP_HOME
-  for daemon in namenode jobtracker; do
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-      bin/hadoop-daemon.sh stop $daemon
-  done
-
-  idx=0
-  for ip in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-      bin/hadoop-daemon.sh --config $HADOOP_HOME/conf-slave-$idx stop tasktracker || true
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-       bin/hadoop-daemon.sh --config $HADOOP_HOME/conf-slave-$idx stop datanode || true
-  done
-
-  popd
-}
-
-
-pushd $GIT_ROOT
-
-if [ ! -d .git ]; then
-  echo "This script moves into your git root, but this has failed."
-  exit 1
-fi
-
-if [ $# -eq 0 ]; then
-  echo "Usage: $0 (all|checkout|build|build_jobsub|build_plugins|configure|start)"
-  exit 1
-fi
-
-ARG=$1
-shift 1
-case $ARG
-in
-  configure|start|write_hadoop_config|stop)
-    $ARG $@
-    ;;
-  all)
-    configure
-    start
-    ;;
-  *)
-    echo "Unrecognized: $ARG"
-    exit 1
-  ;;
-esac