Jelajahi Sumber

HUE-662. [config] Each Hadoop hdfs/mr/yarn cluster to define its own env

* Have Beeswax and Shell define the appropriate HADOOP_MAPRED_HOME for the
  child process.
* Renamed JT_HOST & RM_HOST to just "HOST", and JT_PORT & RM_PORT to just
  "PORT" for consistent reference.
* Fixed README to drop the requirement on having a HADOOP_HOME. The build
  does not depend on one because we're using maven.
bc Wong 13 tahun lalu
induk
melakukan
c7e29b51f7

+ 1 - 1
README.testing

@@ -84,7 +84,7 @@ DESKTOP_DB_CONFIG=<db engine:db name:test db name:username:password:host:port>
 Writing tests that depend on Hadoop
 ===================================
 
-Use mini_cluster.py!  You should tag such tests with "requires_hadoop", as follows:
+Use pseudo_hdfs4.py!  You should tag such tests with "requires_hadoop", as follows:
 
   from nose.plugins.attrib import attr
 

+ 4 - 7
apps/beeswax/beeswax_server.sh

@@ -19,8 +19,8 @@
 
 set -o errexit
 
-if [ -z "$HADOOP_HOME" ]; then
-  echo "\$HADOOP_HOME must be specified" 1>&2
+if [ -z "$HADOOP_CONF_DIR" ]; then
+  echo "\$HADOOP_CONF_DIR must be specified" 1>&2
   exit 1
 fi
 echo \$HADOOP_HOME=$HADOOP_HOME
@@ -46,7 +46,6 @@ fi
 
 echo \$HIVE_HOME=$HIVE_HOME
 
-
 BEESWAX_ROOT=$(dirname $0)
 BEESWAX_JAR=$BEESWAX_ROOT/java-lib/BeeswaxServer.jar
 HIVE_LIB=$HIVE_HOME/lib
@@ -65,15 +64,13 @@ echo \$HADOOP_OPTS=$HADOOP_OPTS
 # and to force hive-default to correspond to the Hive version we have.
 # Because we are abusing HADOOP_CONF_DIR, we have to emulate its default
 # behavior here as well.
-if [ -z "$HADOOP_CONF_DIR" ]; then
-  HADOOP_CONF_DIR="$HADOOP_HOME/conf"
-fi
 if [ -f $HADOOP_CONF_DIR/hadoop-env.sh ]; then
   . $HADOOP_CONF_DIR/hadoop-env.sh
 fi
 
-export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$BEESWAX_ROOT/../../desktop/conf:$HADOOP_CONF_DIR
+export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$HADOOP_CONF_DIR
 echo \$HADOOP_CONF_DIR=$HADOOP_CONF_DIR
+echo \$HADOOP_MAPRED_HOME=$HADOOP_MAPRED_HOME
 
 # Note: I've had trouble running this with just "java -jar" with the classpath
 # determined with a seemingly appropriate find command.

+ 22 - 12
apps/beeswax/src/beeswax/management/commands/beeswax_server.py

@@ -22,26 +22,36 @@ from django.core.management.base import NoArgsCommand
 import beeswax.conf
 import beeswax.hive_site
 import desktop.conf
-import hadoop.conf
-import os
+import hadoop.cluster
+
 import logging
+import os
+import sys
 
 LOG = logging.getLogger(__name__)
 
 class Command(NoArgsCommand):
   """ Starts beeswax daemon.  """
+
   def handle_noargs(self, **options):
+    cluster_conf = hadoop.cluster.get_cluster_conf_for_job_submission()
+    if cluster_conf is None:
+      LOG.error("Configuration does not contain any MR/Yarn clusters with "
+                "`submit_to' enabled. Cannot start BeeswaxServer.")
+      sys.exit(1)
+
     env = os.environ.copy()
-    env['HADOOP_HOME'] = hadoop.conf.HADOOP_HOME.get()
-    env['HADOOP_BIN'] = hadoop.conf.HADOOP_BIN.get()
-    if hadoop.conf.HADOOP_CONF_DIR.get():
-      env['HADOOP_CONF_DIR'] = hadoop.conf.HADOOP_CONF_DIR.get()
-    if beeswax.conf.BEESWAX_HIVE_HOME_DIR.get():
-      env['HIVE_HOME'] = beeswax.conf.BEESWAX_HIVE_HOME_DIR.get()
-    if beeswax.conf.BEESWAX_HIVE_CONF_DIR.get():
-      env['HIVE_CONF_DIR'] = beeswax.conf.BEESWAX_HIVE_CONF_DIR.get()
-    if beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get():
-      env['HADOOP_HEAPSIZE'] = beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get()
+    def set_if_present(name, val):
+      if val:
+        env[name] = val
+
+    env['HADOOP_BIN'] = cluster_conf.HADOOP_BIN.get()
+    set_if_present('HADOOP_MAPRED_HOME', cluster_conf.HADOOP_MAPRED_HOME.get())
+    set_if_present('HADOOP_CONF_DIR', cluster_conf.HADOOP_CONF_DIR.get())
+    set_if_present('HADOOP_HEAPSIZE', beeswax.conf.BEESWAX_SERVER_HEAPSIZE.get())
+    set_if_present('HIVE_HOME', beeswax.conf.BEESWAX_HIVE_HOME_DIR.get())
+    set_if_present('HIVE_CONF_DIR', beeswax.conf.BEESWAX_HIVE_CONF_DIR.get())
+
     bin = beeswax.conf.BEESWAX_SERVER_BIN.get()
 
     # Host that desktop is running on

+ 1 - 1
apps/jobsub/src/jobsub/submit.py

@@ -69,7 +69,7 @@ class Submission(object):
       raise Exception("Job design already submitted (Oozie job id %s)" % (self.job_id,))
 
     fs_defaultfs = self._fs.fs_defaultfs
-    jobtracker = hadoop.cluster.get_cluster_for_job_submission()
+    jobtracker = hadoop.cluster.get_cluster_addr_for_job_submission()
 
     try:
       wf_dir = self._get_and_create_deployment_dir()

+ 1 - 0
apps/shell/src/shell/constants.py

@@ -49,6 +49,7 @@ NO_SUCH_USER = "noSuchUser"
 SHELL_NOT_ALLOWED = "shellNotAllowed"
 HOME = "HOME"
 HADOOP_HOME = "HADOOP_HOME"
+HADOOP_MAPRED_HOME = "HADOOP_MAPRED_HOME"
 HADOOP_TOKEN_FILE_LOCATION = 'HADOOP_TOKEN_FILE_LOCATION'
 EXISTS = "exists"
 

+ 10 - 3
apps/shell/src/shell/shellmanager.py

@@ -21,7 +21,6 @@ Shell class itself, but a lot also happens in ShellManager.
 
 
 import cStringIO
-import desktop.lib.i18n
 import errno
 import eventlet
 import hadoop.conf
@@ -39,7 +38,8 @@ import tty
 from eventlet.green import os
 from eventlet.green import select
 from eventlet.green import time
-from hadoop.cluster import all_mrclusters, get_all_hdfs
+from hadoop.cluster import all_mrclusters, get_all_hdfs, \
+                           get_cluster_conf_for_job_submission
 
 LOG = logging.getLogger(__name__)
 SHELL_OUTPUT_LOGGER = logging.getLogger("shell_output")
@@ -388,6 +388,7 @@ class ShellManager(object):
     self._parse_configs()
     eventlet.spawn_after(1, self._handle_periodic)
 
+
   @classmethod
   def global_instance(cls):
     if not hasattr(cls, "_global_instance"):
@@ -395,9 +396,15 @@ class ShellManager(object):
     return cls._global_instance
 
   def _parse_configs(self):
+    mr_cluster = get_cluster_conf_for_job_submission()
+    if mr_cluster is None:
+      mapred_home = None
+    else:
+      mapred_home = mr_cluster.HADOOP_MAPRED_HOME.get()
+
     shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
     for item in shell.conf.SHELL_TYPES.keys():
-      env_for_shell = { constants.HADOOP_HOME: hadoop.conf.HADOOP_HOME.get() }
+      env_for_shell = { constants.HADOOP_MAPRED_HOME: mapred_home }
       command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
       nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
       executable_exists = utils.executable_exists(command)

+ 39 - 13
desktop/conf.dist/hue.ini

@@ -165,15 +165,6 @@
 
 [hadoop]
 
-  # If you installed Hadoop in a different location, you need to set
-  # hadoop_home, in which bin/hadoop, the Hadoop wrapper script, is found.
-  #
-  # NOTE: Hue depends on Cloudera's Distribution of Hadoop version 4 (CDH4)
-  # or later.
-  hadoop_home=/usr/lib/hadoop
-  hadoop_bin=/usr/bin/hadoop
-  # hadoop_conf_dir=/etc/hadoop/conf
-
   # Configuration for HDFS NameNode
   # ------------------------------------------------------------------------
   [[hdfs_clusters]]
@@ -185,11 +176,23 @@
       # Change this if your HDFS cluster is Kerberos-secured
       ## security_enabled=false
 
-      # Use WebHdfs/HttpFs as the communication mechanism. To fallback to
-      # using the Thrift plugin (used in Hue 1.x), this must be uncommented
-      # and explicitly set to the empty value.
+      # Use WebHdfs/HttpFs as the communication mechanism.
+      # This should be the web service root URL, such as
+      # http://namenode:50070/webhdfs/v1
       ## webhdfs_url=
 
+      # Settings about this HDFS cluster. If you install HDFS in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_HDFS_HOME or /usr/lib/hadoop-hdfs
+      ## hadoop_hdfs_home=/usr/lib/hadoop-hdfs
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
   # Configuration for MapReduce 0.20 JobTracker (MR1)
   # ------------------------------------------------------------------------
   [[mapred_clusters]]
@@ -202,11 +205,22 @@
       # Thrift plug-in port for the JobTracker
       ## thrift_port=9290
       # Whether to submit jobs to this cluster
-      ## submit_to=False
+      ## submit_to=True
 
       # Change this if your MapReduce cluster is Kerberos-secured
       ## security_enabled=false
 
+      # Settings about this MR1 cluster. If you install MR1 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR1_HOME or /usr/lib/hadoop-0.20-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-0.20-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
 
   # Configuration for Yarn (MR2)
   # ------------------------------------------------------------------------
@@ -220,6 +234,18 @@
       # Whether to submit jobs to this cluster
       ## submit_to=False
 
+      # Settings about this MR2 cluster. If you install MR2 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR2_HOME or /usr/lib/hadoop-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
 
 ###########################################################################
 # Settings to configure Beeswax

+ 36 - 11
desktop/conf/pseudo-distributed.ini.tmpl

@@ -171,17 +171,6 @@
 
 [hadoop]
 
-  # If you installed Hadoop in a different location, you need to set
-  # hadoop_home, in which bin/hadoop, the Hadoop wrapper script, is found.
-  #
-  # NOTE: Hue depends on Cloudera's Distribution of Hadoop version 4 (CDH4)
-  # or later.
-  hadoop_home=$HADOOP_HOME
-  hadoop_bin=$HADOOP_HOME/bin/hadoop
-  hadoop_conf_dir=$HADOOP_HOME/conf
-
-  ## hadoop_mr1_home=$HADOOP_MR1_HOME
-
   # Configuration for HDFS NameNode
   # ------------------------------------------------------------------------
   [[hdfs_clusters]]
@@ -195,6 +184,18 @@
       # and explicitly set to the empty value.
       ## webhdfs_url=
 
+      # Settings about this HDFS cluster. If you install HDFS in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_HDFS_HOME or /usr/lib/hadoop-hdfs
+      ## hadoop_hdfs_home=/usr/lib/hadoop-hdfs
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
   # Configuration for MapReduce JobTracker
   # ------------------------------------------------------------------------
   [[mapred_clusters]]
@@ -209,6 +210,18 @@
       # Whether to submit jobs to this cluster
       ## submit_to=False
 
+      # Settings about this MR1 cluster. If you install MR1 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR1_HOME or /usr/lib/hadoop-0.20-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-0.20-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
   # Configuration for Yarn
   # ------------------------------------------------------------------------
   [[yarn_clusters]]
@@ -221,6 +234,18 @@
       # Whether to submit jobs to this cluster
       ## submit_to=False
 
+      # Settings about this MR2 cluster. If you install MR2 in a
+      # different location, you need to set the following.
+
+      # Defaults to $HADOOP_MR2_HOME or /usr/lib/hadoop-mapreduce
+      ## hadoop_mapred_home=/usr/lib/hadoop-mapreduce
+
+      # Defaults to $HADOOP_BIN or /usr/bin/hadoop
+      ## hadoop_bin=/usr/bin/hadoop
+
+      # Defaults to $HADOOP_CONF_DIR or /etc/hadoop/conf
+      ## hadoop_conf_dir=/etc/hadoop/conf
+
 
 ###########################################################################
 # Settings to configure Beeswax

+ 15 - 4
desktop/libs/hadoop/src/hadoop/cluster.py

@@ -91,21 +91,32 @@ def all_mrclusters():
     MR_CACHE[identifier] = _make_mrcluster(identifier)
   return MR_CACHE
 
-def get_cluster_for_job_submission():
+def get_cluster_conf_for_job_submission():
   """
   Check the `submit_to' for each MR/Yarn cluster, and return the
-  host:port of first one that enables submission.
+  config section of first one that enables submission.
   """
   for name in conf.YARN_CLUSTERS.keys():
     yarn = conf.YARN_CLUSTERS[name]
     if yarn.SUBMIT_TO.get():
-      return "%s:%s" % (yarn.RM_HOST.get(), yarn.RM_PORT.get())
+      return yarn
   for name in conf.MR_CLUSTERS.keys():
     mr = conf.MR_CLUSTERS[name]
     if mr.SUBMIT_TO.get():
-      return "%s:%s" % (mr.JT_HOST.get(), mr.JT_PORT.get())
+      return mr
   return None
 
+def get_cluster_addr_for_job_submission():
+  """
+  Check the `submit_to' for each MR/Yarn cluster, and return the
+  host:port of first one that enables submission.
+  """
+  conf = get_cluster_conf_for_job_submission()
+  if conf is None:
+    return None
+  return "%s:%s" % (conf.HOST.get(), conf.PORT.get())
+
+
 def clear_caches():
   """
   Clears cluster's internal caches.  Returns

+ 96 - 88
desktop/libs/hadoop/src/hadoop/conf.py

@@ -22,46 +22,9 @@ import os
 
 DEFAULT_NN_HTTP_PORT = 50070
 
-HADOOP_HOME = Config(
-  key="hadoop_home",
-  default=os.environ.get("HADOOP_HOME", "/usr/lib/hadoop"),
-  help=("Path to directory holding hadoop libs - HADOOP_HOME in " +
-        "hadoop parlance; defaults to environment variable, when" +
-        "set.")
-)
-
-HADOOP_MR1_HOME = Config(
-  key="hadoop_mr1_home",
-  default=os.environ.get("HADOOP_MR1_HOME", "/usr/lib/hadoop-0.20-mapreduce"),
-  help=("Path to directory holding hadoop libs - HADOOP_HOME in " +
-        "hadoop parlance; defaults to environment variable, when" +
-        "set.")
-)
-
-def hadoop_bin_from_hadoop_home():
-  """Returns $HADOOP_HOME/bin/hadoop"""
-  return os.path.join(HADOOP_HOME.get(), "bin/hadoop")
-
-HADOOP_BIN = Config("hadoop_bin",
-  help="Path to your Hadoop binary",
-  dynamic_default=hadoop_bin_from_hadoop_home,
-  type=str)
-
-# TODO(philip): This will need more love for dealing with multiple clusters.
-HADOOP_CONF_DIR = Config(
-  key="hadoop_conf_dir",
-  default="/etc/hadoop/conf",
-  help="Directory to pass to hadoop_bin (from Hadoop configuration) as the --config flag.",
-)
-
-def find_file_recursive(desired_glob, root=None):
-  if root is None:
-    root_f = lambda: HADOOP_HOME.get()
-  else:
-    root_f = lambda: not callable(root) and root or root()
-
+def find_file_recursive(desired_glob, root):
   def f():
-    for dirpath, dirnames, filenames in os.walk(root_f()):
+    for dirpath, dirnames, filenames in os.walk(root):
       matches = fnmatch.filter(filenames, desired_glob)
       if matches:
         if len(matches) != 1:
@@ -72,33 +35,9 @@ def find_file_recursive(desired_glob, root=None):
     logging.error("Trouble finding jars matching %s" % (desired_glob,))
     return None
 
-  if root is None:
-    root_str = "$HADOOP_HOME"
-  else:
-    root_str = root
-  f.__doc__ = "Finds %s/%s" % (root_str, desired_glob)
+  f.__doc__ = "Finds %s/%s" % (root, desired_glob)
   return f
 
-HADOOP_EXAMPLES_JAR = Config(
-  key="hadoop_examples_jar",
-  dynamic_default=find_file_recursive("hadoop-*examples*.jar", lambda: HADOOP_MR1_HOME.get()),
-  help="Path to the hadoop-examples.jar (used for tests and jobdesigner setup)",
-  type=str,
-  private=True)
-
-HADOOP_STREAMING_JAR = Config(
-  key="hadoop_streaming_jar",
-  dynamic_default=find_file_recursive("hadoop-*streaming*.jar", lambda: HADOOP_MR1_HOME.get()),
-  help="Path to the hadoop-streaming.jar (used by jobdesigner)",
-  type=str,
-  private=True)
-
-HADOOP_TEST_JAR = Config("hadoop_test_jar",
-  help="[Used by testing code.] Path to hadoop-test.jar",
-  dynamic_default=find_file_recursive("hadoop-*test*.jar", lambda: HADOOP_MR1_HOME.get()),
-  type=str,
-  private=True)
-
 HADOOP_PLUGIN_CLASSPATH = Config("hadoop_plugin_classpath",
   help="[Used only in testing code.] Path to the Hadoop plugin jar.",
   type=str,
@@ -149,6 +88,31 @@ HDFS_CLUSTERS = UnspecifiedConfigSection(
                               default=False, type=coerce_bool),
       TEMP_DIR=Config("temp_dir", help="HDFS directory for temporary files",
                       default='/tmp', type=str),
+
+      HADOOP_HDFS_HOME = Config(
+        key="hadoop_hdfs_home",
+        default=os.environ.get("HADOOP_HDFS_HOME", "/usr/lib/hadoop-hdfs"),
+        help=("Path to Hadoop HDFS home - HADOOP_HOME or HADOOP_HDFS_HOME in " +
+              "hadoop parlance. For tarball installations, it is the root of " +
+              "the untarred directory. For packages, " +
+              "it is /usr/lib/hadoop-hdfs." +
+              "Defaults to the environment varible HADOOP_BIN when set, " +
+              "or '/usr/bin/hadoop'."),
+      ),
+      HADOOP_BIN = Config(
+        key="hadoop_bin",
+        default=os.environ.get("HADOOP_BIN", "/usr/bin/hadoop"),
+        help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+              "Defaults to the environment varible HADOOP_BIN when set, " +
+              "or '/usr/bin/hadoop'.")
+      ),
+      HADOOP_CONF_DIR = Config(
+        key="hadoop_conf_dir",
+        default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+        help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+              "as the --config flag. Defaults to the environment variable " +
+              "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+      ),
     )
   )
 )
@@ -159,11 +123,11 @@ MR_CLUSTERS = UnspecifiedConfigSection(
   each=ConfigSection(
     help="Information about a single MapReduce cluster",
     members=dict(
-      JT_HOST=Config("jobtracker_host", help="Host/IP for JobTracker"),
-      JT_PORT=Config("jobtracker_port",
-                     default=8021,
-                     help="Service port for the JobTracker",
-                     type=int),
+      HOST=Config("jobtracker_host", help="Host/IP for JobTracker"),
+      PORT=Config("jobtracker_port",
+                  default=8021,
+                  help="Service port for the JobTracker",
+                  type=int),
       JT_THRIFT_PORT=Config("thrift_port", help="Thrift port for JobTracker", default=9290,
                             type=int),
       JT_KERBEROS_PRINCIPAL=Config("jt_kerberos_principal", help="Kerberos principal for JobTracker",
@@ -171,7 +135,29 @@ MR_CLUSTERS = UnspecifiedConfigSection(
       SECURITY_ENABLED=Config("security_enabled", help="Is running with Kerberos authentication",
                               default=False, type=coerce_bool),
       SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
-                       default=False, type=coerce_bool),
+                       default=True, type=coerce_bool),
+
+      HADOOP_MAPRED_HOME = Config(
+        key="hadoop_mapred_home",
+        default=os.environ.get("HADOOP_MR1_HOME", "/usr/lib/hadoop-0.20-mapreduce"),
+        help=("Path to directory holding Hadoop MR1 libs. " +
+              "E.g. /usr/lib/hadoop. Defaults to the environment variable " +
+              "HADOOP_MR1_HOME when set, or '/usr/lib/hadoop'.")
+      ),
+      HADOOP_BIN = Config(
+        key="hadoop_bin",
+        default=os.environ.get("HADOOP_MR1_BIN", "/usr/bin/hadoop"),
+        help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+              "Defaults to the environment varible HADOOP_MR1_BIN when set, " +
+              "or '/usr/bin/hadoop'.")
+      ),
+      HADOOP_CONF_DIR = Config(
+        key="hadoop_conf_dir",
+        default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+        help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+              "as the --config flag. Defaults to the environment variable " +
+              "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+      ),
     )
   )
 )
@@ -183,15 +169,37 @@ YARN_CLUSTERS = UnspecifiedConfigSection(
   each=ConfigSection(
     help="Information about a single Yarn cluster",
     members=dict(
-      RM_HOST=Config("resourcemanager_host",
-                     default='localhost',
-                     help="Host/IP for the ResourceManager"),
-      RM_PORT=Config("resourcemanager_port",
-                     default=8032,
-                     type=int,
-                     help="Service port for the ResourceManager"),
+      HOST=Config("resourcemanager_host",
+                  default='localhost',
+                  help="Host/IP for the ResourceManager"),
+      PORT=Config("resourcemanager_port",
+                  default=8032,
+                  type=int,
+                  help="Service port for the ResourceManager"),
       SUBMIT_TO=Config('submit_to', help="Whether Hue should use this cluster to run jobs",
                        default=False, type=coerce_bool),
+
+      HADOOP_MAPRED_HOME = Config(
+        key="hadoop_mapred_home",
+        default=os.environ.get("HADOOP_MR2_HOME", "/usr/lib/hadoop-mapreduce"),
+        help=("Path to directory holding Hadoop MR2 libs. " +
+              "E.g. /usr/lib/hadoop. Defaults to the environment " +
+              "variable HADOOP_MR2_HOME when set, or '/usr/lib/hadoop'.")
+      ),
+      HADOOP_BIN = Config(
+        key="hadoop_bin",
+        default=os.environ.get("HADOOP_MR2_BIN", "/usr/bin/hadoop"),
+        help=("Path to your Hadoop launcher script. E.g. /usr/bin/hadoop. " +
+              "Defaults to the environment varible HADOOP_MR2_BIN when set, " +
+              "or '/usr/bin/hadoop'.")
+      ),
+      HADOOP_CONF_DIR = Config(
+        key="hadoop_conf_dir",
+        default=os.environ.get("HADOOP_CONF_DIR", "/etc/hadoop/conf"),
+        help=("Directory to pass to hadoop_bin (from Hadoop configuration) " +
+              "as the --config flag. Defaults to the environment variable " +
+              "HADOOP_CONF_DIR when set, or '/etc/hadoop/conf'.")
+      ),
     )
   )
 )
@@ -206,22 +214,15 @@ def config_validator():
   from hadoop.fs import webhdfs
   from hadoop import job_tracker
   res = [ ]
-
-  # HADOOP_HOME
-  res.extend(validate_path(HADOOP_HOME, is_dir=True))
-  # HADOOP_BIN
-  res.extend(validate_path(HADOOP_BIN, is_dir=False))
-
-  # JARs: even though these are private, we need them to run jobsub
-  res.extend(validate_path(HADOOP_EXAMPLES_JAR, is_dir=False))
-  res.extend(validate_path(HADOOP_STREAMING_JAR, is_dir=False))
-
   submit_to = [ ]
 
   # HDFS_CLUSTERS
   has_default = False
   for name in HDFS_CLUSTERS.keys():
     cluster = HDFS_CLUSTERS[name]
+    res.extend(validate_path(cluster.HADOOP_HDFS_HOME, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
     res.extend(webhdfs.test_fs_configuration(cluster))
     if name == 'default':
       has_default = True
@@ -231,16 +232,23 @@ def config_validator():
   # MR_CLUSTERS
   for name in MR_CLUSTERS.keys():
     cluster = MR_CLUSTERS[name]
+    res.extend(validate_path(cluster.HADOOP_MAPRED_HOME, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
     res.extend(job_tracker.test_jt_configuration(cluster))
     if cluster.SUBMIT_TO.get():
       submit_to.append('mapred_clusters.' + name)
 
-  # Only one cluster should have submit_to
+  # YARN_CLUSTERS
   for name in YARN_CLUSTERS.keys():
     cluster = YARN_CLUSTERS[name]
+    res.extend(validate_path(cluster.HADOOP_MAPRED_HOME, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_CONF_DIR, is_dir=True))
+    res.extend(validate_path(cluster.HADOOP_BIN, is_dir=False))
     if cluster.SUBMIT_TO.get():
       submit_to.append('yarn_clusters.' + name)
 
+  # Only one cluster should have submit_to
   if len(submit_to) > 1:
     res.append(("hadoop", "Only one cluster may enable 'submit_to'. "
                 "But it is enabled in the following clusters: " + 

+ 2 - 2
desktop/libs/hadoop/src/hadoop/job_tracker.py

@@ -50,7 +50,7 @@ def test_jt_configuration(cluster):
     jt.runtime_info()
   except TTransport.TTransportException:
     msg = 'Failed to contact JobTracker plugin at %s:%s.' % \
-          (cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get())
+          (cluster.HOST.get(), cluster.JT_THRIFT_PORT.get())
     return [ (cluster, msg) ]
   return []
 
@@ -84,7 +84,7 @@ class LiveJobTracker(object):
   @classmethod
   def from_conf(cls, conf):
     return cls(
-      conf.JT_HOST.get(),
+      conf.HOST.get(),
       conf.JT_THRIFT_PORT.get(),
       security_enabled=conf.SECURITY_ENABLED.get(),
       kerberos_principal=conf.JT_KERBEROS_PRINCIPAL.get())

+ 11 - 2
desktop/libs/hadoop/src/hadoop/mini_cluster.py

@@ -14,7 +14,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
+
+#######################################################
+##               WARNING!!!                          ##
+##   This file is stale. Hadoop 0.23 and CDH4        ##
+##   do not support minicluster. This is replaced    ##
+##   by webhdfs.py, to set up a running cluster.     ##
+#######################################################
+
+
 # A Python-side driver for MiniHadoopClusterManager
 # 
 # See README.testing for hints on how to use this,
@@ -413,7 +422,7 @@ def shared_cluster(conf=False):
     closers.extend([
       hadoop.conf.HDFS_CLUSTERS["default"].NN_HOST.set_for_testing("localhost"),
       hadoop.conf.HDFS_CLUSTERS["default"].NN_HDFS_PORT.set_for_testing(cluster.namenode_port),
-      hadoop.conf.MR_CLUSTERS["default"].JT_HOST.set_for_testing("localhost"),
+      hadoop.conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
       hadoop.conf.MR_CLUSTERS["default"].JT_THRIFT_PORT.set_for_testing(cluster.jt.thrift_port),
     ])
     # Clear the caches

+ 10 - 18
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -199,7 +199,8 @@ class PseudoHdfs4(object):
 
     # More stuff to setup in the environment
     env = dict(
-      HADOOP_HOME = hadoop.conf.HADOOP_HOME.get(),
+      HADOOP_HOME = hadoop.conf.HDFS_CLUSTERS['default'].HADOOP_HDFS_HOME.get(),
+      HADOOP_BIN = hadoop.conf.HDFS_CLUSTERS['default'].HADOOP_BIN.get(),
       HADOOP_CONF_DIR = conf_dir,
       HADOOP_HEAPSIZE = "128",
       HADOOP_LOG_DIR = self._log_dir,
@@ -237,21 +238,9 @@ class PseudoHdfs4(object):
 
     # We need a different env because it's a different hadoop
     env = env.copy()
-    env['HADOOP_HOME'] = hadoop.conf.HADOOP_MR1_HOME.get()
+    env['HADOOP_HOME'] = hadoop.conf.MR_CLUSTERS['default'].HADOOP_MAPRED_HOME.get()
+    env['HADOOP_BIN'] = hadoop.conf.MR_CLUSTERS['default'].HADOOP_BIN.get()
     env["HADOOP_CLASSPATH"] = ':'.join([
-        # -- BEGIN JAVA TRIVIA --
-        # Add the -test- jar to the classpath to work around a subtle issue
-        # involving Java classloaders. In brief, hadoop's RunJar class creates
-        # a child classloader with the test jar on it, but the core classes
-        # are loaded by the system classloader. This is fine except that
-        # some classes in the test jar extend package-protected classes in the
-        # core jar. Even though the classes are in the same package name, they
-        # are thus loaded by different classloaders and therefore an IllegalAccessError
-        # prevents the MiniMRCluster from starting. Adding the test jar to the system
-        # classpath prevents this error since then both the MiniMRCluster and the
-        # core classes are loaded by the system classloader.
-        hadoop.conf.HADOOP_TEST_JAR.get(),
-        # -- END JAVA TRIVIA --
         hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get(),
         # Due to CDH-4537, we need to add test dependencies to run minicluster
         os.path.join(os.path.dirname(__file__), 'test_jars', '*'),
@@ -276,7 +265,7 @@ class PseudoHdfs4(object):
 
   def _format(self, conf_dir, env):
     """Format HDFS"""
-    args = (hadoop.conf.HADOOP_BIN.get(), 
+    args = (self._get_hadoop_bin(env),
             '--config', conf_dir,
             'namenode', '-format')
     LOG.info('Formatting HDFS: %s' % (args,))
@@ -377,7 +366,10 @@ class PseudoHdfs4(object):
       env=env)
 
   def _get_hadoop_bin(self, env):
-    return os.path.join(env['HADOOP_HOME'], 'bin', 'hadoop')
+    try:
+      return env['HADOOP_BIN']
+    except KeyError:
+      return os.path.join(env['HADOOP_HOME'], 'bin', 'hadoop')
 
   def _write_hdfs_site(self):
     self._dfs_http_port = find_unused_port()
@@ -466,7 +458,7 @@ def shared_cluster():
     closers = [
       hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
       hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),
-      hadoop.conf.MR_CLUSTERS['default'].JT_HOST.set_for_testing('localhost'),
+      hadoop.conf.MR_CLUSTERS['default'].HOST.set_for_testing('localhost'),
       hadoop.conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(cluster.jt_thrift_port),
     ]
 

+ 1 - 4
desktop/libs/hadoop/src/hadoop/tests.py

@@ -112,7 +112,6 @@ def test_tricky_confparse():
 
 def test_config_validator_basic():
   reset = (
-    conf.HADOOP_STREAMING_JAR.set_for_testing('/tmp'),
     conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing('http://not.the.re:50070/'),
     conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(70000),
   )
@@ -120,8 +119,6 @@ def test_config_validator_basic():
   try:
     cli = make_logged_in_client()
     resp = cli.get('/debug/check_config')
-    assert_true('hadoop.hadoop_streaming_jar' in resp.content)
-    assert_true('Not a file' in resp.content)
     assert_true('hadoop.hdfs_clusters.default.webhdfs_url' in resp.content)
     assert_true('hadoop.mapred_clusters.default.thrift_port' in resp.content)
     assert_true('Port should be' in resp.content)
@@ -141,7 +138,7 @@ def test_config_validator_more():
   cli = make_logged_in_client()
 
   reset = (
-    conf.MR_CLUSTERS["default"].JT_HOST.set_for_testing("localhost"),
+    conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
     conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(23),
   )
   old = cluster.clear_caches()

+ 6 - 14
dist/README

@@ -5,26 +5,18 @@ If you're impatient, these are the key steps.  Please check the full manual
 for more details.
 
 ## Install
-$ HADOOP_HOME=/usr/lib/hadoop-0.20 PREFIX=/usr/local make install
+$ PREFIX=/usr/share make install
 
 ## Install plug-ins
-$ cd /usr/lib/hadoop/lib
-$ ln -s /usr/local/hue/desktop/libs/hadoop/java-lib/hue*jar
+$ cd /usr/lib/hadoop-0.20-mapreduce/lib
+$ ln -s /usr/share/hue/desktop/libs/hadoop/java-lib/hue*jar
 
 ## Configure Hadoop
 Edit hdfs-site.xml:
 
 <property>
-  <name>dfs.namenode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
-  <description>Comma-separated list of namenode plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>dfs.datanode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
-  <description>Comma-separated list of datanode plug-ins to be activated.
-  </description>
+  <name>dfs.webhdfs.enable</name>
+  <value>true</value>
 </property>
 
 Edit mapred-site.xml:
@@ -37,4 +29,4 @@ Edit mapred-site.xml:
 </property>
 
 ## Run!
-$ /usr/local/hue/build/env/bin/supervisor
+$ /usr/share/hue/build/env/bin/supervisor

+ 46 - 14
docs/manual.txt

@@ -412,19 +412,6 @@ Hue Configuration for Hadoop
 These configuration variables are under the `[hadoop]` section in
 the `/etc/hue/hue.ini` configuration file.
 
-hadoop_home::
-  This becomes the value of `$HADOOP_HOME` for any Shell processes
-  and the Beeswax Server. If you use MR1, set this to the MR1 home.
-
-hadoop_bin::
-  Use this as the hadoop binary. If you use MR1, set this to
-  `<hadoop_home>/bin/hadoop` instead of the default `/usr/bin/hadoop`.
-
-hadoop_conf_dir::
-  This is the configuration directory for any processes to
-  configure their Hadoop client. If you use MR1, set this to the
-  directory containing your MR1 configuration.
-
 
 HDFS Cluster
 ^^^^^^^^^^^^
@@ -433,13 +420,26 @@ Hue only support one HDFS cluster currently. That cluster should be defined
 under the `[[[default]]]` sub-section.
 
 fs_defaultfs::
-  This is the equivalence of ``fs.defaultFS`` (aka ``fs.default.name``) in
+  This is the equivalence of `fs.defaultFS` (aka `fs.default.name`) in
   Hadoop configuration.
 
 webhdfs_url::
   You can also set this to be the HttpFS url. The default value is the HTTP
   port on the NameNode.
 
+hadoop_hdfs_home::
+  This is the home of your Hadoop HDFS installation. It is the
+  root of the Hadoop untarred directory, or usually
+  `/usr/lib/hadoop`.
+
+hadoop_bin::
+  Use this as the HDFS Hadoop launcher script, which is usually
+  `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+  This is the configuration directory of the HDFS, typically
+  `/etc/hadoop/conf`.
+
 
 MapReduce (MR1) Cluster
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -457,6 +457,22 @@ submit_to::
   If your Oozie is configured with to talk to a 0.20 MapReduce service, then
   set this to `true`. Hue will be submitting jobs to this MapReduce cluster.
 
+hadoop_mapred_home::
+  This is the home of your Hadoop MapReduce installation. It is the
+  root of the Hadoop MR1 untarred directory, or the root of the
+  Hadoop 0.20 untarred directory, or `/usr/lib/hadoop-0.20-mapreduce` for
+  CDH packages. If `submit_to` is true for this cluster, this
+  config value becomes the `$HADOOP_MAPRED_HOME` for
+  BeeswaxServer and child shell processes.
+
+hadoop_bin::
+  Use this as the MR1 Hadoop launcher script, which is usually
+  `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+  This is the configuration directory of the MR1 service,
+  typically `/etc/hadoop/conf`.
+
 
 Yarn (MR2) Cluster
 ^^^^^^^^^^^^^^^^^^
@@ -475,6 +491,22 @@ submit_to::
   set this to `true`. Hue will be submitting jobs to this Yarn cluster.
   But note that JobBrowser will not be able to show MR2 jobs.
 
+hadoop_mapred_home::
+  This is the home of your Hadoop MapReduce installation. It is the
+  root of the Hadoop 0.23 untarred directory, or
+  `/usr/lib/hadoop-mapreduce` for CDH packages. If `submit_to` is
+  true for this cluster, this config value becomes the
+  `$HADOOP_MAPRED_HOME` for BeeswaxServer and child shell
+  processes.
+
+hadoop_bin::
+  Use this as the Yarn/MR2 Hadoop launcher script, which is usually
+  `/usr/bin/hadoop`.
+
+hadoop_conf_dir::
+  This is the configuration directory of the Yarn/MR2 service,
+  typically `/etc/hadoop/conf`.
+
 
 Beeswax Configuration
 ~~~~~~~~~~~~~~~~~~~~~

+ 3 - 1
tools/jenkins/build-functions

@@ -35,7 +35,8 @@ build_hadoop() {
   fi
 
   HADOOP_DIR=$HUE_ROOT/ext/hadoop
-  export HADOOP_HOME="$HADOOP_DIR/${CDH_VERSION}"
+  export HADOOP_HDFS_HOME="$HADOOP_DIR/${CDH_VERSION}"
+  export HADOOP_BIN=$HADOOP_HDFS_HOME/bin/hadoop
 
   mkdir -p $HADOOP_DIR
   rm -rf $HADOOP_HOME
@@ -61,6 +62,7 @@ build_mr1() {
 
   MR1_DIR=$HUE_ROOT/ext/mr1
   export HADOOP_MR1_HOME="$MR1_DIR/${MR1_VERSION}"
+  export HADOOP_MR1_BIN="$HADOOP_MR1_HOME/bin/hadoop"
 
   mkdir -p $MR1_DIR
   rm -rf $HADOOP_MR1_HOME

+ 0 - 267
tools/scripts/configure-hadoop.sh

@@ -1,267 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-
-if test -z $HADOOP_HOME ; then
-  echo "Please define your HADOOP_HOME environment variable."
-  exit
-fi
-
-SCRIPT_DIR=$(dirname $0)
-GIT_ROOT=$(cd $SCRIPT_DIR && pwd)/$(cd $SCRIPT_DIR && git rev-parse --show-cdup)
-BIND_IP=${BIND_IP:-localhost}
-HADOOP_TMP_DIR=$HADOOP_HOME/data
-
-if [ $(uname) == "Darwin" ]; then
-  export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/1.6.0/Home
-fi
-SLAVE_IPS=${SLAVE_IPS:-127.0.0.1}
-
-echo -e ".======================== Parameters ========================\n"\
-        "HADOOP_HOME    : $HADOOP_HOME\n"\
-	"GIT_ROOT       : $GIT_ROOT\n"\
-	"HADOOP_TMP_DIR : $HADOOP_TMP_DIR\n"\
-	"============================================================\n"
-set -x
-
-
-# Configure a slave
-# Used by the sandboxer
-function write_hadoop_config() {
-  SLAVE_IP=$1
-  MASTER_IP=$2
-  TARGET_DIR=$3
-  TMP_DIR=$4
-  if [ -z "$SLAVE_IP" -o -z "$MASTER_IP" -o -z "$TARGET_DIR" -o -z "$TMP_DIR" ]; then
-    echo usage: $0 slave_ip master_ip target_conf_dir tmp_dir
-    exit 1
-  fi
-
-  mkdir -p $TARGET_DIR
-  cp $HADOOP_HOME/example-confs/conf.pseudo/hadoop-metrics.properties $TARGET_DIR
-  cat > $TARGET_DIR/core-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>fs.default.name</name>
-  <value>hdfs://${MASTER_IP}:8020</value>
-</property>
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>$TMP_DIR</value>
-  <description>A base for other temporary directories.</description>
-</property>
-<property>
-  <name>webinterface.private.actions</name>
-  <value>true</value>
-</property>
-<property>
-  <name>slave.host.name</name>
-  <value>${SLAVE_IP}</value>
-</property>
-<property>
-  <name>dfs.thrift.address</name>
-  <value>${MASTER_IP}:10090</value>
-</property>
-<property>
-  <name>jobtracker.thrift.address</name>
-  <value>${MASTER_IP}:9290</value>
-</property>
-</configuration>
-END
-  cat > $TARGET_DIR/hdfs-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>dfs.datanode.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>dfs.datanode.ipc.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>dfs.datanode.http.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>dfs.http.address</name>
-  <value>${MASTER_IP}:50070</value>
-</property>
-<property>
-  <name>dfs.namenode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
-  <description>Comma-separated list of namenode plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>dfs.datanode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
-  <description>Comma-separated list of datanode plug-ins to be activated.
-  </description>
-</property>
-<!-- we dont really care about being super safe -->
-<property>
-  <name>dfs.safemode.min.datanodes</name>
-  <value>1</value>
-</property>
-<property>
-  <name>dfs.safemode.extension</name>
-  <value>5000</value>
-  <description>
-    Determines extension of safe mode in milliseconds
-    after the threshold level is reached.
-  </description>
-</property>
-</configuration>
-END
-  cat > $TARGET_DIR/mapred-site.xml <<END
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>mapred.job.tracker</name>
-  <value>${MASTER_IP}:8021</value>
-</property>
-<property>
-  <name>mapred.job.tracker.http.address</name>
-  <value>${MASTER_IP}:50030</value>
-</property>
-<property>
-  <name>mapred.task.tracker.http.address</name>
-  <value>${SLAVE_IP}:0</value>
-</property>
-<property>
-  <name>mapred.jobtracker.plugins</name>
-  <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
-  <description>Comma-separated list of jobtracker plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>mapred.system.dir</name>
-  <value>/hadoop/mapred/system</value>
-</property>
-<property>
-  <name>mapred.local.dir</name>
-  <value>$TMP_DIR/mapred/local</value>
-</property>
-</configuration>
-END
-
-}
-
-# Configure
-function configure() {
-  perl -p -i -e "s,localhost,$BIND_IP,g" desktop/conf/pseudo-distributed.ini
-  mkdir -p $HADOOP_TMP_DIR
-  write_hadoop_config $BIND_IP $BIND_IP $HADOOP_HOME/conf $HADOOP_TMP_DIR
-  idx=0
-  for slave in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    datadir=$HADOOP_TMP_DIR-slave-$idx
-    write_hadoop_config $slave $BIND_IP $HADOOP_HOME/conf-slave-$idx $datadir
-  done
-}
-
-function start() {
-  pushd $HADOOP_HOME
-  export HADOOP_CLASSPATH=$GIT_ROOT/desktop/libs/hadoop/java-lib/\*
-  if [ ! -d $HADOOP_TMP_DIR/dfs/name ]; then
-    bin/hadoop namenode -format
-  fi
-  # Pass HADOOP_OPTS=$JDB_ON to any hadoop-daemon.sh to enable jdb
-  JDB_ON="-Xdebug -Xrunjdwp:transport=dt_socket,address=8901,server=y,suspend=n"
-
-  HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-    bin/hadoop-daemon.sh start namenode
-  idx=0
-  for ip in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-      HADOOP_LOG_DIR="$HADOOP_HOME/logs-slave-$idx" \
-      bin/hadoop-daemon.sh start datanode || true
-  done
-  bin/hadoop dfsadmin -safemode wait
-  HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-    bin/hadoop-daemon.sh start secondarynamenode
-  HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-    bin/hadoop-daemon.sh start jobtracker
-  idx=0
-  for ip in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-      HADOOP_LOG_DIR="$HADOOP_HOME/logs-slave-$idx" \
-      bin/hadoop-daemon.sh start tasktracker || true
-  done
-  popd
-}
-
-function stop() {
-  pushd $HADOOP_HOME
-  for daemon in namenode jobtracker; do
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-master" \
-      bin/hadoop-daemon.sh stop $daemon
-  done
-
-  idx=0
-  for ip in $SLAVE_IPS ; do
-    idx=$[$idx + 1]
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-      bin/hadoop-daemon.sh --config $HADOOP_HOME/conf-slave-$idx stop tasktracker || true
-    HADOOP_PID_DIR="$HADOOP_TMP_DIR-pids-slave-$idx" \
-      HADOOP_CONF_DIR="$HADOOP_HOME/conf-slave-$idx" \
-       bin/hadoop-daemon.sh --config $HADOOP_HOME/conf-slave-$idx stop datanode || true
-  done
-
-  popd
-}
-
-
-pushd $GIT_ROOT
-
-if [ ! -d .git ]; then
-  echo "This script moves into your git root, but this has failed."
-  exit 1
-fi
-
-if [ $# -eq 0 ]; then
-  echo "Usage: $0 (all|checkout|build|build_jobsub|build_plugins|configure|start)"
-  exit 1
-fi
-
-ARG=$1
-shift 1
-case $ARG
-in
-  configure|start|write_hadoop_config|stop)
-    $ARG $@
-    ;;
-  all)
-    configure
-    start
-    ;;
-  *)
-    echo "Unrecognized: $ARG"
-    exit 1
-  ;;
-esac