Просмотр исходного кода

[HUE-345] Provide way to override settings and classpath of minicluster (REVISED)

Aaron T. Myers 15 лет назад
Родитель
Сommit
7976ff4f87

+ 1 - 1
apps/jobsub/src/jobsub/server.py

@@ -227,7 +227,7 @@ class PlanRunner(object):
     env = {      
     env = {      
       'HADOOP_HOME': hadoop.conf.HADOOP_HOME.get(), 
       'HADOOP_HOME': hadoop.conf.HADOOP_HOME.get(), 
       'HADOOP_OPTS': "-javaagent:%s %s" % (jobsub.conf.ASPECTJWEAVER.get(), java_prop_str),
       'HADOOP_OPTS': "-javaagent:%s %s" % (jobsub.conf.ASPECTJWEAVER.get(), java_prop_str),
-      'HADOOP_CLASSPATH': ':'.join([jobsub.conf.ASPECTPATH.get(), hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()]),
+      'HADOOP_CLASSPATH': ':'.join([jobsub.conf.ASPECTPATH.get(), hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()]),
       'HUE_JOBTRACE_LOG': self.internal_file_name("jobs"),
       'HUE_JOBTRACE_LOG': self.internal_file_name("jobs"),
       'HUE_JOBSUB_USER': self.plan.user,
       'HUE_JOBSUB_USER': self.plan.user,
       'HUE_JOBSUB_GROUPS': ",".join(self.plan.groups),
       'HUE_JOBSUB_GROUPS': ",".join(self.plan.groups),

+ 2 - 2
desktop/libs/hadoop/src/hadoop/conf.py

@@ -98,8 +98,8 @@ HADOOP_PLUGIN_CLASSPATH = Config("hadoop_plugin_classpath",
   dynamic_default=find_jar("../../java-lib/hue-plugins-*.jar", root=os.path.dirname(__file__)),
   dynamic_default=find_jar("../../java-lib/hue-plugins-*.jar", root=os.path.dirname(__file__)),
   private=True)
   private=True)
 
 
-HADOOP_STATIC_GROUP_MAPPING_CLASSPATH = Config("hadoop_static_group_mapping_classpath",
-  help="[Used only in testing code.] Path to the Hadoop static group mapping jar.",
+HADOOP_EXTRA_CLASSPATH_STRING = Config('hadoop_extra_classpath_entries',
+  help='[Used only in testing code.] String to add to the end of the HADOOP_CLASSPATH environment variable when calling bin/hadoop.',
   type=str,
   type=str,
   dynamic_default=find_jar("../../static-group-mapping/java-lib/static-group-mapping-*.jar", root=os.path.dirname(__file__)),
   dynamic_default=find_jar("../../static-group-mapping/java-lib/static-group-mapping-*.jar", root=os.path.dirname(__file__)),
   private=True)
   private=True)

+ 2 - 2
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -799,9 +799,9 @@ class FileUpload(object):
     self.subprocess_env = i18n.make_utf8_env()
     self.subprocess_env = i18n.make_utf8_env()
 
 
     if self.subprocess_env.has_key('HADOOP_CLASSPATH'):
     if self.subprocess_env.has_key('HADOOP_CLASSPATH'):
-      self.subprocess_env['HADOOP_CLASSPATH'] += ':' + hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()
+      self.subprocess_env['HADOOP_CLASSPATH'] += ':' + hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()
     else:
     else:
-      self.subprocess_env['HADOOP_CLASSPATH'] = hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()
+      self.subprocess_env['HADOOP_CLASSPATH'] = hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()
 
 
     if hadoop.conf.HADOOP_CONF_DIR.get():
     if hadoop.conf.HADOOP_CONF_DIR.get():
       self.subprocess_env['HADOOP_CONF_DIR'] = hadoop.conf.HADOOP_CONF_DIR.get()
       self.subprocess_env['HADOOP_CONF_DIR'] = hadoop.conf.HADOOP_CONF_DIR.get()

+ 2 - 5
desktop/libs/hadoop/src/hadoop/mini_cluster.py

@@ -89,15 +89,12 @@ class MiniHadoopCluster(object):
     self.num_datanodes = num_datanodes
     self.num_datanodes = num_datanodes
     self.num_tasktrackers = num_tasktrackers
     self.num_tasktrackers = num_tasktrackers
 
 
-  def start(self, extra_classpath_entries=None, extra_configs=None):
+  def start(self, extra_configs=None):
     """
     """
     Start a cluster as a subprocess.
     Start a cluster as a subprocess.
     """
     """
     self.tmpdir = tempfile.mkdtemp()
     self.tmpdir = tempfile.mkdtemp()
 
 
-    if not extra_classpath_entries:
-      extra_classpath_entries = []
-
     if not extra_configs:
     if not extra_configs:
       extra_configs = {}
       extra_configs = {}
 
 
@@ -192,7 +189,7 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
         hadoop.conf.HADOOP_TEST_JAR.get(),
         hadoop.conf.HADOOP_TEST_JAR.get(),
         # -- END JAVA TRIVIA --
         # -- END JAVA TRIVIA --
         hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get(),
         hadoop.conf.HADOOP_PLUGIN_CLASSPATH.get(),
-        hadoop.conf.HADOOP_STATIC_GROUP_MAPPING_CLASSPATH.get()] + extra_classpath_entries)
+        hadoop.conf.HADOOP_EXTRA_CLASSPATH_STRING.get()])
       env["HADOOP_HEAPSIZE"] = "128"
       env["HADOOP_HEAPSIZE"] = "128"
       env["HADOOP_HOME"] = hadoop.conf.HADOOP_HOME.get()
       env["HADOOP_HOME"] = hadoop.conf.HADOOP_HOME.get()
       env["HADOOP_LOG_DIR"] = self.log_dir
       env["HADOOP_LOG_DIR"] = self.log_dir