Эх сурвалжийг харах

HUE-560: Shell app should have per-shell configurable environment variables.

Aditya Acharya 14 жил өмнө
parent
commit
f9e6b7e634

+ 29 - 3
apps/shell/conf/hue-shell.ini

@@ -2,18 +2,44 @@
 
 [shell]
 ## These are commented out so you know where to put lines like these if necessary.
+
+## The shell_buffer_amount specifies the number of bytes of output per shell
+## that the Shell app will keep in memory. If not specified, it defaults to
+## 524288 (512 MiB).
 #shell_buffer_amount=100
+
+## If you run Hue against a Hadoop cluster with Kerberos security enabled, the
+## Shell app needs to acquire delegation tokens for the subprocesses to work
+## correctly. These delegation tokens are stored as temporary files in some
+## directory. You can configure this directory here. If not specified, it
+## defaults to /tmp/hue_delegation_tokens.
 #shell_delegation_token_dir=/tmp/hue_delegation_tokens
+
 [[ shelltypes ]]
+
 [[[ flume ]]]
 nice_name = "Flume Shell"
-command = "flume shell"
+command = "/usr/bin/flume shell"
 help = "The command-line Flume client interface."
+[[[[ environment ]]]]
+## You can specify environment variables for the Flume shell
+## in this section.
+
 [[[ pig ]]]
 nice_name = "Pig Shell (Grunt)"
-command = "pig -l /dev/null"
+command = "/usr/bin/pig -l /dev/null"
 help = "The command-line interpreter for Pig"
+[[[[ environment ]]]]
+## You can specify environment variables for the Pig shell
+## in this section. Note that JAVA_HOME must be configured
+## for the Pig shell to run.
+[[[[[ JAVA_HOME ]]]]]
+value = "/usr/lib/jvm/java-6-sun"
+
 [[[ hbase ]]]
 nice_name = "HBase Shell"
-command = "hbase shell"
+command = "/usr/bin/hbase shell"
 help = "The command-line HBase client interface."
+[[[[ environment ]]]]
+## You can configure environment variables for the HBase shell
+## in this section.

+ 43 - 10
apps/shell/src/shell/conf.py

@@ -22,41 +22,74 @@ from desktop.lib.conf import Config, ConfigSection, UnspecifiedConfigSection
 import shell.utils as utils
 
 SHELL_TYPES = UnspecifiedConfigSection(
-                           key='shelltypes',
-                           each=ConfigSection(members=dict(
-                               nice_name=Config(key='nice_name', required=True),
-                               command=Config(key='command', required=True),
-                               help_doc=Config(key='help', required=False))))
+  key='shelltypes',
+  each=ConfigSection(
+    members=dict(
+      nice_name=Config(
+        key='nice_name',
+        required=True
+      ),
+      command=Config(
+        key='command',
+        required=True
+      ),
+      help_doc=Config(
+        key='help',
+        required=False
+      ),
+      environment=UnspecifiedConfigSection(
+        key='environment',
+        each=ConfigSection(
+          members=dict(
+            value=Config(
+              key='value',
+              required=True
+            ),
+            doc=Config(
+              key='doc',
+              required=False
+            )
+          )
+        )
+      )
+    )
+  )
+)
 
 SHELL_BUFFER_AMOUNT = Config(
   key="shell_buffer_amount",
   help="Configure the number of output characters buffered for each shell",
   default=524288,
-  type=int)
+  type=int
+)
 
 SHELL_TIMEOUT = Config(
   key="shell_timeout",
   help="Number of seconds to keep shells open for users",
   default=600,
-  type=int)
+  type=int
+)
 
 SHELL_WRITE_BUFFER_LIMIT = Config(
   key="shell_write_buffer_limit",
   help="Number of bytes of commands to buffer for users",
   default=10000,
-  type=int)
+  type=int
+)
 
 SHELL_OS_READ_AMOUNT = Config(
   key="shell_os_read_amount",
   help="Number of bytes to read from child subprocess at a time",
   default=40960,
-  type=int)
+  type=int
+)
 
 SHELL_DELEGATION_TOKEN_DIR = Config(
   key="shell_delegation_token_dir",
   help="The directory to store the temporary delegation tokens used by shell subprocesses",
   default="/tmp/hue_shell_delegation_tokens",
-  type=str)
+  type=str
+)
 
 def config_validator():
   """

+ 1 - 5
apps/shell/src/shell/constants.py

@@ -48,15 +48,11 @@ IS_TAB = "isTab"
 NO_SUCH_USER = "noSuchUser"
 SHELL_NOT_ALLOWED = "shellNotAllowed"
 HOME = "HOME"
+HADOOP_HOME = "HADOOP_HOME"
 HADOOP_TOKEN_FILE_LOCATION = 'HADOOP_TOKEN_FILE_LOCATION'
 EXISTS = "exists"
 
 # HTTP Headers used
 HUE_INSTANCE_ID = "HTTP_HUE_INSTANCE_ID"
 
-# Required environment variables
-PRESERVED_ENVIRONMENT_VARIABLES = ["JAVA_HOME", "HADOOP_HOME", "PATH", "LC_ALL", "LANG",
-              "LC_COLLATE", "LC_CTYPE", "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", "TZ",
-              "FLUME_CONF_DIR"]
-
 BROWSER_REQUEST_TIMEOUT = 55    # seconds

+ 22 - 17
apps/shell/src/shell/shellmanager.py

@@ -63,14 +63,7 @@ class Shell(object):
   """
   A class to encapsulate I/O with a shell subprocess.
   """
-  def __init__(self, shell_command, shell_id, username, delegation_token_dir):
-    subprocess_env = {}
-    env = desktop.lib.i18n.make_utf8_env()
-    for item in constants.PRESERVED_ENVIRONMENT_VARIABLES:
-      value = env.get(item)
-      if value:
-        subprocess_env[item] = value
-
+  def __init__(self, shell_command, subprocess_env, shell_id, username, delegation_token_dir):
     try:
       user_info = pwd.getpwnam(username)
     except KeyError:
@@ -365,7 +358,6 @@ class ShellManager(object):
   """
   def __init__(self):
     self._shells = {} # Keys are (username, shell_id) tuples. Each user has his/her own set of shell ids.
-    shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
     self._command_by_short_name = {} # Map each short name to its command (e.g. ["pig", "-l", "/dev/null"])
     self._meta = {} # Map usernames to utils.UserMetadata objects
     self._greenlets_by_hid = {} # Map each Hue Instance ID (HID) to greenlet currently fetching output for that HID.
@@ -373,26 +365,36 @@ class ShellManager(object):
     self._greenlets_to_notify = {} # For each PID, maintain a set of greenlets who are also interested in the output from that process, but are not doing the select.
     self._shells_by_fds = {} # Map each file descriptor to the Shell instance whose output it represents.
     self._greenlet_interruptable = {} # For each greenlet, store if it can be safely interrupted.
+    self._env_by_short_name = {} # Map each short name to a dictionary which contains the environment for shells of that type.
 
     self._delegation_token_dir = shell.conf.SHELL_DELEGATION_TOKEN_DIR.get()
     if not os.path.exists(self._delegation_token_dir):
       os.mkdir(self._delegation_token_dir)
 
+    self._parse_configs()
+    eventlet.spawn_after(1, self._handle_periodic)
+
+  @classmethod
+  def global_instance(cls):
+    if not hasattr(cls, "_global_instance"):
+      cls._global_instance = cls()
+    return cls._global_instance
+
+  def _parse_configs(self):
+    shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
     for item in shell.conf.SHELL_TYPES.keys():
+      env_for_shell = { constants.HADOOP_HOME: hadoop.conf.HADOOP_HOME.get() }
       command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
       nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
       executable_exists = utils.executable_exists(command)
       if executable_exists:
         self._command_by_short_name[item] = command
+        conf_shell_env = shell.conf.SHELL_TYPES[item].environment
+        for env_variable in conf_shell_env.keys():
+          env_for_shell[env_variable] = conf_shell_env[env_variable].value.get()
+        self._env_by_short_name[item] = env_for_shell
       shell_types.append({ constants.NICE_NAME: nice_name, constants.KEY_NAME: item, constants.EXISTS:executable_exists })
     self.shell_types = shell_types
-    eventlet.spawn_after(1, self._handle_periodic)
-
-  @classmethod
-  def global_instance(cls):
-    if not hasattr(cls, "_global_instance"):
-      cls._global_instance = cls()
-    return cls._global_instance
 
   def available_shell_types(self, user):
     username = user.username
@@ -494,7 +496,10 @@ class ShellManager(object):
     shell_id = user_metadata.get_next_id()
     try:
       LOG.debug("Trying to create a %s shell for user %s" % (shell_name, username))
-      shell_instance = Shell(command, shell_id, username, self._delegation_token_dir)
+      # Let's make a copy of the subprocess's environment since the Shell constructor will modify
+      # the dictionary we pass in.
+      subprocess_env = self._env_by_short_name.get(shell_name, {}).copy()
+      shell_instance = Shell(command, subprocess_env, shell_id, username, self._delegation_token_dir)
     except (OSError, ValueError, KeyError, MergeToolException):
       LOG.exception("Could not create %s shell for '%s'" % (shell_name, username))
       return { constants.SHELL_CREATE_FAILED : True }

+ 1 - 1
desktop/core/src/desktop/management/commands/runcpserver.py

@@ -32,7 +32,7 @@ CPSERVER_OPTIONS = {
   'server_name': 'localhost',
   'threads': conf.CHERRYPY_SERVER_THREADS.get(),
   'daemonize': False, # supervisor does this for us
-    'workdir': None,
+  'workdir': None,
   'pidfile': None,
   'server_user': conf.SERVER_USER.get(),
   'server_group': conf.SERVER_GROUP.get(),