cm_environment.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import os
  17. import os.path
  18. import sys
  19. import logging
  20. import subprocess
  21. import re
  22. from hue_shared import which
  23. #logging.basicConfig()
  24. #logging = logging.getLogger(__name__)
  25. """
  26. Class to configure Hue environment from CM
  27. """
  28. def set_cm_environment():
  29. """
  30. Collect environment from CM supervisor
  31. """
  32. hue_config = {}
  33. hue_bin_dir = "/usr/lib/hue"
  34. hue_path = "/usr/lib/hue"
  35. parcel_name = "CDH"
  36. parcel_dir = "/opt/cloudera/parcels"
  37. dbengine = None
  38. cm_agent_process = subprocess.Popen('ps -ef | grep "[c]m agent\|[c]mf-agent" | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
  39. cm_agent_pid = cm_agent_process.communicate()[0].split('\n')[0]
  40. if cm_agent_pid != '':
  41. try:
  42. supervisor_process = subprocess.Popen('ps -ef | grep [s]upervisord | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
  43. supervisor_pid = supervisor_process.communicate()[0].split('\n')[0]
  44. cm_supervisor_dir = os.path.realpath('/proc/%s/cwd' % supervisor_pid)
  45. if supervisor_pid == '':
  46. logging.exception("This appears to be a CM enabled cluster and supervisord is not running")
  47. logging.exception("Make sure you are running as root and CM supervisord is running")
  48. sys.exit(1)
  49. except Exception, e:
  50. logging.exception("This appears to be a CM enabled cluster and supervisord is not running")
  51. logging.exception("Make sure you are running as root and CM supervisord is running")
  52. sys.exit(1)
  53. #Parse CM supervisor include file for Hue and set env vars
  54. cm_supervisor_dir = cm_supervisor_dir + '/include'
  55. cm_agent_run_dir = os.path.dirname(cm_supervisor_dir)
  56. hue_env_conf = None
  57. envline = None
  58. cm_hue_string = "HUE_SERVER"
  59. for file in os.listdir(cm_supervisor_dir):
  60. if cm_hue_string in file:
  61. hue_env_conf = file
  62. if not hue_env_conf == None:
  63. if os.path.isfile(cm_supervisor_dir + "/" + hue_env_conf):
  64. hue_env_conf_file = open(cm_supervisor_dir + "/" + hue_env_conf, "r")
  65. logging.info("Setting CM managed environment using supervisor include: %s" % hue_env_conf_file)
  66. for line in hue_env_conf_file:
  67. if "environment" in line:
  68. envline = line
  69. if "directory" in line:
  70. empty, hue_conf_dir = line.split("directory=")
  71. os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
  72. sys.path.append(os.environ["HUE_CONF_DIR"])
  73. if not envline == None:
  74. empty, environment = envline.split("environment=")
  75. for envvar in environment.split(","):
  76. if "HADOOP_C" in envvar or "PARCEL" in envvar or "DESKTOP" in envvar or "ORACLE" in envvar or "LIBRARY" in envvar or "CMF" in envvar:
  77. envkey, envval = envvar.split("=")
  78. envval = envval.replace("'", "").rstrip()
  79. if "LIBRARY" not in envkey:
  80. os.environ[envkey] = envval
  81. elif "LD_LIBRARY_PATH" not in os.environ.keys():
  82. os.environ[envkey] = envval
  83. if "PARCELS_ROOT" in os.environ:
  84. parcel_dir = os.environ["PARCELS_ROOT"]
  85. if "PARCEL_DIRNAMES" in os.environ:
  86. parcel_names = os.environ["PARCEL_DIRNAMES"].split(':')
  87. for parcel_name_temp in parcel_names:
  88. if parcel_name_temp.startswith("CDH"):
  89. parcel_name = parcel_name_temp
  90. if os.path.isdir("%s/%s/lib/hue" % (parcel_dir, parcel_name)):
  91. hue_path = "%s/%s/lib/hue" % (parcel_dir, parcel_name)
  92. hue_bin_dir = "%s/build/env/bin" % hue_path
  93. cloudera_config_script = None
  94. if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
  95. cloudera_config_script = '/usr/lib64/cmf/service/common/cloudera-config.sh'
  96. elif os.path.isfile('/opt/cloudera/cm-agent/service/common/cloudera-config.sh'):
  97. cloudera_config_script = '/opt/cloudera/cm-agent/service/common/cloudera-config.sh'
  98. JAVA_HOME = None
  99. if cloudera_config_script is not None:
  100. locate_java = subprocess.Popen(['bash', '-c', '. %s; locate_java_home' % cloudera_config_script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  101. for line in iter(locate_java.stdout.readline,''):
  102. if 'JAVA_HOME' in line:
  103. JAVA_HOME = line.rstrip().split('=')[1]
  104. if JAVA_HOME is not None:
  105. os.environ["JAVA_HOME"] = JAVA_HOME
  106. if "JAVA_HOME" not in os.environ:
  107. print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
  108. sys.exit(1)
  109. hue_config["LD_LIBRARY_PATH"] = None
  110. for line in open(os.environ["HUE_CONF_DIR"] + "/hue_safety_valve_server.ini"):
  111. if re.search("engine=", line):
  112. dbengine = line
  113. if dbengine is None:
  114. for line in open(os.environ["HUE_CONF_DIR"] + "/hue_safety_valve.ini"):
  115. if re.search("engine=", line):
  116. dbengine = line
  117. if dbengine is None:
  118. for line in open(os.environ["HUE_CONF_DIR"] + "/hue.ini"):
  119. if re.search("engine=", line):
  120. dbengine = line
  121. if dbengine is not None and "oracle" in dbengine.lower():
  122. #Make sure we set Oracle Client if configured
  123. if "LD_LIBRARY_PATH" not in os.environ.keys():
  124. if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
  125. for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
  126. if "ORACLE_INSTANT_CLIENT" in scm_script:
  127. if os.path.isfile(scm_script):
  128. oracle_source = subprocess.Popen(". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash")
  129. for line in oracle_source.communicate()[0].splitlines():
  130. if "LD_LIBRARY_PATH" in line:
  131. var, oracle_ld_path = line.split("=")
  132. os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
  133. if "LD_LIBRARY_PATH" not in os.environ.keys() or not os.path.isfile("%s/libclntsh.so.11.1" % os.environ["LD_LIBRARY_PATH"]):
  134. print "You are using Oracle for backend DB"
  135. if "LD_LIBRARY_PATH" in os.environ.keys():
  136. print "LD_LIBRARY_PATH set to %s" % os.environ["LD_LIBRARY_PATH"]
  137. print "LD_LIBRARY_PATH does not contain libclntsh.so.11.1"
  138. print "Please set LD_LIBRARY_PATH correctly and rerun"
  139. else:
  140. print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
  141. print "then it must be set, if not, you can ignore"
  142. print "Here is an exmple, ONLY INCLUDE ONE PATH and NO VARIABLES"
  143. print " export LD_LIBRARY_PATH=/path/to/instantclient"
  144. sys.exit(1)
  145. else:
  146. print "CM does not appear to be running on this server"
  147. print "If this is a CM managed cluster make sure the agent and supervisor are running"
  148. print "Running with /etc/hue/conf as the HUE_CONF_DIR"
  149. os.environ["HUE_CONF_DIR"] = "/etc/hue/conf"
  150. hue_config['hue_path'] = hue_path
  151. hue_config['hue_bin_dir'] = hue_bin_dir
  152. hue_config['HUE_CONF_DIR'] = os.environ["HUE_CONF_DIR"]
  153. hue_config['parcel_name'] = parcel_name
  154. hue_config['parcel_dir'] = parcel_dir
  155. if dbengine is not None and "oracle" in dbengine.lower():
  156. hue_config['LD_LIBRARY_PATH'] = os.environ["LD_LIBRARY_PATH"]
  157. return hue_config
  158. def reload_with_cm_env():
  159. try:
  160. from django.db.backends.oracle.base import Oracle_datetime
  161. except:
  162. os.environ["SKIP_RELOAD"] = "True"
  163. if 'LD_LIBRARY_PATH' in os.environ:
  164. logging.info("We need to reload the process to include any LD_LIBRARY_PATH changes")
  165. try:
  166. os.execv(sys.argv[0], sys.argv)
  167. except Exception, exc:
  168. logging.warn('Failed re-exec:', exc)
  169. sys.exit(1)
  170. def check_security():
  171. from hadoop import conf
  172. hdfs_config = conf.HDFS_CLUSTERS['default']
  173. security_enabled = False
  174. if hdfs_config.SECURITY_ENABLED.get():
  175. KLIST = which('klist')
  176. if KLIST is None:
  177. logging.exception("klist is required, please install and rerun")
  178. sys.exit(1)
  179. klist_cmd = '%s | grep "Default principal"' % KLIST
  180. klist_check = subprocess.Popen(klist_cmd, shell=True, stdout=subprocess.PIPE)
  181. klist_princ = klist_check.communicate()[0].split(': ')[1]
  182. if not 'hue/' in klist_princ:
  183. logging.exception("klist failed, please contact support: %s" % klist_princ)
  184. sys.exit(1)
  185. logging.info("Security enabled using principal: %s" % klist_princ)
  186. logging.info("You can imitate by running following export:")
  187. logging.info("OSRUN: export KRB5CCNAME=%s" % os.environ['KRB5CCNAME'])
  188. security_enabled = True
  189. return security_enabled