hue_collect_data.sh 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. #!/bin/bash
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. # NOTE: This script requires curl, strace and lsof to be installed. It
  18. # must be run on the Hue server. Set HUE_USER to a user in Hue with
  19. # Superuser access to get thread dumps. Set HUE_PASSWORD to HUE_USER's
  20. # password.
  21. # Please change USER to contain the user to login
  22. HUE_USER="admin"
  23. # Please change PASSWORD to contain the password for the above user
  24. HUE_PASSWORD="admin"
  25. #parse command line arguments
  26. parse_arguments()
  27. {
  28. # Test that we're using compatible getopt version.
  29. getopt -T > /dev/null
  30. if [[ $? -ne 4 ]]; then
  31. echo "Incompatible getopt version."
  32. exit 1
  33. fi
  34. # Parse short and long option parameters.
  35. STRACE_WAIT=15
  36. RUNS=4
  37. RUN_WAIT=30
  38. OUTPUT_DIR_BASE=/tmp/hue_collect_data
  39. COLLECT_STRACE=true
  40. COLLECT_LSOF=true
  41. COLLECT_NETSTAT=true
  42. COLLECT_THREADS=
  43. #This is necessary to handle AD auth, doesn't seem to hurt non-ad auth
  44. #if they have multiple ldap servers or for some reason the drop down
  45. #at login says something other than "LDAP", then this must match the drop
  46. #down
  47. HUE_AUTH_SERVER="LDAP"
  48. COLLECT_INSTANCE=true
  49. GETOPT=`getopt -n $0 -o s,l,n,w:,r:,t,u:,p:,i,o:,d:,y:,a:,h \
  50. -l strace,,lsof,netstat,wait:,runs:,threads,hueuser:,huepass:,instance,outdir:,huelog:,swait:,authserver:,help \
  51. -- "$@"`
  52. eval set -- "$GETOPT"
  53. while true;
  54. do
  55. case "$1" in
  56. -s|--strace)
  57. COLLECT_STRACE=
  58. shift
  59. ;;
  60. -y|--swait)
  61. STRACE_WAIT=$2
  62. shift 2
  63. ;;
  64. -l|--lsof)
  65. COLLECT_LSOF=
  66. shift
  67. ;;
  68. -n|--netstat)
  69. COLLECT_NETSTAT=
  70. shift
  71. ;;
  72. -t|--threads)
  73. COLLECT_THREADS=$1
  74. shift
  75. ;;
  76. -u|--hueuser)
  77. HUE_USER=$2
  78. shift 2
  79. ;;
  80. -p|--huepass)
  81. HUE_PASSWORD=$2
  82. shift 2
  83. ;;
  84. -a|--authserver)
  85. HUE_AUTH_SERVER=$2
  86. shift 2
  87. ;;
  88. -i|--instance)
  89. COLLECT_INSTANCE=
  90. shift
  91. ;;
  92. -w|--wait)
  93. RUN_WAIT=$2
  94. shift 2
  95. ;;
  96. -r|--runs)
  97. RUNS=$2
  98. shift 2
  99. ;;
  100. -o|--outdir)
  101. OUTPUT_DIR_BASE=$2
  102. shift 2
  103. ;;
  104. -d|--huelog)
  105. HUE_LOG_DIR=$2
  106. shift 2
  107. ;;
  108. --)
  109. shift
  110. break
  111. ;;
  112. *)
  113. usage
  114. exit 1
  115. ;;
  116. esac
  117. done
  118. #
  119. }
  120. usage()
  121. {
  122. cat << EOF
  123. usage: $0 [options]
  124. Collects troubleshooting data from Hue server:
  125. OPTIONS
  126. -s|--strace Setting this will disable collecting strace, default
  127. strace will be collected
  128. -y|--swait <secs> Interval to wait before stopping strace - default 15.
  129. -l|--lsof Setting this will disable collecting lsof, default
  130. lsof will be collected
  131. -n|--netstat Setting this will disable collecting netstat, default
  132. netstat will be collected
  133. -t|--threads This will enable thread collection, default threads
  134. will NOT be collected
  135. -u|--hueuser Hue username for collecting threads - must be admin user - default admin.
  136. -p|--huepass Hue password for collecting threads - default admin.
  137. -a|--authserver This is the Ldap auth server name in the hue.ini if using
  138. multiple ldap servers for auth. Must be set to the auth
  139. server that "hueuser" belongs to.
  140. -i|--instance Setting this will disable collecting CM Hue configs, default
  141. CM Hue configs will be collected
  142. -w|--wait <secs> Seconds to wait between runs of collecting data - default 30.
  143. -r|--runs <numruns> Number of times to collect data - default 4.
  144. -o|--outdir <outdir> Location to dump collected data - default /tmp/hue_collect_data.
  145. -h|--help Show this message.
  146. EOF
  147. }
  148. main()
  149. {
  150. parse_arguments "$@"
  151. if [[ ! ${USER} =~ .*root* ]]
  152. then
  153. echo "Script must be run as root: exiting"
  154. exit 1
  155. fi
  156. AGENT_PROCESS_DIR="/var/run/cloudera-scm-agent/process"
  157. START_DATE=$(date '+%Y%m%d-%H%M')
  158. MKDIR="mkdir -p"
  159. for PID in `ps aux | grep [r]uncherrypyserver | awk '{print $2}'`
  160. do
  161. if [[ ! ${PID} == ?(-)+([0-9]) ]]
  162. then
  163. echo "Unable to get PID from Process, either Hue is not running on this host or Hue is not using CherryPy server"
  164. exit 1
  165. fi
  166. OUTPUT_DIR_DATE=${OUTPUT_DIR_BASE}/${START_DATE}
  167. OUTPUT_DIR=${OUTPUT_DIR_DATE}/${PID}
  168. HUE_USAGE_FILE=${OUTPUT_DIR}/cpu_mem_usage/cpu_mem_usage
  169. HUE_THREADS_FILE=${OUTPUT_DIR}/threads/threads
  170. HUE_STRACE_FILE=${OUTPUT_DIR}/strace/strace
  171. HUE_LSOF_FILE=${OUTPUT_DIR}/lsof/lsof
  172. HUE_ENVIRON_FILE=${OUTPUT_DIR}/environ/environ
  173. HUE_CMDLINE_FILE=${OUTPUT_DIR}/cmdline/cmdline
  174. HUE_LIMITS_FILE=${OUTPUT_DIR}/limits/limits
  175. HUE_NETSTAT_FILE=${OUTPUT_DIR}/netstat/netstat
  176. HUE_SUDO=${OUTPUT_DIR}/sudo
  177. HUE_CONFS=${OUTPUT_DIR}/confs
  178. COOKIE_JAR=${OUTPUT_DIR}/${USER}_cookie.jar
  179. HOSTNAME=$(hostname)
  180. HUE_SERVER=${HOSTNAME}
  181. echo "Making ${OUTPUT_DIR} if it does not exist"
  182. ${MKDIR} ${OUTPUT_DIR}
  183. get_cm_process_dir ${PID}
  184. if [[ -f ${CM_PROPS} ]]
  185. then
  186. HUE_LOG_DIR=`grep log_dir ${CM_PROPS} | awk '{print $3}'`
  187. else
  188. HUE_LOG_DIR=/var/log/hue
  189. fi
  190. HUE_PORT=`grep http_port ${HUE_INI} | awk -F= '{print $2}'`
  191. if [[ -z ${HUE_PORT} ]]
  192. then
  193. HUE_PORT=8888
  194. fi
  195. SSL_CERT=`grep ssl_certificate ${HUE_INI} | awk -F= '{print $2}'`
  196. if [[ ! -z ${SSL_CERT} ]]
  197. then
  198. HUE_HTTP="https"
  199. else
  200. HUE_HTTP="http"
  201. fi
  202. HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
  203. HUE_THREADS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/desktop/debug/threads"
  204. if [[ ! -z ${COLLECT_THREADS} ]]
  205. then
  206. hue_login
  207. fi
  208. echo "Gathering info:"
  209. for (( x=1; x<=${RUNS}; x++ ))
  210. do
  211. DATE=$(date '+%Y%m%d-%H%M%S')
  212. echo "DATE: ${DATE}"
  213. echo "Getting CPU and Memory usage"
  214. do_ps ${PID}
  215. ${MKDIR} `dirname ${HUE_USAGE_FILE}`
  216. echo "PID CPU MEM MEM_MB" >> ${HUE_USAGE_FILE}_${DATE}
  217. echo "${PID} ${CPU} ${MEM} ${MEM_MB}" >> ${HUE_USAGE_FILE}_${DATE}
  218. echo "free -m results" >> ${HUE_USAGE_FILE}_${DATE}
  219. free -m >> ${HUE_USAGE_FILE}_${DATE}
  220. if [[ ${COLLECT_NETSTAT} ]]
  221. then
  222. echo "Gathering netstat info"
  223. ${MKDIR} `dirname ${HUE_NETSTAT_FILE}`
  224. netstat -anp | grep ${PID} >> ${HUE_NETSTAT_FILE}_${DATE}
  225. fi
  226. if [[ ${COLLECT_STRACE} ]]
  227. then
  228. echo "Getting strace"
  229. ${MKDIR} `dirname ${HUE_STRACE_FILE}`
  230. do_strace \
  231. ${PID} \
  232. ${STRACE_WAIT} \
  233. -o ${HUE_STRACE_FILE}_${DATE} -T -t
  234. fi
  235. if [[ ${COLLECT_LSOF} ]]
  236. then
  237. echo "Getting open connections"
  238. ${MKDIR} `dirname ${HUE_LSOF_FILE}`
  239. do_lsof \
  240. ${PID} \
  241. ${HUE_LSOF_FILE}_${DATE}
  242. fi
  243. if [[ ! -z ${COLLECT_THREADS} ]]
  244. then
  245. echo "Getting a thread dump:"
  246. ${MKDIR} `dirname ${HUE_THREADS_FILE}`
  247. do_curl \
  248. GET \
  249. "${HUE_THREADS_URL}" \
  250. -L -o ${HUE_THREADS_FILE}_${DATE}
  251. fi
  252. sleep ${RUN_WAIT}
  253. done
  254. if [[ ! -z ${COLLECT_INSTANCE} ]]
  255. then
  256. echo "Gathering CM config instances"
  257. do_instances ${HUE_PORT}
  258. fi
  259. echo "Gathering process info"
  260. ${MKDIR} `dirname ${HUE_ENVIRON_FILE}`
  261. strings /proc/${PID}/environ >> ${HUE_ENVIRON_FILE}_${DATE}
  262. ${MKDIR} `dirname ${HUE_CMDLINE_FILE}`
  263. strings /proc/${PID}/cmdline >> ${HUE_CMDLINE_FILE}_${DATE}
  264. ${MKDIR} `dirname ${HUE_LIMITS_FILE}`
  265. strings /proc/${PID}/limits >> ${HUE_LIMITS_FILE}_${DATE}
  266. ${MKDIR} ${OUTPUT_DIR}/logs
  267. cp -pr ${HUE_LOG_DIR}/* ${OUTPUT_DIR}/logs
  268. done
  269. echo "Gathering environment info"
  270. do_sudo \
  271. ${HUE_SUDO} \
  272. ${DATE}
  273. if [[ ! -z ${STRACE_PID} ]]
  274. then
  275. echo "strace still running, waiting for it to complete: ${STRACE_PID}"
  276. wait ${STRACE_PID}
  277. fi
  278. echo "Collecting done, please zip ${OUTPUT_DIR_DATE} and upload to the ticket"
  279. }
  280. function do_curl() {
  281. METHOD=$1
  282. shift
  283. URL=$1
  284. shift
  285. ARGS=$@
  286. CURL=$(which curl)
  287. if [ -z ${COOKIE_JAR} ]
  288. then
  289. COOKIE_JAR=/tmp/cookie.jar
  290. fi
  291. if [ -f ${COOKIE_JAR} ]
  292. then
  293. CSRF_TOKEN=`grep ${HOSTNAME} ${COOKIE_JAR} | grep csrftoken | cut -f 7`
  294. fi
  295. if [ ! -f ${CURL} ]
  296. then
  297. echo "curl not found, unable to run any curl commands"
  298. else
  299. ${CURL} \
  300. ${CURL_OPTS} \
  301. -k \
  302. -e "${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/" \
  303. -b @${COOKIE_JAR} \
  304. -c ${COOKIE_JAR} \
  305. -H "X-CSRFToken: ${CSRF_TOKEN}" \
  306. -X ${METHOD} \
  307. -s \
  308. -f \
  309. ${URL} \
  310. ${ARGS}
  311. fi
  312. }
  313. function hue_login() {
  314. echo "Login to Hue to get Cookie:"
  315. do_curl \
  316. GET \
  317. "${HUE_PASS_URL}" \
  318. -L 2>&1 > /dev/null
  319. do_curl \
  320. POST \
  321. "${HUE_PASS_URL}" \
  322. -F username=${HUE_USER} -F password="${HUE_PASSWORD}" -F server="${HUE_AUTH_SERVER}" 2>&1 > /dev/null
  323. }
  324. function do_strace()
  325. {
  326. SPID=$1
  327. shift
  328. WAIT=$1
  329. shift
  330. ARGS=$@
  331. STRACE=$(which strace)
  332. if [ ! -f ${STRACE} ]
  333. then
  334. echo "strace not found, unable to collect strace info"
  335. else
  336. if [[ ! -z ${STRACE_PID} ]]
  337. then
  338. echo "strace still running, waiting for it to complete: ${STRACE_PID}"
  339. wait ${STRACE_PID}
  340. fi
  341. timeout ${WAIT}s ${STRACE} -f -v -p ${SPID} ${ARGS} &
  342. STRACE_PID=$!
  343. fi
  344. }
  345. function do_lsof()
  346. {
  347. LPID=$1
  348. shift
  349. LOG_FILE=$1
  350. shift
  351. ARGS=$@
  352. if [ -z ${LOG_FILE} ]
  353. then
  354. LOG_FILE=/tmp/lsof.log
  355. fi
  356. LSOF=$(which lsof)
  357. if [ ! -f ${LSOF} ]
  358. then
  359. echo "lsof not found, unable to determine number of connections"
  360. else
  361. ${LSOF} -P -p ${LPID} ${ARGS} > ${LOG_FILE}
  362. fi
  363. }
  364. function do_ps()
  365. {
  366. PID=$1
  367. shift
  368. PS_COMMAND=$(ps aux | grep ${PID} | grep [r]uncherrypyserver | tail -1 | awk '{print $6" "$2" "$3" "$12}')
  369. MEM=$(echo ${PS_COMMAND} | awk '{print $1}')
  370. CPU=$(echo ${PS_COMMAND} | awk '{print $3}')
  371. PROC=$(echo ${PS_COMMAND} | awk '{print $4}')
  372. MEM_MB=$(expr ${MEM} / 1024)
  373. }
  374. function do_sudo()
  375. {
  376. HUE_SUDO=$1
  377. shift
  378. DATE=$1
  379. shift
  380. ARGS=$@
  381. sudo -u hue /bin/bash -c "/usr/bin/env" >> ${HUE_SUDO}_env_${DATE}
  382. sudo -u hue /bin/bash -c "/usr/bin/env which python2.6" >> ${HUE_SUDO}_python_${DATE}
  383. }
  384. function get_cm_process_dir()
  385. {
  386. PID=$1
  387. shift
  388. if [[ -d ${AGENT_PROCESS_DIR} ]]
  389. then
  390. HUE_CONF_DIR=`strings /proc/${PID}/environ | grep HUE_CONF_DIR | awk -F\= '{print $2}'`
  391. else
  392. HUE_CONF_DIR=/etc/hue/conf
  393. fi
  394. HUE_INI=${HUE_CONF_DIR}/hue.ini
  395. CM_PROPS=${HUE_CONF_DIR}/cloudera-monitor.properties
  396. }
  397. function do_instances()
  398. {
  399. HUE_PORT=$1
  400. ${MKDIR} ${HUE_CONFS}
  401. if [[ -d ${AGENT_PROCESS_DIR} ]]
  402. then
  403. for x in `find ${AGENT_PROCESS_DIR}/*hue-HUE_SERVER -name "hue.ini" -exec grep -H ${HUE_PORT} {} \; | awk -F\/ '{print $6}' | sort -n | tail -3`
  404. do
  405. cp -pr ${AGENT_PROCESS_DIR}/$x ${HUE_CONFS}
  406. done
  407. else
  408. cp -pr /etc/hue/conf ${HUE_CONFS}/0-hue-HUE_SERVER
  409. fi
  410. for x in `find ${HUE_CONFS} -name "*" -type f`
  411. do
  412. sed -i "/password/Id" $x
  413. done
  414. }
  415. main "$@"