hue_collect_data.sh 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. #!/bin/bash
  2. #NOTE: This script requires curl, strace and lsof to be installed. It
  3. # must be run on the Hue server. Set HUE_USER to a user in Hue with
  4. # Superuser access to get thread dumps. Set HUE_PASSWORD to HUE_USER's
  5. # password.
  6. # Location for output files
  7. OUTPUT_DIR=/tmp/hue_collect_data
  8. # Location of Hue logs
  9. HUE_LOG_DIR=/var/log/hue
  10. # Please change USER to contain the user to login
  11. HUE_USER="admin"
  12. # Please change PASSWORD to contain the password for the above user
  13. HUE_PASSWORD="admin"
  14. # Set to true for more debug
  15. VERBOSE="false"
  16. #How many seconds of strace to capture on each run
  17. STRACE_WAIT=15
  18. #Number of data gathering runs
  19. RUNS=4
  20. #Time to wait between each run
  21. RUN_WAIT=30
  22. main()
  23. {
  24. if [[ ! ${USER} =~ .*root* ]]
  25. then
  26. echo "Script must be run as root: exiting"
  27. exit 1
  28. fi
  29. get_cm_process_dir
  30. #This is necessary to handle AD auth, doesn't seem to hurt non-ad auth
  31. #if they have multiple ldap servers or for some reason the drop down
  32. #at login says something other than "LDAP", then this must match the drop
  33. #down
  34. HUE_AUTH_SERVER="LDAP"
  35. DATE=$(date '+%Y%m%d-%H%M')
  36. OUTPUT_DIR=${OUTPUT_DIR}/${DATE}
  37. HOSTNAME=$(hostname)
  38. HUE_SERVER=${HOSTNAME}
  39. HUE_PORT=`grep http_port ${HUE_INI} | awk -F= '{print $2}'`
  40. if [[ -z ${HUE_PORT} ]]
  41. then
  42. HUE_PORT=8888
  43. fi
  44. SSL_CERT=`grep ssl_certificate ${HUE_INI} | awk -F= '{print $2}'`
  45. if [[ ! -z ${SSL_CERT} ]]
  46. then
  47. HUE_HTTP="https"
  48. else
  49. HUE_HTTP="http"
  50. fi
  51. HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
  52. HUE_THREADS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/desktop/debug/threads"
  53. HUE_USAGE_FILE=${OUTPUT_DIR}/cpu_mem_usage
  54. HUE_THREADS_FILE=${OUTPUT_DIR}/threads
  55. HUE_STRACE_FILE=${OUTPUT_DIR}/strace
  56. HUE_LSOF_FILE=${OUTPUT_DIR}/lsof
  57. COOKIE_JAR=${OUTPUT_DIR}/${USER}_cookie.jar
  58. echo "Making ${OUTPUT_DIR} if it does not exist"
  59. mkdir -p ${OUTPUT_DIR}
  60. hue_login
  61. echo "Gathering info:"
  62. for (( x=1; x<=${RUNS}; x++ ))
  63. do
  64. DATE=$(date '+%Y%m%d-%H%M%S')
  65. echo "DATE: ${DATE}"
  66. echo "Getting a thread dump:"
  67. do_curl \
  68. GET \
  69. "${HUE_THREADS_URL}" \
  70. -L -o ${HUE_THREADS_FILE}_${DATE}
  71. echo "Getting CPU and Memory usage"
  72. do_ps
  73. echo "PID CPU MEM MEM_MB" >> ${HUE_USAGE_FILE}_${DATE}
  74. echo "${PID} ${CPU} ${MEM} ${MEM_MB}" >> ${HUE_USAGE_FILE}_${DATE}
  75. echo "free -m results" >> ${HUE_USAGE_FILE}_${DATE}
  76. free -m >> ${HUE_USAGE_FILE}_${DATE}
  77. echo "Getting strace"
  78. do_strace \
  79. ${PID} \
  80. ${STRACE_WAIT} \
  81. -o ${HUE_STRACE_FILE}_${DATE} -T -t
  82. echo "Getting open connections"
  83. do_lsof \
  84. ${PID} \
  85. ${HUE_LSOF_FILE}_${DATE}
  86. sleep ${RUN_WAIT}
  87. done
  88. mkdir ${OUTPUT_DIR}/logs
  89. cp -pr ${HUE_LOG_DIR}/* ${OUTPUT_DIR}/logs
  90. echo "Collecting done, please zip ${OUTPUT_DIR} and upload to the ticket"
  91. }
  92. function do_curl() {
  93. METHOD=$1
  94. shift
  95. URL=$1
  96. shift
  97. ARGS=$@
  98. CURL=$(which curl)
  99. if [ -z ${COOKIE_JAR} ]
  100. then
  101. COOKIE_JAR=/tmp/cookie.jar
  102. fi
  103. if [ -f ${COOKIE_JAR} ]
  104. then
  105. CSRF_TOKEN=`grep ${HOSTNAME} ${COOKIE_JAR} | grep csrftoken | cut -f 7`
  106. fi
  107. if [ ! -f ${CURL} ]
  108. then
  109. echo "curl not found, unable to run any curl commands"
  110. else
  111. ${CURL} \
  112. ${CURL_OPTS} \
  113. -k \
  114. -e "${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/" \
  115. -b @${COOKIE_JAR} \
  116. -c ${COOKIE_JAR} \
  117. -H "X-CSRFToken: ${CSRF_TOKEN}" \
  118. -X ${METHOD} \
  119. -s \
  120. -f \
  121. ${URL} \
  122. ${ARGS}
  123. fi
  124. }
  125. function hue_login() {
  126. echo "Login to Hue to get Cookie:"
  127. do_curl \
  128. GET \
  129. "${HUE_PASS_URL}" \
  130. -L 2>&1 > /dev/null
  131. do_curl \
  132. POST \
  133. "${HUE_PASS_URL}" \
  134. -F username=${HUE_USER} -F password="${HUE_PASSWORD}" -F server="${HUE_AUTH_SERVER}" 2>&1 > /dev/null
  135. }
  136. function do_strace()
  137. {
  138. SPID=$1
  139. shift
  140. WAIT=$1
  141. shift
  142. ARGS=$@
  143. if [ -z ${LOG_FILE} ]
  144. then
  145. LOG_FILE=/tmp/lsof.log
  146. fi
  147. STRACE=$(which strace)
  148. if [ ! -f ${STRACE} ]
  149. then
  150. echo "strace not found, unable to collect strace info"
  151. else
  152. ${STRACE} -o ${LOG_FILE} -p ${SPID} ${ARGS} &
  153. sleep ${WAIT}
  154. kill $!
  155. fi
  156. }
  157. function do_lsof()
  158. {
  159. LPID=$1
  160. shift
  161. LOG_FILE=$1
  162. shift
  163. ARGS=$@
  164. if [ -z ${LOG_FILE} ]
  165. then
  166. LOG_FILE=/tmp/lsof.log
  167. fi
  168. LSOF=$(which lsof)
  169. if [ ! -f ${LSOF} ]
  170. then
  171. echo "lsof not found, unable to determine number of connections"
  172. else
  173. ${LSOF} -P -p ${LPID} ${ARGS} > ${LOG_FILE}
  174. fi
  175. }
  176. function do_ps()
  177. {
  178. PS_COMMAND=$(ps aux | grep [r]uncherrypyserver | awk '{print $6" "$2" "$3" "$12}')
  179. MEM=$(echo ${PS_COMMAND} | awk '{print $1}')
  180. MEM_MB=$(expr ${MEM} / 1024)
  181. PID=$(echo ${PS_COMMAND} | awk '{print $2}')
  182. CPU=$(echo ${PS_COMMAND} | awk '{print $3}')
  183. PROC=$(echo ${PS_COMMAND} | awk '{print $4}')
  184. }
  185. function get_cm_process_dir()
  186. {
  187. if [[ -d /var/run/cloudera-scm-agent/process ]]
  188. then
  189. HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
  190. else
  191. HUE_CONF_DIR=/etc/hue/conf
  192. fi
  193. HUE_INI=${HUE_CONF_DIR}/hue.ini
  194. }
  195. debug()
  196. {
  197. if [[ ! -z $VERBOSE ]]
  198. then
  199. echo "$1" >> ${LOG_FILE}
  200. fi
  201. }
  202. main "$@"