hue_collect_data.sh 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260
  1. #!/bin/bash
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #NOTE: This script requires curl, strace and lsof to be installed. It
  18. # must be run on the Hue server. Set HUE_USER to a user in Hue with
  19. # Superuser access to get thread dumps. Set HUE_PASSWORD to HUE_USER's
  20. # password.
  21. # Location for output files
  22. OUTPUT_DIR=/tmp/hue_collect_data
  23. # Location of Hue logs
  24. HUE_LOG_DIR=/var/log/hue
  25. # Please change USER to contain the user to login
  26. HUE_USER="admin"
  27. # Please change PASSWORD to contain the password for the above user
  28. HUE_PASSWORD="admin"
  29. # Set to true for more debug
  30. VERBOSE="false"
  31. #How many seconds of strace to capture on each run
  32. STRACE_WAIT=15
  33. #Number of data gathering runs
  34. RUNS=4
  35. #Time to wait between each run
  36. RUN_WAIT=30
  37. main()
  38. {
  39. if [[ ! ${USER} =~ .*root* ]]
  40. then
  41. echo "Script must be run as root: exiting"
  42. exit 1
  43. fi
  44. get_cm_process_dir
  45. #This is necessary to handle AD auth, doesn't seem to hurt non-ad auth
  46. #if they have multiple ldap servers or for some reason the drop down
  47. #at login says something other than "LDAP", then this must match the drop
  48. #down
  49. HUE_AUTH_SERVER="LDAP"
  50. DATE=$(date '+%Y%m%d-%H%M')
  51. OUTPUT_DIR=${OUTPUT_DIR}/${DATE}
  52. HOSTNAME=$(hostname)
  53. HUE_SERVER=${HOSTNAME}
  54. HUE_PORT=`grep http_port ${HUE_INI} | awk -F= '{print $2}'`
  55. if [[ -z ${HUE_PORT} ]]
  56. then
  57. HUE_PORT=8888
  58. fi
  59. SSL_CERT=`grep ssl_certificate ${HUE_INI} | awk -F= '{print $2}'`
  60. if [[ ! -z ${SSL_CERT} ]]
  61. then
  62. HUE_HTTP="https"
  63. else
  64. HUE_HTTP="http"
  65. fi
  66. HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
  67. HUE_THREADS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/desktop/debug/threads"
  68. HUE_USAGE_FILE=${OUTPUT_DIR}/cpu_mem_usage
  69. HUE_THREADS_FILE=${OUTPUT_DIR}/threads
  70. HUE_STRACE_FILE=${OUTPUT_DIR}/strace
  71. HUE_LSOF_FILE=${OUTPUT_DIR}/lsof
  72. COOKIE_JAR=${OUTPUT_DIR}/${USER}_cookie.jar
  73. echo "Making ${OUTPUT_DIR} if it does not exist"
  74. mkdir -p ${OUTPUT_DIR}
  75. do_ps
  76. hue_login
  77. echo "Gathering info:"
  78. for (( x=1; x<=${RUNS}; x++ ))
  79. do
  80. DATE=$(date '+%Y%m%d-%H%M%S')
  81. echo "DATE: ${DATE}"
  82. echo "Getting CPU and Memory usage"
  83. do_ps
  84. echo "PID CPU MEM MEM_MB" >> ${HUE_USAGE_FILE}_${DATE}
  85. echo "${PID} ${CPU} ${MEM} ${MEM_MB}" >> ${HUE_USAGE_FILE}_${DATE}
  86. echo "free -m results" >> ${HUE_USAGE_FILE}_${DATE}
  87. free -m >> ${HUE_USAGE_FILE}_${DATE}
  88. echo "Getting strace"
  89. do_strace \
  90. ${PID} \
  91. ${STRACE_WAIT} \
  92. -o ${HUE_STRACE_FILE}_${DATE} -T -t
  93. echo "Getting open connections"
  94. do_lsof \
  95. ${PID} \
  96. ${HUE_LSOF_FILE}_${DATE}
  97. echo "Getting a thread dump:"
  98. do_curl \
  99. GET \
  100. "${HUE_THREADS_URL}" \
  101. -L -o ${HUE_THREADS_FILE}_${DATE}
  102. sleep ${RUN_WAIT}
  103. done
  104. mkdir ${OUTPUT_DIR}/logs
  105. cp -pr ${HUE_LOG_DIR}/* ${OUTPUT_DIR}/logs
  106. echo "Collecting done, please zip ${OUTPUT_DIR} and upload to the ticket"
  107. }
  108. function do_curl() {
  109. METHOD=$1
  110. shift
  111. URL=$1
  112. shift
  113. ARGS=$@
  114. CURL=$(which curl)
  115. if [ -z ${COOKIE_JAR} ]
  116. then
  117. COOKIE_JAR=/tmp/cookie.jar
  118. fi
  119. if [ -f ${COOKIE_JAR} ]
  120. then
  121. CSRF_TOKEN=`grep ${HOSTNAME} ${COOKIE_JAR} | grep csrftoken | cut -f 7`
  122. fi
  123. if [ ! -f ${CURL} ]
  124. then
  125. echo "curl not found, unable to run any curl commands"
  126. else
  127. ${CURL} \
  128. ${CURL_OPTS} \
  129. -k \
  130. -e "${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/" \
  131. -b @${COOKIE_JAR} \
  132. -c ${COOKIE_JAR} \
  133. -H "X-CSRFToken: ${CSRF_TOKEN}" \
  134. -X ${METHOD} \
  135. -s \
  136. -f \
  137. ${URL} \
  138. ${ARGS}
  139. fi
  140. }
  141. function hue_login() {
  142. echo "Login to Hue to get Cookie:"
  143. do_curl \
  144. GET \
  145. "${HUE_PASS_URL}" \
  146. -L 2>&1 > /dev/null
  147. do_curl \
  148. POST \
  149. "${HUE_PASS_URL}" \
  150. -F username=${HUE_USER} -F password="${HUE_PASSWORD}" -F server="${HUE_AUTH_SERVER}" 2>&1 > /dev/null
  151. }
  152. function do_strace()
  153. {
  154. SPID=$1
  155. shift
  156. WAIT=$1
  157. shift
  158. ARGS=$@
  159. STRACE=$(which strace)
  160. if [ ! -f ${STRACE} ]
  161. then
  162. echo "strace not found, unable to collect strace info"
  163. else
  164. timeout ${WAIT}s ${STRACE} -p ${SPID} ${ARGS} &
  165. fi
  166. }
  167. function do_lsof()
  168. {
  169. LPID=$1
  170. shift
  171. LOG_FILE=$1
  172. shift
  173. ARGS=$@
  174. if [ -z ${LOG_FILE} ]
  175. then
  176. LOG_FILE=/tmp/lsof.log
  177. fi
  178. LSOF=$(which lsof)
  179. if [ ! -f ${LSOF} ]
  180. then
  181. echo "lsof not found, unable to determine number of connections"
  182. else
  183. ${LSOF} -P -p ${LPID} ${ARGS} > ${LOG_FILE}
  184. fi
  185. }
  186. function do_ps()
  187. {
  188. PS_COMMAND=$(ps aux | grep [r]uncherrypyserver | awk '{print $6" "$2" "$3" "$12}')
  189. MEM=$(echo ${PS_COMMAND} | awk '{print $1}')
  190. PID=$(echo ${PS_COMMAND} | awk '{print $2}')
  191. CPU=$(echo ${PS_COMMAND} | awk '{print $3}')
  192. PROC=$(echo ${PS_COMMAND} | awk '{print $4}')
  193. if [[ ! ${PID} == ?(-)+([0-9]) ]]
  194. then
  195. echo "Unable to get PID from Process, either Hue is not running on this host or Hue is not using CherryPy server"
  196. exit 1
  197. fi
  198. MEM_MB=$(expr ${MEM} / 1024)
  199. }
  200. function get_cm_process_dir()
  201. {
  202. if [[ -d /var/run/cloudera-scm-agent/process ]]
  203. then
  204. HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
  205. else
  206. HUE_CONF_DIR=/etc/hue/conf
  207. fi
  208. HUE_INI=${HUE_CONF_DIR}/hue.ini
  209. }
  210. debug()
  211. {
  212. if [[ ! -z $VERBOSE ]]
  213. then
  214. echo "$1" >> ${LOG_FILE}
  215. fi
  216. }
  217. main "$@"