浏览代码

HUE-2687 [core] add script to collect data for troubleshooting

Chris Conner 10 年之前
父节点
当前提交
e2115b4e2b
共有 1 个文件被更改,包括 241 次插入0 次删除
  1. 241 0
      tools/troubleshooting/hue_collect_data.sh

+ 241 - 0
tools/troubleshooting/hue_collect_data.sh

@@ -0,0 +1,241 @@
+#!/bin/bash
+
+#NOTE: This script requires curl, strace and lsof to be installed. It
+# must be run on the Hue server. Set HUE_USER to a user in Hue with 
+# Superuser access to get thread dumps.  Set HUE_PASSWORD to HUE_USER's 
+# password.
+
+# Location for output files
+OUTPUT_DIR=/tmp/hue_collect_data
+
+# Location of Hue logs
+
+HUE_LOG_DIR=/var/log/hue
+
+# Please change USER to contain the user to login
+HUE_USER="admin"
+
+# Please change PASSWORD to contain the password for the above user
+HUE_PASSWORD="admin"
+
+# Set to true for more debug
+VERBOSE="false"
+
+#How many seconds of strace to capture on each run
+STRACE_WAIT=15
+
+#Number of data gathering runs
+RUNS=4
+
+#Time to wait between each run
+RUN_WAIT=30
+
+main()
+{
+
+if [[ ! ${USER} =~ .*root* ]]
+then
+   echo "Script must be run as root: exiting"
+   exit 1
+fi
+
+get_cm_process_dir
+
+#This is necessary to handle AD auth, doesn't seem to hurt non-ad auth
+#if they have multiple ldap servers or for some reason the drop down
+#at login says something other than "LDAP", then this must match the drop
+#down
+HUE_AUTH_SERVER="LDAP"
+DATE=$(date '+%Y%m%d-%H%M')
+OUTPUT_DIR=${OUTPUT_DIR}/${DATE}
+HOSTNAME=$(hostname)
+HUE_SERVER=${HOSTNAME}
+HUE_PORT=`grep http_port ${HUE_INI} | awk -F= '{print $2}'`
+if [[ -z ${HUE_PORT} ]]
+then
+   HUE_PORT=8888
+fi
+SSL_CERT=`grep ssl_certificate ${HUE_INI} | awk -F= '{print $2}'`
+if [[ ! -z ${SSL_CERT} ]]
+then
+   HUE_HTTP="https"
+else
+   HUE_HTTP="http"
+fi
+HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_THREADS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/desktop/debug/threads"
+HUE_USAGE_FILE=${OUTPUT_DIR}/cpu_mem_usage
+HUE_THREADS_FILE=${OUTPUT_DIR}/threads
+HUE_STRACE_FILE=${OUTPUT_DIR}/strace
+HUE_LSOF_FILE=${OUTPUT_DIR}/lsof
+COOKIE_JAR=${OUTPUT_DIR}/${USER}_cookie.jar
+
+echo "Making ${OUTPUT_DIR} if it does not exist"
+mkdir -p ${OUTPUT_DIR}
+
+hue_login
+echo "Gathering info:"
+for (( x=1; x<=${RUNS}; x++ ))
+do
+   DATE=$(date '+%Y%m%d-%H%M%S')
+   echo "DATE: ${DATE}"
+   echo "Getting a thread dump:"
+   do_curl \
+	GET \
+	"${HUE_THREADS_URL}" \
+	-L -o ${HUE_THREADS_FILE}_${DATE}
+
+   echo "Getting CPU and Memory usage"
+   do_ps
+   echo "PID CPU MEM MEM_MB" >> ${HUE_USAGE_FILE}_${DATE}
+   echo "${PID} ${CPU} ${MEM} ${MEM_MB}" >> ${HUE_USAGE_FILE}_${DATE}
+   echo "free -m results" >> ${HUE_USAGE_FILE}_${DATE}
+   free -m >> ${HUE_USAGE_FILE}_${DATE}
+
+   echo "Getting strace"
+   do_strace \
+        ${PID} \
+        ${STRACE_WAIT} \
+        -o ${HUE_STRACE_FILE}_${DATE} -T -t
+   echo "Getting open connections"
+   do_lsof \
+        ${PID} \
+        ${HUE_LSOF_FILE}_${DATE}
+
+   sleep ${RUN_WAIT}
+done
+
+mkdir ${OUTPUT_DIR}/logs
+cp -pr ${HUE_LOG_DIR}/* ${OUTPUT_DIR}/logs 
+
+echo "Collecting done, please zip ${OUTPUT_DIR} and upload to the ticket"
+}
+
+function do_curl() {
+
+   METHOD=$1
+   shift
+   URL=$1
+   shift
+   ARGS=$@
+
+   CURL=$(which curl)
+   if [ -z ${COOKIE_JAR} ]
+   then
+      COOKIE_JAR=/tmp/cookie.jar
+   fi
+   if [ -f ${COOKIE_JAR} ]
+   then
+      CSRF_TOKEN=`grep ${HOSTNAME} ${COOKIE_JAR} | grep csrftoken | cut -f 7`
+   fi
+   if [ ! -f ${CURL} ]
+   then
+      echo "curl not found, unable to run any curl commands"
+   else
+      ${CURL} \
+         ${CURL_OPTS} \
+         -k \
+         -e "${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/" \
+         -b @${COOKIE_JAR} \
+         -c ${COOKIE_JAR} \
+         -H "X-CSRFToken: ${CSRF_TOKEN}" \
+         -X ${METHOD} \
+         -s \
+         -f \
+         ${URL} \
+         ${ARGS}
+   fi
+
+}
+
+function hue_login() {
+   echo "Login to Hue to get Cookie:"
+   do_curl \
+	GET \
+	"${HUE_PASS_URL}" \
+	-L 2>&1 > /dev/null
+
+   do_curl \
+        POST \
+        "${HUE_PASS_URL}" \
+        -F username=${HUE_USER} -F password="${HUE_PASSWORD}" -F server="${HUE_AUTH_SERVER}" 2>&1 > /dev/null
+}
+
+function do_strace()
+{
+   SPID=$1
+   shift
+   WAIT=$1
+   shift
+   ARGS=$@
+
+   if [ -z ${LOG_FILE} ]
+   then
+      LOG_FILE=/tmp/lsof.log
+   fi
+
+   STRACE=$(which strace)
+   if [ ! -f ${STRACE} ]
+   then
+      echo "strace not found, unable to collect strace info"
+   else
+      ${STRACE} -o ${LOG_FILE} -p ${SPID} ${ARGS} &
+      sleep ${WAIT}
+      kill $!
+   fi
+}
+
+function do_lsof()
+{
+   LPID=$1
+   shift
+   LOG_FILE=$1
+   shift
+   ARGS=$@
+
+   if [ -z ${LOG_FILE} ]
+   then
+      LOG_FILE=/tmp/lsof.log
+   fi
+
+   LSOF=$(which lsof)
+   if [ ! -f ${LSOF} ]
+   then
+      echo "lsof not found, unable to determine number of connections"
+   else
+      ${LSOF} -P -p ${LPID} ${ARGS} > ${LOG_FILE}
+   fi
+}
+
+function do_ps()
+{
+   PS_COMMAND=$(ps aux | grep [r]uncherrypyserver | awk '{print $6" "$2" "$3" "$12}')
+   MEM=$(echo ${PS_COMMAND} | awk '{print $1}')
+   MEM_MB=$(expr ${MEM} / 1024)
+   PID=$(echo ${PS_COMMAND} | awk '{print $2}')
+   CPU=$(echo ${PS_COMMAND} | awk '{print $3}')
+   PROC=$(echo ${PS_COMMAND} | awk '{print $4}')
+}
+
+function get_cm_process_dir()
+{
+   if [[ -d /var/run/cloudera-scm-agent/process ]]
+   then
+      HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
+   else
+      HUE_CONF_DIR=/etc/hue/conf
+   fi
+   HUE_INI=${HUE_CONF_DIR}/hue.ini
+}
+
+debug()
+{
+
+  if [[ ! -z $VERBOSE ]]
+  then
+    echo "$1" >> ${LOG_FILE}
+  fi
+
+}
+
+main "$@"