hue_mem_cron_cm.sh 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. #!/bin/bash
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #This script will check CM to see if Hue is using too much memory
  18. #then will restart if necessary.
  19. CM_HOSTNAME="cdh53-1.qa.test.com"
  20. CM_PORT="7180"
  21. CM_USERNAME="admin"
  22. CM_PASSWORD="admin"
  23. KILL_ME=5000 #This is the number of MB at which it will kill.
  24. #Starting with 5000(5gb)
  25. VERBOSE=true #true then this writes out the proc info each time it runs, leave blank
  26. #to only write out when we kill the process
  27. LOG_FILE=/var/log/hue/`basename "$0" | awk -F\. '{print $1}'`.log
  28. ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1, we only keep
  29. #2 log files, so 20MB max
  30. TMP_LOCATION=/tmp/hue_mem_cron
  31. main()
  32. {
  33. DATE=`date '+%Y%m%d-%H%M'`
  34. YEAR=`date '+%Y'`
  35. MONTH=`date '+%m'`
  36. DAY=`date '+%d'`
  37. HOUR=`date '+%H'`
  38. MINUTE=`date '+%M'`
  39. YEAR_PRIOR=`date --date='1 minutes ago' '+%Y'`
  40. MONTH_PRIOR=`date --date='1 minutes ago' '+%m'`
  41. DAY_PRIOR=`date --date='1 minutes ago' '+%d'`
  42. HOUR_PRIOR=`date --date='1 minutes ago' '+%H'`
  43. MINUTE_PRIOR=`date --date='1 minutes ago' '+%M'`
  44. MEM_JSON_FILE=${TMP_LOCATION}/mem.json
  45. MB_BYTES="1048576"
  46. mkdir -p ${TMP_LOCATION}
  47. if [[ -f ${LOG_FILE} ]]
  48. then
  49. LOG_SIZE=`du -sm ${LOG_FILE} | awk '{print $1}'`
  50. if [[ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]]
  51. then
  52. mv ${LOG_FILE} ${LOG_FILE}.1
  53. fi
  54. fi
  55. MEM_API_URL="/api/v6/timeseries?query=select+mem_rss+where+roleType+%3D+HUE_SERVER&contentType=application%2Fjson&from=${YEAR_PRIOR}-${MONTH_PRIOR}-${DAY_PRIOR}T${HOUR_PRIOR}%3A${MINUTE_PRIOR}%3A00.000Z&to=${YEAR}-${MONTH}-${DAY}T${HOUR}%3A${MINUTE}%3A00.000Z"
  56. #Get memory usage for all Hue roles:
  57. curl -X GET -u ${CM_USERNAME}:${CM_PASSWORD} -i -o ${MEM_JSON_FILE} "http://${CM_HOSTNAME}:${CM_PORT}${MEM_API_URL}"
  58. while read -r LINE
  59. do
  60. if [[ ${LINE} =~ .*clusterName* ]]
  61. then
  62. CLUSTERNAME=`echo ${LINE} | awk -F\" '{print $4}'`
  63. fi
  64. if [[ ${LINE} =~ .*serviceName* ]]
  65. then
  66. SERVICENAME=`echo ${LINE} | awk -F\" '{print $4}'`
  67. fi
  68. if [[ ${LINE} =~ .*roleName* ]]
  69. then
  70. ROLENAME=`echo ${LINE} | awk -F\" '{print $4}'`
  71. fi
  72. if [[ ${LINE} =~ .*value* ]]
  73. then
  74. MEM=`echo ${LINE} | awk '{print $3}' | awk -F, '{print $1}'`
  75. MEM=`printf "%.f" $MEM` # convert from scientific to decimal
  76. MEM_MB=`expr ${MEM} / ${MB_BYTES}`
  77. debug "${DATE} - ROLENAME: ${ROLENAME} - MEM: ${MEM} - MEM_MB: ${MEM_MB}"
  78. if [ ${MEM_MB} -gt ${KILL_ME} ]
  79. then
  80. echo "${DATE} - Restart the Hue Process: Too much memory: ${MEM_MB} : ROLENAME: ${ROLENAME}" >> ${LOG_FILE}
  81. RESTART_API_URL="/api/v8/clusters/${CLUSTERNAME}/services/${SERVICENAME}/roleCommands/restart"
  82. curl -X POST -u ${CM_USERNAME}:${CM_PASSWORD} -i -H "content-type:application/json" -d "{\"items\" : [\"${ROLENAME}\"]}" "http://${CM_HOSTNAME}:${CM_PORT}${RESTART_API_URL}"
  83. exit 0
  84. fi
  85. fi
  86. done < <(cat ${MEM_JSON_FILE})
  87. }
  88. debug()
  89. {
  90. if [[ ! -z $VERBOSE ]]
  91. then
  92. echo "$1" >> ${LOG_FILE}
  93. fi
  94. }
  95. main "$@"