hue_history_cron.sh 2.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677
  1. #!/bin/bash
  2. #
  3. #Cleans up old oozie workflow and beeswax savedqueries to
  4. #prevent the DB from getting too large.
  5. PARCEL_DIR=/opt/cloudera/parcels/CDH
  6. LOG_FILE=/var/log/hue_history_cron.log
  7. LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
  8. LOG_ROTATE_COUNT=2 #number of log files, so 20MB max
  9. DATE=`date '+%Y%m%d-%H%M'`
  10. KEEP_DAYS=7 #Number of days of beeswax and oozie history to keep
  11. if [ ! -d "/usr/lib/hadoop" ]
  12. then
  13. CDH_HOME=$PARCEL_DIR
  14. else
  15. CDH_HOME=/usr
  16. fi
  17. if [ -d "/var/run/cloudera-scm-agent/process" ]
  18. then
  19. HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
  20. else
  21. HUE_CONF_DIR="/etc/hue/conf"
  22. fi
  23. if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
  24. then
  25. COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
  26. else
  27. COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
  28. fi
  29. export CDH_HOME HUE_CONF_DIR COMMAND
  30. ${COMMAND} <<EOF
  31. from beeswax.models import SavedQuery
  32. from datetime import date, timedelta
  33. from oozie.models import Workflow
  34. import logging
  35. import logging.handlers
  36. import sys
  37. LOGFILE="${LOG_FILE}"
  38. keepDays = ${KEEP_DAYS}
  39. log = logging.getLogger('')
  40. log.setLevel(logging.INFO)
  41. format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
  42. fh = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes=(1048576*${LOG_ROTATE_SIZE}), backupCount=${LOG_ROTATE_COUNT})
  43. fh.setFormatter(format)
  44. log.addHandler(fh)
  45. log.info('HUE_CONF_DIR: ${HUE_CONF_DIR}')
  46. log.info("Cleaning up anything in the Hue tables oozie*, desktop* and beeswax* older than ${KEEP_DAYS} old")
  47. savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))
  48. count = len(savedQuerys)
  49. log.info("SavedQuery count is: %s" % count)
  50. savedQuerys.delete()
  51. savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))
  52. count = len(savedQuerys)
  53. log.info("SavedQuery new count is: %s" % count)
  54. totalWorkflows = len(Workflow.objects.filter(is_trashed=True, last_modified__lte=date.today() - timedelta(days=keepDays)))
  55. loopCount = 1
  56. maxCount = 1000
  57. log.info("workflows left: %s" % totalWorkflows)
  58. log.info("Looping through workflows")
  59. for w in Workflow.objects.filter(is_trashed=True, last_modified__lte=date.today() - timedelta(days=keepDays)):
  60. w.delete(skip_trash=True)
  61. loopCount += 1
  62. if (loopCount == maxCount):
  63. totalWorkflows = totalWorkflows - maxCount
  64. loopCount = 1
  65. log.info("workflows left: %s" % totalWorkflows)
  66. EOF