hue_history_cron.sh 3.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. #!/bin/bash
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #
  18. #Cleans up old oozie workflow and beeswax savedqueries to
  19. #prevent the DB from getting too large.
  20. PARCEL_DIR=/opt/cloudera/parcels/CDH
  21. LOG_FILE=/var/log/hue_history_cron.log
  22. LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
  23. LOG_ROTATE_COUNT=2 #number of log files, so 20MB max
  24. DATE=`date '+%Y%m%d-%H%M'`
  25. KEEP_DAYS=7 #Number of days of beeswax and oozie history to keep
  26. if [ ! -d "/usr/lib/hadoop" ]
  27. then
  28. CDH_HOME=$PARCEL_DIR
  29. else
  30. CDH_HOME=/usr
  31. fi
  32. if [ -d "/var/run/cloudera-scm-agent/process" ]
  33. then
  34. HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
  35. else
  36. HUE_CONF_DIR="/etc/hue/conf"
  37. fi
  38. if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
  39. then
  40. COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
  41. else
  42. COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
  43. fi
  44. export CDH_HOME HUE_CONF_DIR COMMAND
  45. ${COMMAND} <<EOF
  46. from beeswax.models import SavedQuery
  47. from datetime import date, timedelta
  48. from oozie.models import Workflow
  49. import logging
  50. import logging.handlers
  51. import sys
  52. LOGFILE="${LOG_FILE}"
  53. keepDays = ${KEEP_DAYS}
  54. log = logging.getLogger('')
  55. log.setLevel(logging.INFO)
  56. format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
  57. fh = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes=(1048576*${LOG_ROTATE_SIZE}), backupCount=${LOG_ROTATE_COUNT})
  58. fh.setFormatter(format)
  59. log.addHandler(fh)
  60. log.info('HUE_CONF_DIR: ${HUE_CONF_DIR}')
  61. log.info("Cleaning up anything in the Hue tables oozie*, desktop* and beeswax* older than ${KEEP_DAYS} old")
  62. savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))
  63. log.info("SavedQuery count is: %s" % savedQuerys.count())
  64. savedQuerys.delete()
  65. log.info("SavedQuery new count is: %s" % savedQuerys.count())
  66. workflows = Workflow.objects.filter(is_trashed=True, last_modified__lte=date.today() - timedelta(days=keepDays))
  67. totalWorkflows = workflows.count()
  68. loopCount = 1
  69. maxCount = 1000
  70. log.info("workflows left: %s" % totalWorkflows)
  71. log.info("Looping through workflows")
  72. for w in workflows:
  73. w.delete(skip_trash=True)
  74. loopCount += 1
  75. if loopCount == maxCount:
  76. totalWorkflows = totalWorkflows - maxCount
  77. loopCount = 1
  78. log.info("workflows left: %s" % totalWorkflows)
  79. EOF