hue_history_cron.sh 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114
  1. #!/bin/bash
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #Cleans up old oozie workflow and beeswax savedqueries to
  18. #prevent the DB from getting too large.
  19. PARCEL_DIR=/opt/cloudera/parcels/CDH
  20. LOG_FILE=/var/log/hue/`basename "$0" | awk -F\. '{print $1}'`.log
  21. LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
  22. LOG_ROTATE_COUNT=2 #number of log files, so 20MB max
  23. DATE=`date '+%Y%m%d-%H%M'`
  24. KEEP_DAYS=7 #Number of days of beeswax and oozie history to keep
  25. if [ ! -d "/usr/lib/hadoop" ]
  26. then
  27. CDH_HOME=$PARCEL_DIR
  28. else
  29. CDH_HOME=/usr
  30. fi
  31. if [ -d "/var/run/cloudera-scm-agent/process" ]
  32. then
  33. HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
  34. else
  35. HUE_CONF_DIR="/etc/hue/conf"
  36. fi
  37. if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
  38. then
  39. COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
  40. else
  41. COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
  42. fi
  43. ORACLE_HOME=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2/
  44. LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
  45. export CDH_HOME HUE_CONF_DIR ORACLE_HOME LD_LIBRARY_PATH COMMAND
  46. ${COMMAND} >> /dev/null 2>&1 <<EOF
  47. from beeswax.models import SavedQuery
  48. from datetime import date, timedelta
  49. from oozie.models import Workflow
  50. from django.db.utils import DatabaseError
  51. import logging
  52. import logging.handlers
  53. import sys
  54. LOGFILE="${LOG_FILE}"
  55. keepDays = ${KEEP_DAYS}
  56. deleteRecords = 900
  57. errorCount = 0
  58. log = logging.getLogger('')
  59. log.setLevel(logging.INFO)
  60. format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
  61. fh = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes=(1048576*${LOG_ROTATE_SIZE}), backupCount=${LOG_ROTATE_COUNT})
  62. fh.setFormatter(format)
  63. log.addHandler(fh)
  64. log.info('HUE_CONF_DIR: ${HUE_CONF_DIR}')
  65. log.info("Cleaning up anything in the Hue tables oozie*, desktop* and beeswax* older than ${KEEP_DAYS} old")
  66. savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))
  67. totalQuerys = savedQuerys.count()
  68. loopCount = totalQuerys
  69. deleteCount = deleteRecords
  70. log.info("SavedQuerys left: %s" % totalQuerys)
  71. log.info("Looping through querys")
  72. while loopCount > 0:
  73. if loopCount < deleteCount:
  74. deleteCount = loopCount
  75. excludeCount = loopCount - deleteCount
  76. savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))[:excludeCount].values_list("id", flat=True)
  77. try:
  78. SavedQuery.objects.exclude(pk__in=list(savedQuerys)).delete()
  79. loopCount -= deleteCount
  80. errorCount = 0
  81. deleteCount = deleteRecords
  82. except DatabaseError, e:
  83. log.info("Non Fatal Exception: %s: %s" % (e.__class__.__name__, e))
  84. errorCount += 1
  85. deleteCount = 1
  86. if errorCount > 9:
  87. raise
  88. log.info("querys left: %s" % loopCount)
  89. workflows = Workflow.objects.filter(is_trashed=True, last_modified__lte=date.today() - timedelta(days=keepDays))
  90. totalWorkflows = workflows.count()
  91. loopCount = 1
  92. maxCount = 1000
  93. log.info("Workflows left: %s" % totalWorkflows)
  94. log.info("Looping through workflows")
  95. for w in workflows:
  96. w.delete(skip_trash=True)
  97. loopCount += 1
  98. if loopCount == maxCount:
  99. totalWorkflows = totalWorkflows - maxCount
  100. loopCount = 1
  101. log.info("Workflows left: %s" % totalWorkflows)
  102. EOF