Ver código fonte

HUE-2753 [core] Add script to cleanup history in oozie and beeswax tables

Chris Conner 10 anos atrás
pai
commit
11a26e4ea5
1 arquivos alterados com 77 adições e 0 exclusões
  1. 77 0
      tools/history_cleanup/hue_history_cron.sh

+ 77 - 0
tools/history_cleanup/hue_history_cron.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+#Cleans up old oozie workflow and beeswax savedqueries to
+#prevent the DB from getting too large.
+PARCEL_DIR=/opt/cloudera/parcels/CDH
+LOG_FILE=/var/log/hue_history_cron.log
+LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
+LOG_ROTATE_COUNT=2 #number of log files, so 20MB max
+DATE=`date '+%Y%m%d-%H%M'`
+KEEP_DAYS=7    #Number of days of beeswax and oozie history to keep
+
+if [ ! -d "/usr/lib/hadoop" ]
+then
+   CDH_HOME=$PARCEL_DIR
+else
+   CDH_HOME=/usr
+fi
+
+if [ -d "/var/run/cloudera-scm-agent/process" ]
+then
+   HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
+else
+   HUE_CONF_DIR="/etc/hue/conf"
+fi
+
+if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+then
+   COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
+else
+   COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
+fi
+
+export CDH_HOME HUE_CONF_DIR COMMAND
+
+${COMMAND} <<EOF
+from beeswax.models import SavedQuery
+from datetime import date, timedelta
+from oozie.models import Workflow
+import logging
+import logging.handlers
+import sys
+
+LOGFILE="${LOG_FILE}"
+keepDays = ${KEEP_DAYS}
+log = logging.getLogger('')
+log.setLevel(logging.INFO)
+format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+fh = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes=(1048576*${LOG_ROTATE_SIZE}), backupCount=${LOG_ROTATE_COUNT})
+fh.setFormatter(format)
+log.addHandler(fh)
+
+log.info('HUE_CONF_DIR: ${HUE_CONF_DIR}')
+log.info("Cleaning up anything in the Hue tables oozie*, desktop* and beeswax* older than ${KEEP_DAYS} old")
+
+savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))
+count = len(savedQuerys)
+log.info("SavedQuery count is: %s" % count)
+savedQuerys.delete()
+savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))
+count = len(savedQuerys)
+log.info("SavedQuery new count is: %s" % count)
+
+totalWorkflows = len(Workflow.objects.filter(is_trashed=True, last_modified__lte=date.today() - timedelta(days=keepDays)))
+loopCount = 1
+maxCount = 1000
+log.info("workflows left: %s" % totalWorkflows)
+log.info("Looping through workflows")
+for w in Workflow.objects.filter(is_trashed=True, last_modified__lte=date.today() - timedelta(days=keepDays)):
+   w.delete(skip_trash=True)
+   loopCount += 1
+   if (loopCount == maxCount):
+      totalWorkflows = totalWorkflows - maxCount
+      loopCount = 1
+      log.info("workflows left: %s" % totalWorkflows)
+
+EOF