Browse Source

Added hue scripts to the Cloudera/hue repo (#1784)

* Add hue scripts from Chris Conner's personal repo

* Added Apache 2.0 license note

* Updated the download URL

* Edited and formatted the Readme

for hue scripts

Co-authored-by: Johan Åhlén <johan@johanahlen.com>
Priyanka Chheda 5 years ago
parent
commit
60c1d75b0b
57 changed files with 7470 additions and 0 deletions
  1. 97 0
      tools/ops/script_runner/README.md
  2. 156 0
      tools/ops/script_runner/extract_archive_in_hdfs.sh
  3. 78 0
      tools/ops/script_runner/get_hue_version.sh
  4. 115 0
      tools/ops/script_runner/hue_change_dashboard_owner.sh
  5. 267 0
      tools/ops/script_runner/hue_clean_duplicate_permissions.sh
  6. 156 0
      tools/ops/script_runner/hue_create_db.sh
  7. 175 0
      tools/ops/script_runner/hue_download_watcher.sh
  8. 217 0
      tools/ops/script_runner/hue_dump_user.sh
  9. 21 0
      tools/ops/script_runner/hue_history_cron.sh
  10. 130 0
      tools/ops/script_runner/hue_multiple_home_cleanup.sh
  11. 204 0
      tools/ops/script_runner/hue_restart_cm.sh
  12. 15 0
      tools/ops/script_runner/lib/__init__.py
  13. 32 0
      tools/ops/script_runner/lib/adapter_factory.py
  14. 218 0
      tools/ops/script_runner/lib/cm_environment.py
  15. 67 0
      tools/ops/script_runner/lib/conversion_runner.py
  16. 15 0
      tools/ops/script_runner/lib/custom_commands/__init__.py
  17. 15 0
      tools/ops/script_runner/lib/custom_commands/management/__init__.py
  18. 15 0
      tools/ops/script_runner/lib/custom_commands/management/commands/__init__.py
  19. 306 0
      tools/ops/script_runner/lib/custom_commands/management/commands/backend_test_curl.py
  20. 68 0
      tools/ops/script_runner/lib/custom_commands/management/commands/c6_test_command.py
  21. 81 0
      tools/ops/script_runner/lib/custom_commands/management/commands/change_owner_of_docs.py
  22. 164 0
      tools/ops/script_runner/lib/custom_commands/management/commands/db_query_test.py
  23. 67 0
      tools/ops/script_runner/lib/custom_commands/management/commands/delete_user.py
  24. 178 0
      tools/ops/script_runner/lib/custom_commands/management/commands/estimate_concurrent_users.py
  25. 149 0
      tools/ops/script_runner/lib/custom_commands/management/commands/hue_desktop_document_cleanup.py
  26. 73 0
      tools/ops/script_runner/lib/custom_commands/management/commands/list_groups.py
  27. 66 0
      tools/ops/script_runner/lib/custom_commands/management/commands/promote_to_superuser.py
  28. 92 0
      tools/ops/script_runner/lib/custom_commands/management/commands/remove_doc2_without_content_object.py
  29. 59 0
      tools/ops/script_runner/lib/custom_commands/management/commands/remove_duplicate_user_preferences.py
  30. 112 0
      tools/ops/script_runner/lib/custom_commands/management/commands/remove_orphaned_docs.py
  31. 155 0
      tools/ops/script_runner/lib/custom_commands/management/commands/rename_duplicate_users.py
  32. 133 0
      tools/ops/script_runner/lib/custom_commands/management/commands/run_hive_impala_query.py
  33. 88 0
      tools/ops/script_runner/lib/custom_commands/management/commands/set_default_editor.py
  34. 152 0
      tools/ops/script_runner/lib/custom_commands/management/commands/share_all_workflows.py
  35. 21 0
      tools/ops/script_runner/lib/custom_commands/settings.py
  36. 245 0
      tools/ops/script_runner/lib/customdumpdata.py
  37. 97 0
      tools/ops/script_runner/lib/doc2_utils.py
  38. 109 0
      tools/ops/script_runner/lib/doc_count_util.py
  39. 353 0
      tools/ops/script_runner/lib/hue_adapters.py
  40. 365 0
      tools/ops/script_runner/lib/hue_converters.py
  41. 75 0
      tools/ops/script_runner/lib/hue_curl.py
  42. 24 0
      tools/ops/script_runner/lib/hue_shared.py
  43. 108 0
      tools/ops/script_runner/lib/log.conf
  44. 226 0
      tools/ops/script_runner/lib/log/__init__.py
  45. 168 0
      tools/ops/script_runner/lib/log/access.py
  46. 50 0
      tools/ops/script_runner/lib/log/formatter.py
  47. 78 0
      tools/ops/script_runner/lib/log/log_buffer.py
  48. 41 0
      tools/ops/script_runner/lib/log/log_buffer_test.py
  49. 95 0
      tools/ops/script_runner/lib/log/tests.py
  50. 234 0
      tools/ops/script_runner/lib/url_util.py
  51. 273 0
      tools/ops/script_runner/patches/5.12_all_logging.patch
  52. 273 0
      tools/ops/script_runner/patches/5.13_all_logging.patch
  53. 239 0
      tools/ops/script_runner/patches/5.14_all_logging.patch
  54. 224 0
      tools/ops/script_runner/patches/5.15_all_logging.patch
  55. 36 0
      tools/ops/script_runner/patches/fix_cm_config_file.patch
  56. 66 0
      tools/ops/script_runner/queries.txt
  57. 134 0
      tools/ops/script_runner/script_runner

+ 97 - 0
tools/ops/script_runner/README.md

@@ -0,0 +1,97 @@
+# Ops Scripts
+
+This is a set of scripts that you can use to manage Hue.  While some of these are self contained, the most important ones have library calls to the `lib` directory.  You must download the entire "script_runner" directory for these to work.  Also note, they all make use of Hue's configuration and the `/proc` directory.  This means these scripts must be run as ROOT.
+
+Clone the script_runner directory to the `/opt/cloudera/hue_scripts` directory:
+
+    git clone https://github.com/cloudera/hue/tree/master/tools/ops/script_runner /opt/cloudera/hue_scripts
+
+The following commands show the available scripts in [custom_scripts] section, the rest are normal Hue commands:
+
+    cd /opt/cloudera/hue_scripts
+    [root@nightly516-1 hue_scripts]# ./script_runner
+
+## Usage
+    script_runner sub-command [options] [args]
+
+## Options
+|Option|Description|
+|--|--|
+|`-v VERBOSITY`, `--verbosity=VERBOSITY`|Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output |
+|`--settings=SETTINGS`|The Python path to a settings module, e.g. "myproject.settings.main". If this isn't provided, the DJANGO_SETTINGS_MODULE environment variable is used.|
+|`--pythonpath=PYTHONPATH`|A directory to add to the Python path, e.g. "/home/djangoprojects/myproject".|
+|`--traceback`|Raise on exception.|
+|`--version`|Show program's version number and exit.|
+|`-h`, `--help`|Show this help message and exit.|
+
+Type `script_runner help <sub-command>` for help on a specific subcommand.
+
+## Sub-commands
+
+[auth]
+`changepassword`
+
+[custom_commands]
+`backend_test_curl`
+`change_owner_of_docs`
+`delete_user`
+`estimate_concurrent_users`
+`hue_desktop_document_cleanup`
+`list_groups`
+`promote_to_superuser`
+`emove_doc2_without_content_object`
+`remove_duplicate_user_preferences`
+`remove_orphaned_docs`
+`rename_duplicate_users`
+`run_hive_impala_query`
+`share_all_workflows`
+
+## Commands
+
+- `script_runner backend_test_curl`
+This tests all of the REST API backend services to make sure they are running.  This includes HTTPFS, Resource Manager, Job History Server, Oozie, Solr.  Then outputs the matching curl commands to test outside of Hue.
+
+- `script_runner change_owner_of_docs --olduser cconner --newuser cconner1`
+This changes the owner of any documents owned by oldusuer to newuser.
+
+- `script_runner delete_user --username cconner`
+This deletes the user specified by --username.
+
+- `script_runner estimate_concurrent_users --last1h`
+This checks the access logs and try to estimate the number of active users over the last hour or given timeframe.
+
+- `script_runner list_groups`
+This shows all of the groups that exist in Hue.
+
+- `script_runner promote_to_superuser --username cconner`
+This promotes a non-superuser to a superuser.
+
+- `script_runner remove_doc2_without_content_object`
+This removes bad doc2 objects that do not have the content object.
+
+- `script_runner remove_duplicate_user_preferences`
+This removes duplicate entries in the table "desktop_userpreferences".
+
+- `script_runner remove_orphaned_docs`
+This removes broken docs that do not have an owner.
+
+- `script_runner rename_duplicate_users`
+If you end up with duplicate usernames somehow, then this renames the most recently created one to prevent data loss, but fix errors.
+
+- `script_runner run_hive_impala_query --impala --user cconner --query "select * from sample_07"`
+This runs a query against Hive or Impala using Hue code, but outside the Hue process.  It impersonates the specified user.  This is good for determining if an issue is caused by Hue or the backend service.
+
+- `script_runner share_all_workflows --owner cconner --sharegroups test1,test2 --permissioins read,write`
+This shares workflows from --owner to a comma-separated list of shareusers or sharegroups and a list of permissions specified by --permissions.
+
+- `hue_download_watcher.sh`
+This script can be used to capture info surrounding downloads from Hue.
+
+- `hue_clean_duplicate_permissions.sh`
+This script is old, but it would have cleaned up duplicate entries for HuePermission objects.
+
+- `extract_archive_in_hdfs.sh`
+Debug version of the same extraction script in Hue source.
+
+## License
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)

+ 156 - 0
tools/ops/script_runner/extract_archive_in_hdfs.sh

@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+# Linked 6 to stdout
+exec 6>&1 
+## Close STDOUT file descriptor
+#exec 1<&-
+## Close STDERR FD
+#exec 2<&-
+
+export LOG_FILE=/tmp/$0.log
+# Open STDOUT as $LOG_FILE file for read and write.
+exec 1<>$LOG_FILE
+
+# Redirect STDERR to STDOUT
+exec 2>&1
+
+env
+
+UPLOAD_PATH=
+FILE_NAME=
+OUTPUT_PATH=
+
+function usage()
+{
+    echo "Arguments '-u' and '-f' are mandatory."
+    echo "Usage:"
+    echo "\t-h --help"
+    echo "\t[-u | --upload-path]=<PATH_IN_HDFS>"
+    echo "\t[-f | --file-name]=<FILE_NAME>"
+    echo ""
+}
+
+function cat_exit()
+{
+   STATUS=$1
+   if [[ -z ${STATUS} ]]
+   then
+       STATUS=0
+   fi
+
+   # Restore stdout and close 6 
+   exec 1>&6 6>&- 
+
+   cat ${LOG_FILE}
+   exit ${STATUS}
+}
+
+while [ "$1" != "" ]; do
+    PARAM=`echo $1 | awk -F= '{print $1}'`
+    VALUE=`echo $1 | awk -F= '{print $2}'`
+    case $PARAM in
+        -h | --help)
+            usage
+            cat_exit
+            ;;
+        -u | --upload-path)
+            UPLOAD_PATH=$VALUE
+            ;;
+        -f | --file-name)
+            FILE_NAME=$VALUE
+            ;;
+        -o | --output-path)
+            OUTPUT_PATH=$VALUE
+            ;;
+        *)
+            echo "ERROR: unknown parameter \"$PARAM\""
+            usage
+            cat_exit 1
+            ;;
+    esac
+    shift
+done
+
+if [ -z "$UPLOAD_PATH" ] || [ -z "$FILE_NAME" ] || [ -z "$OUTPUT_PATH" ]
+then
+	echo "ERROR: Missing Arguments"
+	usage
+	cat_exit 1
+fi
+
+exit_status=0
+
+temp_output_dir=`mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir'`
+echo 'Created temporary output directory: '$temp_output_dir
+
+if [[ "$FILE_NAME" =~ \.zip$ ]]
+then
+        echo "Unzipping $FILE_NAME"
+        which unzip
+	unzip "$FILE_NAME" -d $temp_output_dir
+	exit_status=$(echo $?)
+elif [[ "$FILE_NAME" =~ \.tar\.gz$ ]] || [[ "$FILE_NAME" =~ \.tgz$ ]]
+then
+        echo "Untarring $FILE_NAME"
+        which tar
+	tar -xvzf "$FILE_NAME" -C $temp_output_dir
+	exit_status=$(echo $?)
+elif [[ "$FILE_NAME" =~ \.bz2$ ]] || [[ "$FILE_NAME" =~ \.bzip2$ ]]
+then
+        echo "Bunzipping $FILE_NAME"
+	filename_without_extension=$(echo "$FILE_NAME" | cut -f 1 -d '.')
+        which bzip2
+	bzip2 -dc "$FILE_NAME" > $temp_output_dir/"$filename_without_extension"
+	exit_status=$(echo $?)
+else
+	echo 'ERROR: Could not interpret archive type.'
+	exit_status=1
+fi
+
+echo "Exit status: $exit_status"
+echo "Getting extracted file count"
+extracted_file_count=$(($(find $temp_output_dir/* -type d -maxdepth 0 | wc -l) + $(find $temp_output_dir/* -type f -maxdepth 0 | wc -l)))
+echo "Extracted file count: $extracted_file_count"
+if [ $extracted_file_count != 0 ] && [ $exit_status == 0 ]
+then
+    echo "File count > 0 and exit status ==0"
+    if ! $(hadoop fs -test -d $OUTPUT_PATH)
+    then
+        echo "Creating output directory '$OUTPUT_PATH' in HDFS"
+        hadoop fs -mkdir $OUTPUT_PATH
+    fi
+	echo "Copying extracted files to '$OUTPUT_PATH' in HDFS"
+	hadoop fs -put $temp_output_dir/* "$OUTPUT_PATH"
+	exit_status=$(echo $?)
+	if [ $exit_status != 0 ]
+	then
+	    echo "Failed to copy files to HDFS directory '$OUTPUT_PATH'."
+	else
+	    echo "Copy to HDFS directory '$OUTPUT_PATH' complete."
+	fi
+else
+	exit_status=1
+fi
+
+echo "Deleting temporary output directory exit status: $exit_status"
+rm -rf $temp_output_dir
+echo 'Deleted temporary output directory: '$temp_output_dir
+
+cat_exit $exit_status

+ 78 - 0
tools/ops/script_runner/get_hue_version.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -x
+
+TMP_DIR=/tmp/$(basename "$0" | awk -F\. '{print $1}')
+
+HUE_VERSION=$1
+
+if [[ -d /opt/cloudera/parcels/CDH/lib/hue-${HUE_VERSION} ]]
+then
+  echo "This version already downloaded"
+  exit 1
+fi
+
+HUE_MAJOR=$(echo ${HUE_VERSION} | cut -f1 -d.)
+OS_CDH_VERSION=$(basename $(hadoop version | grep "\.jar" | awk '{print $6}') | awk -Fcdh '{print $2}' | awk -F\. '{print $1}')
+OS_EL_VERSION="el$(lsb_release -rs | cut -f1 -d.)"
+
+HUE_VERSION_TEST=
+if [[ ${HUE_MAJOR} -eq 5 ]]
+then
+  HUE_VERSION_TEST=$(echo ${HUE_VERSION} | grep "^[0-9]\.[0-9][0-9]\.[0-9]$")
+  ARCHIVE_BASE_URL=https://archive.cloudera.com/cdh${HUE_MAJOR}/parcels/${HUE_VERSION}/
+elif [[ ${HUE_MAJOR} -eq 6 ]]
+then
+  HUE_VERSION_TEST=$(echo ${HUE_VERSION} | grep "^[0-9]\.[0-9]\.[0-9]$")
+  ARCHIVE_BASE_URL=https://archive.cloudera.com/cdh${HUE_MAJOR}/${HUE_VERSION}/parcels/
+elif [[ ${HUE_MAJOR} -eq 7 ]]
+then
+  HUE_VERSION_TEST=$(echo ${HUE_VERSION} | grep "^[0-9]\.[0-9]\.[0-9]$")
+  ARCHIVE_BASE_URL=https://archive.cloudera.com/cdh${HUE_MAJOR}/${HUE_VERSION}/parcels/
+else
+  echo "Major version was ${HUE_MAJOR} this script only works on major version 5, 6, 7"
+  exit 1
+fi
+
+if [[ -z ${HUE_VERSION_TEST} ]]
+then
+  echo "Invalid Hue version, example of good 5.16.0 or 6.3.2"
+  exit 1
+fi
+
+if [[ ! -f /etc/redhat-release ]]
+then
+  echo "This is not RedHat, Oracle Linux or CentOS, this script only works on those OS's"
+  exit 1
+fi
+
+PARCEL_NAME=$(curl -s ${ARCHIVE_BASE_URL} | grep "${OS_EL_VERSION}.parcel<" | sed "s/.*href=\"//g" | sed "s/\">.*//g")
+if [[ ! -z ${PARCEL_NAME} ]]
+then
+  PARCEL_DIR_NAME=$(echo ${PARCEL_NAME} | sed "s/\-${OS_EL_VERSION}.*//g")
+
+  mkdir -p ${TMP_DIR}
+  cd ${TMP_DIR} && wget ${ARCHIVE_BASE_URL}/${PARCEL_NAME}
+  cd ${TMP_DIR} && tar xvf ${PARCEL_NAME}
+  if [[ ! -d /opt/cloudera/parcels/CDH/lib/hue-${HUE_VERSION} ]]
+  then
+    cd ${TMP_DIR} && mv ${PARCEL_DIR_NAME}/lib/hue /opt/cloudera/parcels/CDH/lib/hue-${HUE_VERSION}
+  fi
+else
+  echo "${HUE_VERSION} does not exist"
+fi
+cd && rm -Rf ${TMP_DIR}

+ 115 - 0
tools/ops/script_runner/hue_change_dashboard_owner.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Changes owner of Search Dashboard
+PARCEL_DIR=/opt/cloudera/parcels/CDH
+
+DASHBOARD=$1
+NEWOWNER=$2
+USAGE="usage: $0 <dashboardname> <new_owner_name>"
+
+if [[ ! ${USER} =~ .*root* ]]
+then
+  echo "Script must be run as root: exiting"
+  exit 1
+fi
+
+if [[ -z ${NEWOWNER} ]]
+then
+  echo "No new_owner_name specified:"
+  echo ${USAGE}
+  exit 1
+fi
+
+if [[ -z ${DASHBOARD} ]]
+then
+  echo "No dashboard_name specified:"
+  echo ${USAGE}
+  exit 1
+fi
+
+if [ ! -d "/usr/lib/hadoop" ]
+then
+   CDH_HOME=$PARCEL_DIR
+else
+   CDH_HOME=/usr
+fi
+
+if [ -d "/var/run/cloudera-scm-agent/process" ]
+then
+   HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE_SERVER | sort -n | tail -1 `"
+else
+   HUE_CONF_DIR="/etc/hue/conf"
+fi
+
+if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+then
+   COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
+else
+   COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
+fi
+
+ORACLE_HOME=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2/
+LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
+export CDH_HOME HUE_CONF_DIR ORACLE_HOME LD_LIBRARY_PATH COMMAND
+
+echo "HUE_CONF_DIR: ${HUE_CONF_DIR}"
+echo "COMMAND: ${COMMAND}"
+
+${COMMAND} <<EOF
+dashboard = "${DASHBOARD}"
+newowner = "${NEWOWNER}"
+from django.contrib.auth.models import User
+from search.models import Collection
+user = User.objects.get(username=newowner)
+#for collection in Collection.objects.filter(name=dashboard):
+#collection = Collection.objects.get(name=dashboard)
+
+for collection in Collection.objects.filter(name=dashboard):
+  print "Changing owner of colection(%s) from user(%s) to user(%s)" % (collection.name, collection.owner, user.username)
+  collection.owner = user
+  collection.save()
+
+#collection = Collection.objects.get(name=dashboard)
+for collection in Collection.objects.filter(name=dashboard):
+  print "Owner of colection(%s) is now user(%s)" % (collection.name, collection.owner)
+
+#Useful other examples:
+#from django.contrib.auth.models import User, Group
+#user = User.objects.get(username="tuser4")
+#user2 = User.objects.get(username="cconner")
+#from search.models import Collection
+#for collection in Collection.objects.filter(owner=user):
+#  collection.name
+#  collection.owner
+#  print ""
+#  collection.owner = user2
+#  collection.save()
+
+#from django.contrib.auth.models import User, Group
+#dashboardname = "students"
+#user = User.objects.get(username="tuser4")
+#user2 = User.objects.get(username="cconner")
+#from search.models import Collection
+#for collection in Collection.objects.filter(name=dashboardname):
+#  collection.name
+#  collection.owner
+#  print ""
+#  collection.owner = user2
+#  collection.save()
+
+EOF

+ 267 - 0
tools/ops/script_runner/hue_clean_duplicate_permissions.sh

@@ -0,0 +1,267 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Clean up duplicate Hue permissions
+
+#parse command line arguments
+parse_arguments()
+{
+  # Test that we're using compatible getopt version.
+  getopt -T > /dev/null
+  if [[ $? -ne 4 ]]; then
+    echo "Incompatible getopt version."
+    exit 1
+  fi
+
+  # Parse short and long option parameters.
+  OVERRIDE=
+  VERBOSE=
+  DESKTOP_DEBUG=false
+  GETOPT=`getopt -n $0 -o o,v,h \
+      -l override,verbose,help \
+      -- "$@"`
+  eval set -- "$GETOPT"
+  while true;
+  do
+    case "$1" in
+    -o|--override)
+      OVERRIDE=true
+      shift
+      ;;
+    -v|--verbose)
+      VERBOSE=true
+      shift
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+    esac
+  done
+  #
+}
+
+usage()
+{
+cat << EOF
+usage: $0 [options]
+
+Removes all duplicate permissions
+
+OPTIONS
+   -o|--override           Allow script to run as non-root, must set HUE_CONF_DIR manually before running
+   -v|--verbose		   Enable verbose logging
+   -h|--help               Show this message.
+EOF
+}
+
+debug()
+{
+   if [[ ! -z $VERBOSE ]]
+   then
+      echo "$1" >> ${LOG_FILE}
+   fi
+}
+
+main()
+{
+
+  parse_arguments "$@"
+
+  SCRIPT_DIR="$( cd -P "$( dirname "$0" )" && pwd )"
+  PYTHONPATH=${SCRIPT_DIR}/lib:${PYTHONPATH}
+  export SCRIPT_DIR PYTHONPATH
+
+  #SET IMPORTANT ENV VARS
+  if [[ -z ${HUE_CONF_DIR} ]]
+  then
+    if [ -d "/var/run/cloudera-scm-agent/process" ]
+    then
+      HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE_SERVER | sort -n | tail -1 `"
+    else
+      HUE_CONF_DIR="/etc/hue/conf"
+    fi
+    export HUE_CONF_DIR
+  fi
+
+  if [[ ! ${USER} =~ .*root* ]]
+  then
+    if [[ -z ${OVERRIDE} ]]
+    then
+      echo "Script must be run as root: exiting"
+      exit 1
+    fi
+  else
+    if [[ $(ps -ef | grep "[h]ue runc" | awk '{print }') ]]
+    then
+      DESKTOP_LOG_DIR=$(strings /proc/$(ps -ef | grep "[h]ue runc" | awk '{print }' | awk '{print $2}')/environ | grep DESKTOP_LOG_DIR | awk -F\= '{print $2}')
+    fi
+  fi
+
+  if [[ -z ${DESKTOP_LOG_DIR} ]]
+  then
+    DESKTOP_LOG_DIR=${HUE_CONF_DIR}/logs
+  fi
+  if [[ ! -f ${DESKTOP_LOG_DIR} ]]
+  then
+    mkdir -p ${DESKTOP_LOG_DIR}
+  fi
+  LOG_FILE=${DESKTOP_LOG_DIR}/`basename "$0" | awk -F\. '{print $1}'`.log
+  LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
+  LOG_ROTATE_COUNT=5 #number of log files, so 20MB max
+
+  PARCEL_DIR=/opt/cloudera/parcels/CDH
+  if [ ! -d "/usr/lib/hadoop" ]
+  then
+    CDH_HOME=$PARCEL_DIR
+  else
+    CDH_HOME=/usr
+  fi
+
+  if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+  then
+    COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
+    TEST_COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue dbshell"
+  else
+    COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
+    TEST_COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue dbshell"
+  fi
+
+  ORACLE_ENGINE_CHECK=$(grep engine ${HUE_CONF_DIR}/hue* | grep -i oracle)
+  if [[ ! -z ${ORACLE_ENGINE_CHECK} ]]
+  then
+    if [[ -z ${ORACLE_HOME} ]]
+    then
+      ORACLE_PARCEL=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2
+      if [[ -d ${ORACLE_PARCEL} ]]
+      then
+        ORACLE_HOME=${ORACLE_PARCEL}
+        LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
+        export LD_LIBRARY_PATH ORACLE_HOME
+      fi
+    fi
+    if [[ -z ${ORACLE_HOME} ]]
+    then
+      echo "It looks like you are using Oracle as your backend"
+      echo "ORACLE_HOME must be set to the correct Oracle client"
+      echo "before running this script"
+      exit 1
+    fi
+  fi
+
+  HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1
+  if [[ -z ${HUE_DATABASE_PASSWORD} ]]
+  then
+    echo "CDH 5.5 and above requires that you set the environment variable:"
+    echo "HUE_DATABASE_PASSWORD=<dbpassword>"
+    exit 1
+  fi
+  PGPASSWORD=${HUE_DATABASE_PASSWORD}
+  export CDH_HOME COMMAND HUE_IGNORE_PASSWORD_SCRIPT_ERRORS PGPASSWORD
+
+  debug "Validating DB connectivity"
+#  echo "COMMAND: echo \"from django.db import connection; cursor = connection.cursor(); cursor.execute('select count(*) from auth_user')\" | ${TEST_COMMAND}" | tee -a ${LOG_FILE}
+#  echo "from django.db import connection; cursor = connection.cursor(); cursor.execute('select count(*) from auth_user')" | ${TEST_COMMAND} | tee -a ${LOG_FILE}
+
+  QUIT_COMMAND="quit"
+  PG_ENGINE_CHECK=$(grep engine ${HUE_CONF_DIR}/hue* | grep -i postgres)
+  if [[ ! -z ${PG_ENGINE_CHECK} ]]
+  then
+    QUIT_COMMAND='\q'
+  fi
+
+#  echo "Running echo ${QUIT_COMMAND} | ${TEST_COMMAND}"
+#  echo ${QUIT_COMMAND} | ${TEST_COMMAND}
+  if [[ $? -ne 0 ]]
+  then
+    echo "HUE_DATABASE_PASSWORD is incorrect.  Please check CM: http://${HOSTNAME}:7180/api/v5/cm/deployment and search for HUE_SERVER and database to find correct password"
+    exit 1
+  fi
+
+debug "Running ${COMMAND}"
+${COMMAND} >> /dev/null 2>&1 <<EOF
+from useradmin.models import HuePermission, GroupPermission
+from datetime import date, timedelta
+import desktop.conf
+import logging
+import logging.handlers
+import sys
+
+LOGFILE="${LOG_FILE}"
+log = logging.getLogger('')
+log.setLevel(logging.INFO)
+format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+fh = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes = (1048576*logrotatesize), backupCount = backupcount)
+fh.setFormatter(format)
+log.addHandler(fh)
+
+log.info('HUE_CONF_DIR: ${HUE_CONF_DIR}')
+log.info("DB Engine: %s" % desktop.conf.DATABASE.ENGINE.get())
+log.info("DB Name: %s" % desktop.conf.DATABASE.NAME.get())
+log.info("DB User: %s" % desktop.conf.DATABASE.USER.get())
+log.info("DB Host: %s" % desktop.conf.DATABASE.HOST.get())
+log.info("DB Port: %s" % str(desktop.conf.DATABASE.PORT.get()))
+log.info("Removing duplicate Hue permissions")
+
+apps = []
+for perm in HuePermission.objects.all():
+  apps.append(perm.app)
+
+
+for app in sorted(set(apps)):
+  log.info("")
+  log.info("Working on app: %s" % app)
+  perms = HuePermission.objects.filter(app=app)
+  for perm in perms:
+    permsdistinct = HuePermission.objects.filter(app=app, description = perm.description)
+    print "len: %s: app: %s: description: %s: id: %s" % (len(permsdistinct), perm.app, perm.description, perm.id)
+    if len(permsdistinct) > 1:
+      try:
+        group = GroupPermission.objects.get(hue_permission=perm)
+        if group.group_id != 1:
+          log.info("Cleaning up Hue permission for only default group: %s" % perm.__dict__)
+          HuePermission.objects.filter(app=app, id = perm.id).delete()
+      except:
+        pass
+
+
+for app in sorted(set(apps)):
+  log.info("")
+  log.info("Working on app because no special perms assigned: %s" % app)
+  perms = HuePermission.objects.filter(app=app)
+  for perm in perms:
+    permsdistinct = HuePermission.objects.filter(app=app, description = perm.description)
+    if len(permsdistinct) > 1:
+      log.info("Cleaning up Hue permission: %s" % perm.__dict__)
+      HuePermission.objects.filter(app=app, id = perm.id).delete()
+
+
+EOF
+
+echo ""
+echo "Logs can be found in ${DESKTOP_LOG_DIR}"
+
+unset PGPASSWORD
+
+}
+
+main "$@"

+ 156 - 0
tools/ops/script_runner/hue_create_db.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+PARCEL_DIR=/opt/cloudera/parcels/CDH
+LOG_FILE=/var/log/hue/`basename "$0" | awk -F\. '{print $1}'`.log
+DATABASE=$1
+PASSWORD=$2
+TYPE=$3
+
+if [[ -z ${DBUSER} ]]
+then
+   DBUSER="cloudera"
+fi
+
+if [[ -z ${DBPASSWORD} ]]
+then
+   DBPASSWORD="cloudera"
+fi
+
+if [[ -z ${TYPE} ]]
+then
+   TYPE="mysql"
+fi
+
+if [[ -z ${DATABASE} ]]
+then
+   echo "Usage: hue_create_db.sh <database_name> <password> <dbtype-mysql-postgres>"
+   exit 1
+fi
+
+
+export HUE_DATABASE_PASSWORD=${PASSWORD}
+export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1
+
+HUE_CONF_DIR=/tmp/hue_create_db/${DATABASE}
+mkdir -p ${HUE_CONF_DIR}
+
+if [ ! -d "/usr/lib/hadoop" ]
+then
+   CDH_HOME=$PARCEL_DIR
+else
+   CDH_HOME=/usr
+fi
+
+if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+then
+   COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue"
+else
+   COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue"
+fi
+
+DATABASE_DUMP=${HUE_CONF_DIR}/hue_database_dump.json
+ORACLE_HOME=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2/
+LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
+export CDH_HOME HUE_CONF_DIR ORACLE_HOME LD_LIBRARY_PATH COMMAND PASSWORD DATABASE
+
+${COMMAND} dumpdata --indent 2 > ${HUE_CONF_DIR}/hue_database_dump.json
+
+if [[ ${TYPE} =~ .*mysql.* ]]
+then
+cat > ${HUE_CONF_DIR}/hue.ini << EOF
+[desktop]
+[[database]]
+#engine=sqlite3
+#name=/var/lib/hue/desktop.db
+engine=mysql
+host=`hostname`
+port=3306
+user=${DATABASE}
+password=${PASSWORD}
+name=${DATABASE}
+EOF
+
+cat > ${HUE_CONF_DIR}/create.sql << EOF
+drop database if exists ${DATABASE};
+create database ${DATABASE};
+grant all on *.* to '${DATABASE}'@'%' identified by '${PASSWORD}';
+EOF
+
+mysql -uroot -p${PASSWORD} < ${HUE_CONF_DIR}/create.sql
+elif [[ ${TYPE} =~ .*postgres.* ]]
+then
+   yum -y install postgresql-server
+   chkconfig postgresql on
+   if [[ ! -f /var/lib/pgsql/data/postgresql.conf ]]
+   then
+      service postgresql initdb
+   fi
+   CHECK_HBA=$(grep ${DATABASE} /var/lib/pgsql/data/pg_hba.conf)
+   if [[ -z ${CHECK_HBA} ]]
+   then
+      echo "host        ${DATABASE}     ${DATABASE}     0.0.0.0/0       md5" >> /var/lib/pgsql/data/pg_hba.conf
+   fi
+   sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '0.0.0.0'/g" /var/lib/pgsql/data/postgresql.conf
+   service postgresql restart
+   sudo -u postgres /bin/bash -c psql -U postgres << EOF
+create database ${DATABASE};
+\c ${DATABASE};
+create user ${DATABASE} with password '${PASSWORD}';
+grant all privileges on database ${DATABASE} to ${DATABASE};
+\q
+EOF
+cat > ${HUE_CONF_DIR}/hue.ini << EOF
+[desktop]
+[[database]]
+#engine=sqlite3
+#name=/var/lib/hue/desktop.db
+engine=postgresql_psycopg2
+host=`hostname`
+port=5432
+user=${DATABASE}
+password=${PASSWORD}
+name=${DATABASE}
+EOF
+fi
+
+${COMMAND} syncdb --noinput
+${COMMAND} migrate --merge
+
+if [[ ${TYPE} =~ .*mysql.* ]]
+then
+   CONSTRAINT_ID=$(mysql -uroot -p${PASSWORD} ${DATABASE} -e "show create table auth_permission" | grep content_type_id_refs_id | awk -Fid_ '{print $3}' | awk -F\` '{print $1}')
+
+cat > ${HUE_CONF_DIR}/prepare.sql << EOF
+ALTER TABLE auth_permission DROP FOREIGN KEY content_type_id_refs_id_${CONSTRAINT_ID};
+delete from django_content_type;
+EOF
+
+mysql -uroot -p${PASSWORD} ${DATABASE} < ${HUE_CONF_DIR}/prepare.sql
+
+elif [[ ${TYPE} =~ .*postgres.* ]]
+then
+  CONSTRAINT_ID=$(PGPASSWORD=${PASSWORD} psql -h `hostname` -U ${DATABASE} -d ${DATABASE} -c '\d auth_permission;' | grep content_type_id_refs_id | awk -Fid_ '{print $3}' | awk -F\" '{print $1}')
+#PGPASSWORD=${PASSWORD} psql -h `hostname` -U ${DATABASE} -d ${DATABASE} << EOF
+#ALTER TABLE auth_permission DROP CONSTRAINT content_type_id_refs_id_${CONSTRAINT_ID};
+#TRUNCATE django_content_type CASCADE;
+#EOF
+fi
+
+
+

+ 175 - 0
tools/ops/script_runner/hue_download_watcher.sh

@@ -0,0 +1,175 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ 
+#parse command line arguments
+parse_arguments()
+{
+  # Test that we're using compatible getopt version.
+  getopt -T > /dev/null
+  if [[ $? -ne 4 ]]; then
+    echo "Incompatible getopt version."
+    exit 1
+  fi
+
+  # Parse short and long option parameters.
+  ROTATE_SIZE=10
+  SPAWN_WATCHER=
+
+  GETOPT=`getopt -n $0 -o d:,s:,w,h \
+      -l hue-log-dir:,max-log-size:,watcher,help \
+      -- "$@"`
+  eval set -- "$GETOPT"
+  while true;
+  do
+    case "$1" in
+    -d|--hue-log-dir)
+      DESKTOP_LOG_DIR=$2
+      shift 2
+      ;;
+    -s|--max-log-size)
+      ROTATE_SIZE=$2
+      shift 2
+      ;;
+    -w|--watcher)
+      SPAWN_WATCHER=$1
+      shift 1
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+    esac
+  done
+}
+
+usage() {
+cat << EOF
+usage: $0
+
+Watches Hue log for downloads and queries:
+
+OPTIONS
+   -d|--hue-log-dir </log/path Path to Hue logs if not in /var/log/hue.
+   -s|--max-log-size <MB>      Max log size before rotating.
+   -w|--watcher                Spawns the watcher daemon.  DO NOT RUN FLAG YOURSELF
+   -h|--help                   Show this message.
+EOF
+}
+  
+main() {
+  INITIAL_ARGS=("$@")
+  parse_arguments "$@"
+  
+  if [[ -z ${DESKTOP_LOG_DIR} ]]
+  then
+    DESKTOP_LOG_DIR=/var/log/hue
+  fi
+ 
+  ENDFILE=${DESKTOP_LOG_DIR}/$(basename "$0" | awk -F\. '{print $1}').finish
+  RUNCPSERVER=${DESKTOP_LOG_DIR}/runcpserver.log
+  LOG_FILE=${DESKTOP_LOG_DIR}/$(basename "$0" | awk -F\. '{print $1}').log
+
+
+  if [[ ! ${USER} =~ .*root* ]]
+  then
+    echo "Script must be run as root: exiting"
+    exit 1
+  fi
+  
+  if [[ -f ${ENDFILE} ]] && [[ -z ${SPAWN_WATCHER} ]]
+  then
+    echo "$0 already running.  Please run:"
+    echo "rm -f ${ENDFILE}"
+    echo "to exit the previous running version."
+    exit 1
+  fi
+ 
+  if [[ ! -z ${SPAWN_WATCHER} ]] && [[ ! -f ${ENDFILE} ]]
+  then
+    echo "$0 must not be run manually with -w or --watcher flags"
+    exit 1
+  fi 
+
+  if [[ ! -f ${RUNCPSERVER} ]]
+  then
+    echo "${RUNCPSERVER} does not exist"
+    echo "Please run command with correct log directory for Hue"
+    usage
+    exit 1
+  fi
+ 
+  if [[ ! "${ROTATE_SIZE}"  =~ ^[0-9]+$ ]]
+  then
+    echo "--max-log-size or -s must be a number"
+    exit 1
+  fi
+ 
+  OLDTIME=1800
+  # Get current and file times
+  CURTIME=$(date +%s)
+  FILETIME=$(stat ${RUNCPSERVER} -c %Y)
+  TIMEDIFF=$(expr ${CURTIME} - ${FILETIME})
+  
+  # Check if file older
+  if [ ${TIMEDIFF} -gt ${OLDTIME} ]; then
+    echo "${RUNCPSERVER} has not been written to in 30 mins"
+    echo "Be sure that ${DEKSTOP_LOG_DIR} is the correct log path"
+    echo "If it is then run:"
+    echo "touch ${RUNCPSERVER}"
+    echo "And run this command again"
+    echo "Otherwise run command with correct log directory for Hue"
+    echo "$0 --hue-log-dir /path/to/hue/logs"
+    echo "For example:"
+    echo "$0 --hue-log-dir /var/log/hue"
+    exit 1
+  fi
+  
+  if [[ ! -z ${SPAWN_WATCHER} ]]
+  then   
+    tail -F ${RUNCPSERVER} | egrep --line-buffered "\/download|downloaded|TExecuteStatementResp\(|[[:digit:]]{5,9}ms" >> ${LOG_FILE} &
+    PID=$!
+    while [[ -f ${ENDFILE} ]]
+    do
+      sleep 10
+      if [[ -f ${LOG_FILE} ]]
+      then
+        LOG_SIZE=$(du -sm ${LOG_FILE} | awk '{print $1}')
+        if [[ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]]
+        then
+          cat ${LOG_FILE} > ${LOG_FILE}.1
+          cat /dev/null > ${LOG_FILE}
+        fi
+      fi
+    done
+    kill -9 ${PID}
+    rm ${ENDFILE}
+  else
+    echo "Launching watcher daemon.  To end process run:"
+    echo "rm ${ENDFILE}"
+    touch ${ENDFILE}
+    exec nohup "${BASH_SOURCE[0]}" --watcher "${INITIAL_ARGS[@]}" 0<&- &> /dev/null &     
+  fi
+  
+  exit
+}
+  
+main "$@"
+  

+ 217 - 0
tools/ops/script_runner/hue_dump_user.sh

@@ -0,0 +1,217 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Test to search for doc1 and doc2
+
+#parse command line arguments
+parse_arguments()
+{
+  # Test that we're using compatible getopt version.
+  getopt -T > /dev/null
+  if [[ $? -ne 4 ]]; then
+    echo "Incompatible getopt version."
+    exit 1
+  fi
+
+  # Parse short and long option parameters.
+  OVERRIDE=
+  USERNAME=
+  VERBOSE=
+  DESKTOP_DEBUG=false
+  GETOPT=`getopt -n $0 -o o,u:,v,h \
+      -l override,username:,verbose,help \
+      -- "$@"`
+  eval set -- "$GETOPT"
+  while true;
+  do
+    case "$1" in
+    -o|--override)
+      OVERRIDE=true
+      shift
+      ;;
+    -u|--username)
+      USERNAME=$2
+      shift 2
+      ;;
+    -v|--verbose)
+      VERBOSE=true
+      DESKTOP_DEBUG=true
+      shift
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+    esac
+  done
+
+  #
+}
+
+usage()
+{
+cat << EOF
+usage: $0 [options]
+
+Exports all user objects:
+
+OPTIONS
+   -o|--override           Allow script to run as non-root, must set HUE_CONF_DIR manually before running
+   -u|--username <user>    User to export objects from.
+   -v|--verbose            Verbose logging, off by default
+   -h|--help               Show this message.
+EOF
+}
+
+main()
+{
+
+
+  parse_arguments "$@"
+
+  SCRIPT_DIR="$( cd -P "$( dirname "$0" )" && pwd )"
+  PYTHONPATH=${SCRIPT_DIR}/lib:${PYTHONPATH}
+  export SCRIPT_DIR PYTHONPATH
+
+  #SET IMPORTANT ENV VARS
+  if [[ -z ${HUE_CONF_DIR} ]]
+  then
+    if [ -d "/var/run/cloudera-scm-agent/process" ]
+    then
+      HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE_SERVER | sort -n | tail -1 `"
+    else
+      HUE_CONF_DIR="/etc/hue/conf"
+    fi
+    export HUE_CONF_DIR
+  fi
+
+  if [[ ! ${USER} =~ .*root* ]]
+  then
+    if [[ -z ${OVERRIDE} ]]
+    then
+      echo "Script must be run as root: exiting"
+      exit 1
+    fi
+  else
+    if [[ $(ps -ef | grep "[h]ue runc" | awk '{print }') ]]
+    then
+      DESKTOP_LOG_DIR=$(strings /proc/$(ps -ef | grep "[h]ue runc" | awk '{print }' | awk '{print $2}')/environ | grep DESKTOP_LOG_DIR | awk -F\= '{print $2}')
+    fi
+  fi
+
+  if [[ -z ${DESKTOP_LOG_DIR} ]]
+  then
+    DESKTOP_LOG_DIR=${HUE_CONF_DIR}/logs
+  fi
+  if [[ ! -f ${DESKTOP_LOG_DIR} ]]
+  then
+    mkdir -p ${DESKTOP_LOG_DIR}
+  fi
+  LOG_FILE=${DESKTOP_LOG_DIR}/`basename "$0" | awk -F\. '{print $1}'`.log
+  LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
+  LOG_ROTATE_COUNT=5 #number of log files, so 20MB max
+  
+  PARCEL_DIR=/opt/cloudera/parcels/CDH
+  if [ ! -d "/usr/lib/hadoop" ]
+  then
+    CDH_HOME=$PARCEL_DIR
+  else
+    CDH_HOME=/usr
+  fi
+
+  if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+  then
+    COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
+    TEST_COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue dbshell"
+  else
+    COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
+    TEST_COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue dbshell"
+  fi
+
+  ORACLE_ENGINE_CHECK=$(grep engine ${HUE_CONF_DIR}/hue* | grep -i oracle)
+  if [[ ! -z ${ORACLE_ENGINE_CHECK} ]]
+  then
+    if [[ -z ${ORACLE_HOME} ]]
+    then
+      ORACLE_PARCEL=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2
+      if [[ -d ${ORACLE_PARCEL} ]]
+      then
+        ORACLE_HOME=${ORACLE_PARCEL}
+        LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
+        export LD_LIBRARY_PATH ORACLE_HOME
+      fi
+    fi
+    if [[ -z ${ORACLE_HOME} ]]
+    then
+      echo "It looks like you are using Oracle as your backend"
+      echo "ORACLE_HOME must be set to the correct Oracle client"
+      echo "before running this script"
+      exit 1
+    fi
+  fi
+
+  HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1
+  if [[ -z ${HUE_DATABASE_PASSWORD} ]]
+  then
+    echo "CDH 5.5 and above requires that you set the environment variable:"
+    echo "HUE_DATABASE_PASSWORD=<dbpassword>"
+    exit 1
+  fi
+  export CDH_HOME COMMAND HUE_IGNORE_PASSWORD_SCRIPT_ERRORS
+
+#  echo "COMMAND: echo \"from django.db import connection; cursor = connection.cursor(); cursor.execute('select count(*) from auth_user')\" | ${TEST_COMMAND}" | tee -a ${LOG_FILE}
+#  echo "from django.db import connection; cursor = connection.cursor(); cursor.execute('select count(*) from auth_user')" | ${TEST_COMMAND} | tee -a ${LOG_FILE}
+  if [[ $? -ne 0 ]]
+  then
+    echo "DB connect test did not work, HUE_DATABASE_PASSWORD may not be correct"
+    echo "If the next query test fails check password in CM: http://<cmhostname>:7180/api/v5/cm/deployment and search for HUE_SERVER and database to find correct password"
+  fi
+
+#  ${COMMAND} <<EOF 2>&1 > /dev/null
+  ${COMMAND} <<EOF
+username = "${USERNAME}"
+LOGFILE = "${LOG_FILE}"
+logrotatesize=${LOG_ROTATE_SIZE}
+backupcount=${LOG_ROTATE_COUNT}
+
+import logging
+import logging.handlers
+from django.contrib.auth.models import User
+import customdumpdata
+
+LOG = logging.getLogger()
+format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+fh = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes = (1048576*logrotatesize), backupCount = backupcount)
+fh.setFormatter(format)
+LOG.addHandler(fh)
+LOG.setLevel(logging.INFO)
+
+user = User.objects.get(username = username)
+
+print ""
+print ""
+
+customdumpdata.Command().handle_noargs('desktop.Document', 'desktop.Document2', 'beeswax.SavedQuery', 'beeswax.QueryHistory', format='json', indent=2, exclude=[], use_natural_keys=True, use_base_manager=False, user=user)
+EOF
+
+}
+
+main "$@"

+ 21 - 0
tools/ops/script_runner/hue_history_cron.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Clean up old history to keep DB from growing too large
+
+SCRIPT_DIR="$( cd -P "$( dirname "$0" )" && pwd )"
+${SCRIPT_DIR}/script_runner hue_desktop_document_cleanup --keep-days 30

+ 130 - 0
tools/ops/script_runner/hue_multiple_home_cleanup.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Changes owner of Search Dashboard
+PARCEL_DIR=/opt/cloudera/parcels/CDH
+
+USAGE="usage: $0"
+
+OVERRIDE=$1
+
+if [[ ! ${USER} =~ .*root* ]]
+then
+  if [[ -z ${OVERRIDE} ]]
+  then
+    echo "Script must be run as root: exiting"
+    exit 1
+  fi
+fi
+
+if [ ! -d "/usr/lib/hadoop" ]
+then
+   CDH_HOME=$PARCEL_DIR
+else
+   CDH_HOME=/usr
+fi
+
+if [[ -z ${HUE_CONF_DIR} ]]
+then
+   if [ -d "/var/run/cloudera-scm-agent/process" ]
+   then
+      HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE_SERVER | sort -n | tail -1 `"
+   else
+      HUE_CONF_DIR="/etc/hue/conf"
+   fi
+   export HUE_CONF_DIR
+fi
+
+if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+then
+   COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
+else
+   COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
+fi
+
+if [[ -z ${ORACLE_HOME} ]]
+then
+   ORACLE_HOME=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2/
+   LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
+   export ORACLE_HOME LD_LIBRARY_PATH
+fi
+HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1
+export CDH_HOME COMMAND HUE_IGNORE_PASSWORD_SCRIPT_ERRORS
+
+echo "HUE_CONF_DIR: ${HUE_CONF_DIR}"
+echo "COMMAND: ${COMMAND}"
+
+${COMMAND} <<EOF
+from datetime import datetime
+from django.db import models
+from desktop.models import Document, Document2
+from django.contrib.auth.models import User
+
+for user in User.objects.filter():
+#  print user.username
+  last_modified=datetime.now()
+  oldest_doc=0L
+  movecount=1
+  homedir_count=0
+  for document in Document2.objects.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR):
+    homedir_count=homedir_count + 1
+  if homedir_count > 1:
+    print "%s has more than 1 homedir" % user.username
+    print "Fixing by moving newer ones to subdirectories"
+    for document in Document2.objects.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR):
+      homedir_count=homedir_count + 1
+      if document.last_modified < last_modified:
+        last_modified=document.last_modified
+        oldest_doc_id=document.id 
+    print "Oldest doc is %s" % oldest_doc_id
+    oldest_doc = Document2.objects.get(id=oldest_doc_id)
+    for document in Document2.objects.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR):
+      if document.id != oldest_doc_id:
+        document.name="scriptmoved%s" % movecount
+        document.parent_directory=oldest_doc
+        document.save()
+        movecount=movecount + 1
+
+
+for user in User.objects.filter():
+  print user.username
+  last_modified=datetime.now()
+  oldest_doc=0L
+  movecount=1
+  trashdir_count=0
+  for document in Document2.objects.filter(owner=user, name=Document2.TRASH_DIR):
+    print "User has a trash"
+    trashdir_count=trashdir_count + 1
+  if trashdir_count > 1:
+    print "%s has more than 1 trashdir" % user.username
+    print "Fixing by moving newer ones to subdirectories"
+    for document in Document2.objects.filter(owner=user, name=Document2.TRASH_DIR):
+      trashdir_count=trashdir_count + 1
+      if document.last_modified < last_modified:
+        last_modified=document.last_modified
+        oldest_doc_id=document.id
+    print "Oldest doc is %s" % oldest_doc_id
+    oldest_doc = Document2.objects.get(id=oldest_doc_id)
+    for document in Document2.objects.filter(owner=user, name=Document2.TRASH_DIR):
+      if document.id != oldest_doc_id:
+        document.name="scriptmoved%s" % movecount
+        document.parent_directory=oldest_doc
+        document.save()
+        movecount=movecount + 1
+
+
+EOF

+ 204 - 0
tools/ops/script_runner/hue_restart_cm.sh

@@ -0,0 +1,204 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Restarts Hue
+
+#parse command line arguments
+parse_arguments()
+{
+  # Test that we're using compatible getopt version.
+  getopt -T > /dev/null
+  if [[ $? -ne 4 ]]; then
+    echo "Incompatible getopt version."
+    exit 1
+  fi
+
+  # Parse short and long option parameters.
+  CM_HOSTNAME="localhost"
+  CM_PORT="7180"
+  CM_HTTP="http"
+  CM_API="v11"
+  CM_USERNAME="admin"
+  CM_PASSWORD_INPUT=
+  ENCODE_LOCATION=/var/lib/hue
+
+  GETOPT=`getopt -n $0 -o c:,p:,u:,w:,n,s,l:,h \
+      -l cmhost:,cmport:,cmuser:,cmpass:,newpass,ssl,encodeloc:,help \
+      -- "$@"`
+  eval set -- "$GETOPT"
+  while true;
+  do
+    case "$1" in
+    -c|--cmhost)
+      CM_HOSTNAME=$2
+      shift 2
+      ;;
+    -p|--cmport)
+      CM_PORT=$2
+      shift 2
+      ;;
+    -u|--cmuser)
+      CM_USERNAME=$2
+      shift 2
+      ;;
+    -w|--cmpass)
+      CM_PASSWORD_INPUT=$2
+      shift 2
+      ;;
+    -n|--newpass)
+      NEW_PASS=1
+      shift
+      ;;
+    -s|--ssl)
+      CM_HTTP="https"
+      shift
+      ;;
+    -l|--encodeloc)
+      ENCODE_LOCATION=$2
+      shift 2
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+    esac
+  done
+  #
+
+  ENC_PASSWORD_FILE=${ENCODE_LOCATION}/`basename "$0" | awk -F\. '{print $1}'`.enc
+}
+
+usage() {
+cat << EOF
+usage: $0 [options]
+
+Restarts Hue instances with high memory utilization through CM:
+
+OPTIONS
+   -c|--cmhost <hostname>      Host where CM is running - default localhost.
+   -p|--cmport <port>          Port CM is running on - default ${CM_PORT}.
+   -u|--cmuser <cm_user>       Admin User in CM - default admin.
+   -w|--cmpass <user_pass>     Admin User password in CM, required on first run, no default. Will prompt
+                               if not provided through this flag. Future runs will use
+                               encrypted version in <enc_loc>/`basename "$0" | awk -F\. '{print $1}'`.enc
+   -s|--ssl                    Enable SSL.
+   -n|--newpass                Prompt for a new password.
+   -l|--encodeloc <enc_loc>    Location to store encoded password in file - default /var/lib/hue.
+   -v|--verbose                Enable verbose logging.
+   -h|--help                   Show this message.
+EOF
+}
+
+main() {
+
+   parse_arguments "$@"
+
+   if [[ ! ${USER} =~ .*root.* ]]
+   then
+      echo "Script must be run as root: exiting"
+      exit 1
+   fi
+
+   if [[ ! -d ${ENCODE_LOCATION} ]]
+   then
+      mkdir -p ${ENCODE_LOCATION}
+   fi
+
+   if [[ ! -z ${CM_PASSWORD_INPUT} ]]
+   then
+      echo ${CM_PASSWORD_INPUT} | base64 > ${ENC_PASSWORD_FILE}
+      chown root:root ${ENC_PASSWORD_FILE}
+      chmod 600 ${ENC_PASSWORD_FILE}
+   fi
+
+   if [[ -z ${CM_PASSWORD_INPUT} ]]
+   then
+      if [[ ! -f ${ENC_PASSWORD_FILE} ]] || [[ ! -z ${NEW_PASS} ]]
+      then
+         message "CM Admin user password required on first run"
+         read -s -p "Please enter password:" CM_PASSWORD_INPUT
+         echo "New password provided"
+         echo ${CM_PASSWORD_INPUT} | base64 > ${ENC_PASSWORD_FILE}
+         chown root:root ${ENC_PASSWORD_FILE}
+         chmod 600 ${ENC_PASSWORD_FILE}
+      fi
+   fi
+
+   if [[ ! -f ${ENC_PASSWORD_FILE} ]]
+   then
+      message "CM Admin password has not been provided and this is"
+      message "is first run of the script.  Please run again and"
+      message "provide password."
+      exit 1
+   else
+      CM_PASSWORD=`cat ${ENC_PASSWORD_FILE} | base64 --decode`
+   fi
+
+   if [[ ${CM_HTTP} =~ .*https.* ]]
+   then
+      if [[ ${CM_PORT} =~ .*7180.* ]]
+      then
+         CM_PORT=7183
+      fi
+   fi
+
+   CLUSTERNAME=$(urlencode "$(curl -L -s -k -X GET -u ${CM_USERNAME}:${CM_PASSWORD} "${CM_HTTP}://${CM_HOSTNAME}:${CM_PORT}/api/${CM_API}/clusters" | grep '"name" :' | awk -F\" '{print $4}')")
+
+   SERVICENAME=$(urlencode "$(curl -L -s -k -X GET -u ${CM_USERNAME}:${CM_PASSWORD} "${CM_HTTP}://${CM_HOSTNAME}:${CM_PORT}/api/${CM_API}/clusters/${CLUSTERNAME}/services" | grep -B1 '"HUE"' | grep '"name" :' | awk -F\" '{print $4}')")
+
+   ROLES_JSON="{ \"items\" : [ \""
+
+   while read -r ROLE
+   do
+      ROLES_JSON="${ROLES_JSON}${ROLE}\",\""
+   done < <(curl -L -s -k -X GET -u ${CM_USERNAME}:${CM_PASSWORD} "${CM_HTTP}://${CM_HOSTNAME}:${CM_PORT}/api/${CM_API}/clusters/${CLUSTERNAME}/services/${SERVICENAME}/roles" | grep ${SERVICENAME}- | grep '"name" :' | awk -F\" '{print $4}')
+
+   ROLES_JSON=$(echo ${ROLES_JSON} | sed "s/,\"$/ ] }/g")
+
+   RESTART_API_URL="/api/${CM_API}/clusters/${CLUSTERNAME}/services/${SERVICENAME}/roleCommands/restart"
+
+   message "Restarting Hue process -u ${CM_USERNAME}:${CM_PASSWORD}: ${CM_HTTP}://${CM_HOSTNAME}:${CM_PORT}${RESTART_API_URL}: Roles: ${ROLES_JSON}"
+   RESULTS=`curl -s -X POST -u ${CM_USERNAME}:${CM_PASSWORD} -i -H "content-type:application/json" -d "${ROLES_JSON}" "${CM_HTTP}://${CM_HOSTNAME}:${CM_PORT}${RESTART_API_URL}"`
+}
+
+urlencode() {
+    # urlencode <string>
+    old_lc_collate=$LC_COLLATE
+    LC_COLLATE=C
+
+    local length="${#1}"
+    for (( i = 0; i < length; i++ )); do
+        local c="${1:i:1}"
+        case $c in
+            [a-zA-Z0-9.~_-]) printf "$c" ;;
+            *) printf '%%%02X' "'$c" ;;
+        esac
+    done
+
+    LC_COLLATE=$old_lc_collate
+}
+
+message()
+{
+  echo "$1"
+}
+
+main "$@"

+ 15 - 0
tools/ops/script_runner/lib/__init__.py

@@ -0,0 +1,15 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 32 - 0
tools/ops/script_runner/lib/adapter_factory.py

@@ -0,0 +1,32 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import cmf.monitor.generic.adapter
+import hue_adapters
+
+class AdapterFactory(object):
+  """
+  Factory for making monitoring Adapter classes.
+  """
+
+  def make_adapter(self, service_type, role_type, safety_valve, daemon = None):
+    """
+    Makes an Adapter for the input role type.
+    """
+    if service_type == "HUE" and role_type == "HUE_SERVER":
+      return hue_adapters.HueServerAdapter(safety_valve)
+    else:
+      return cmf.monitor.generic.adapter.Adapter(service_type, role_type, safety_valve)

+ 218 - 0
tools/ops/script_runner/lib/cm_environment.py

@@ -0,0 +1,218 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path
+import sys
+import logging
+import subprocess
+import re
+
+from hue_shared import which
+
+#logging.basicConfig()
+#logging = logging.getLogger(__name__)
+
+"""
+Class to configure Hue environment from CM
+"""
+
+def set_cm_environment():
+  """
+  Collect environment from CM supervisor
+  """
+  hue_config = {}
+  hue_bin_dir = "/usr/lib/hue"
+  hue_path = "/usr/lib/hue"
+  parcel_name = "CDH"
+  parcel_dir = "/opt/cloudera/parcels"
+  dbengine = None
+  cm_agent_process = subprocess.Popen('ps -ef | grep "[c]m agent\|[c]mf-agent" | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
+  cm_agent_pid = cm_agent_process.communicate()[0].split('\n')[0]
+  if cm_agent_pid != '':
+    try:
+      supervisor_process = subprocess.Popen('ps -ef | grep [s]upervisord | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
+      supervisor_pid = supervisor_process.communicate()[0].split('\n')[0]
+      cm_supervisor_dir = os.path.realpath('/proc/%s/cwd' % supervisor_pid)
+      if supervisor_pid == '':
+        logging.exception("This appears to be a CM enabled cluster and supervisord is not running")
+        logging.exception("Make sure you are running as root and CM supervisord is running")
+        sys.exit(1)
+    except Exception, e:
+      logging.exception("This appears to be a CM enabled cluster and supervisord is not running")
+      logging.exception("Make sure you are running as root and CM supervisord is running")
+      sys.exit(1)
+
+    #Parse CM supervisor include file for Hue and set env vars
+    cm_supervisor_dir = cm_supervisor_dir + '/include'
+    cm_agent_run_dir = os.path.dirname(cm_supervisor_dir)
+    hue_env_conf = None
+    envline = None
+    cm_hue_string = "HUE_SERVER"
+
+    for file in os.listdir(cm_supervisor_dir):
+      if cm_hue_string in file:
+        hue_env_conf = file
+
+    if not hue_env_conf == None:
+      if os.path.isfile(cm_supervisor_dir + "/" + hue_env_conf):
+        hue_env_conf_file = open(cm_supervisor_dir + "/" + hue_env_conf, "r")
+        logging.info("Setting CM managed environment using supervisor include: %s" % hue_env_conf_file)
+        for line in hue_env_conf_file:
+          if "environment" in line:
+            envline = line
+          if "directory" in line:
+            empty, hue_conf_dir = line.split("directory=")
+            os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
+            sys.path.append(os.environ["HUE_CONF_DIR"])
+  
+    if not envline == None:
+      empty, environment = envline.split("environment=")
+      for envvar in environment.split(","):
+        if "HADOOP_C" in envvar or "PARCEL" in envvar or "DESKTOP" in envvar or "ORACLE" in envvar or "LIBRARY" in envvar or "CMF" in envvar:
+          envkey, envval = envvar.split("=")
+          envval = envval.replace("'", "").rstrip()
+          if "LIBRARY" not in envkey:
+            os.environ[envkey] = envval
+          elif "LD_LIBRARY_PATH" not in os.environ.keys():
+            os.environ[envkey] = envval
+
+    if "PARCELS_ROOT" in os.environ:
+      parcel_dir = os.environ["PARCELS_ROOT"]
+
+    if "PARCEL_DIRNAMES" in os.environ:
+      parcel_names = os.environ["PARCEL_DIRNAMES"].split(':')
+      for parcel_name_temp in parcel_names:
+        if parcel_name_temp.startswith("CDH"):
+          parcel_name = parcel_name_temp
+
+
+    if os.path.isdir("%s/%s/lib/hue" % (parcel_dir, parcel_name)):
+      hue_path = "%s/%s/lib/hue" % (parcel_dir, parcel_name)
+    hue_bin_dir = "%s/build/env/bin" % hue_path
+
+    cloudera_config_script = None
+    if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
+      cloudera_config_script = '/usr/lib64/cmf/service/common/cloudera-config.sh'
+    elif os.path.isfile('/opt/cloudera/cm-agent/service/common/cloudera-config.sh'):
+      cloudera_config_script = '/opt/cloudera/cm-agent/service/common/cloudera-config.sh'
+
+    JAVA_HOME = None
+    if cloudera_config_script is not None:
+      locate_java = subprocess.Popen(['bash', '-c', '. %s; locate_java_home' % cloudera_config_script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      for line in iter(locate_java.stdout.readline,''):
+        if 'JAVA_HOME' in line:
+          JAVA_HOME = line.rstrip().split('=')[1]
+
+    if JAVA_HOME is not None:
+      os.environ["JAVA_HOME"] = JAVA_HOME
+
+    if "JAVA_HOME" not in os.environ:
+      print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
+      sys.exit(1)
+
+    hue_config["LD_LIBRARY_PATH"] = None
+    for line in open(os.environ["HUE_CONF_DIR"] + "/hue_safety_valve_server.ini"):
+      if re.search("engine=", line):
+        dbengine = line
+    if dbengine is None:
+      for line in open(os.environ["HUE_CONF_DIR"] + "/hue_safety_valve.ini"):
+        if re.search("engine=", line):
+          dbengine = line
+    if dbengine is None:
+      for line in open(os.environ["HUE_CONF_DIR"] + "/hue.ini"):
+        if re.search("engine=", line):
+          dbengine = line
+
+    if dbengine is not None and "oracle" in dbengine.lower():
+      #Make sure we set Oracle Client if configured
+      if "LD_LIBRARY_PATH" not in os.environ.keys():
+        if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
+          for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
+            if "ORACLE_INSTANT_CLIENT" in scm_script:
+              if os.path.isfile(scm_script):
+                oracle_source = subprocess.Popen(". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash")
+                for line in oracle_source.communicate()[0].splitlines():
+                  if "LD_LIBRARY_PATH" in line:
+                    var, oracle_ld_path = line.split("=")
+                    os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
+
+      if "LD_LIBRARY_PATH" not in os.environ.keys() or not os.path.isfile("%s/libclntsh.so.11.1" % os.environ["LD_LIBRARY_PATH"]):
+        print "You are using Oracle for backend DB"
+        if "LD_LIBRARY_PATH" in os.environ.keys():
+          print "LD_LIBRARY_PATH set to %s" % os.environ["LD_LIBRARY_PATH"]
+          print "LD_LIBRARY_PATH does not contain libclntsh.so.11.1"
+          print "Please set LD_LIBRARY_PATH correctly and rerun"
+
+        else:
+          print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
+          print "then it must be set, if not, you can ignore"
+
+        print "Here is an exmple, ONLY INCLUDE ONE PATH and NO VARIABLES"
+        print "  export LD_LIBRARY_PATH=/path/to/instantclient"
+        sys.exit(1)
+
+  else:
+    print "CM does not appear to be running on this server"
+    print "If this is a CM managed cluster make sure the agent and supervisor are running"
+    print "Running with /etc/hue/conf as the HUE_CONF_DIR"
+    os.environ["HUE_CONF_DIR"] = "/etc/hue/conf"
+
+  hue_config['hue_path'] = hue_path
+  hue_config['hue_bin_dir'] = hue_bin_dir
+  hue_config['HUE_CONF_DIR'] = os.environ["HUE_CONF_DIR"]
+  hue_config['parcel_name'] = parcel_name
+  hue_config['parcel_dir'] = parcel_dir
+  if dbengine is not None and "oracle" in dbengine.lower():
+    hue_config['LD_LIBRARY_PATH'] = os.environ["LD_LIBRARY_PATH"]
+
+  return hue_config
+
+def reload_with_cm_env():
+  try:
+    from django.db.backends.oracle.base import Oracle_datetime
+  except:
+    os.environ["SKIP_RELOAD"] = "True"
+    if 'LD_LIBRARY_PATH' in os.environ:
+      logging.info("We need to reload the process to include any LD_LIBRARY_PATH changes")
+      try:
+        os.execv(sys.argv[0], sys.argv)
+      except Exception, exc:
+        logging.warn('Failed re-exec:', exc)
+        sys.exit(1)
+
+def check_security():
+  from hadoop import conf
+  hdfs_config = conf.HDFS_CLUSTERS['default']
+  security_enabled = False
+  if hdfs_config.SECURITY_ENABLED.get():
+    KLIST = which('klist')
+    if KLIST is None:
+      logging.exception("klist is required, please install and rerun")
+      sys.exit(1)
+    klist_cmd = '%s | grep "Default principal"' % KLIST
+    klist_check = subprocess.Popen(klist_cmd, shell=True, stdout=subprocess.PIPE)
+    klist_princ = klist_check.communicate()[0].split(': ')[1]
+    if not 'hue/' in klist_princ:
+      logging.exception("klist failed, please contact support: %s" % klist_princ)
+      sys.exit(1)
+    logging.info("Security enabled using principal: %s" % klist_princ)
+    logging.info("You can imitate by running following export:")
+    logging.info("OSRUN: export KRB5CCNAME=%s" % os.environ['KRB5CCNAME'])
+    security_enabled = True
+
+  return security_enabled
+

+ 67 - 0
tools/ops/script_runner/lib/conversion_runner.py

@@ -0,0 +1,67 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import logging
+from django.contrib.auth.models import User
+from hue_converters import DocumentConverterHueScripts
+
+LOG = logging.getLogger(__name__)
+
+class DocumentConversionRunner(object):
+  """
+  Given a user, converts any existing Document objects to Document2 objects
+  """
+
+  def __init__(self, usernames, allowdupes=False, startqueryname=None, startuser=None):
+    self.usernames = usernames
+    self.allowdupes = allowdupes
+    self.startqueryname = startqueryname
+    self.startuser = startuser
+
+
+  def runconversions(self):
+    if not self.usernames:
+      users = User.objects.all()
+    else:
+      userlist = self.usernames.split(",")
+      users = User.objects.filter(username__in = userlist)
+
+    if self.startqueryname or self.startuser:
+      processdocs = False
+    else:
+      processdocs = True
+
+    LOG.info("Converting docs for %s users" % users.count())
+    for user in users:
+
+      LOG.info("Converting docs for user: %s" % user.username)
+      if user.username == self.startuser:
+        processdocs = True
+
+      start = time.time()
+
+      try:
+        converter = DocumentConverterHueScripts(user, allowdupes = self.allowdupes, startqueryname = self.startqueryname, startuser = self.startuser, processdocs = processdocs)
+        processdocs = converter.convertfailed()
+      except:
+        LOG.warn("Conversions failed for user: %s" % user.username)
+      end = time.time()
+      elapsed = (end - start) / 60
+      LOG.info("Finished user: %s : elapsed time: %s" % (user.username, elapsed))
+
+
+

+ 15 - 0
tools/ops/script_runner/lib/custom_commands/__init__.py

@@ -0,0 +1,15 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 15 - 0
tools/ops/script_runner/lib/custom_commands/management/__init__.py

@@ -0,0 +1,15 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 15 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/__init__.py

@@ -0,0 +1,15 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 306 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/backend_test_curl.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import logging
+import datetime
+import time
+import subprocess
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+import desktop.conf
+from desktop.conf import TIME_ZONE
+from search.conf import SOLR_URL, SECURITY_ENABLED as SOLR_SECURITY_ENABLED
+from liboozie.conf import OOZIE_URL, SECURITY_ENABLED as OOZIE_SECURITY_ENABLED
+from hadoop import conf as hdfs_conf
+from hadoop import cluster
+
+from hue_curl import Curl
+
+DEFAULT_LOG_DIR = 'logs'
+log_dir = os.getenv("DESKTOP_LOG_DIR", DEFAULT_LOG_DIR)
+
+current_milli_time = lambda: int(round(time.time() * 1000))
+
+def get_service_info(service):
+  service_info = {}
+  if service.lower() == 'solr':
+    service_info['url'] = SOLR_URL.get()
+    service_info['security_enabled'] = SOLR_SECURITY_ENABLED.get()
+  if service.lower() == 'oozie':
+    service_info['url'] = OOZIE_URL.get()
+    service_info['security_enabled'] = OOZIE_SECURITY_ENABLED.get()
+  if service.lower() == 'httpfs':
+    hdfs_config = hdfs_conf.HDFS_CLUSTERS['default']
+    service_info['url'] = hdfs_config.WEBHDFS_URL.get()
+    service_info['security_enabled'] = hdfs_config.SECURITY_ENABLED.get()
+  if service.lower() == 'rm':
+    yarn_cluster = cluster.get_cluster_conf_for_job_submission()
+    service_info['url'] = yarn_cluster.RESOURCE_MANAGER_API_URL.get()
+    service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
+  if service.lower() == 'jhs':
+    yarn_cluster = cluster.get_cluster_conf_for_job_submission()
+    service_info['url'] = yarn_cluster.HISTORY_SERVER_API_URL.get()
+    service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
+  if service.lower() == 'sparkhs':
+    yarn_cluster = cluster.get_cluster_conf_for_job_submission()
+    service_info['url'] = yarn_cluster.SPARK_HISTORY_SERVER_URL.get()
+    service_info['security_enabled'] = yarn_cluster.SPARK_HISTORY_SERVER_SECURITY_ENABLED.get()
+
+  if 'url' not in service_info:
+    logging.info("Hue does not have %s configured, cannot test %s" % (service, service))
+  elif service_info['url'] is None:
+    logging.info("Hue does not have %s configured, cannot test %s" % (service, service))
+
+  if service_info['url'].endswith('/'):
+    service_info['url'] = service_info['url'][:-1]
+
+  return service_info
+
+
+def add_service_test(available_services, options=None, service_name=None, testname=None, suburl=None, method='GET', teststring=None, test_options=None):
+  if options['service'] == "all" or options['service'] == service_name.lower():
+    if not service_name in available_services:
+      service_info = get_service_info(service_name)
+      url = service_info['url']
+      security_enabled = service_info['security_enabled']
+      available_services[service_name] = {}
+      available_services[service_name]['url'] = url
+      available_services[service_name]['security_enabled'] = security_enabled
+    # Tests
+    if not 'tests' in available_services[service_name]:
+      available_services[service_name]['tests'] = {}
+    if not testname in available_services[service_name]['tests']:
+      for test_option in test_options.keys():
+        suburl = suburl.replace(test_option, str(test_options[test_option]))
+      available_services[service_name]['tests'][testname] = {}
+      available_services[service_name]['tests'][testname]['url'] = '%s/%s' % (available_services[service_name]['url'], suburl)
+      available_services[service_name]['tests'][testname]['method'] = method
+      available_services[service_name]['tests'][testname]['test'] = teststring
+
+
+class Command(BaseCommand):
+  """
+  Handler for renaming duplicate User objects
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--service", help=_t("Comma separated services to test, all, httpfs, solr, oozie, rm, jhs, sparkhs."),
+                  action="store", default='all', dest='service'),
+      make_option("--testname", help=_t("Test for a given service, must only include one service name."),
+                  action="store", default=None, dest='testname'),
+      make_option("--testoptions", help=_t("Comma separated list of options for test. IE: oozie_job=0000778-190820133637006-oozie-oozi-C,getlogs=true"),
+                  action="store", default=None, dest='testoptions'),
+      make_option("--showcurl", help=_t("Show curl commands."),
+                  action="store_true", default=False, dest='showcurl'),
+      make_option("--response", help=_t("Show entire REST response."),
+                  action="store_true", default=False, dest='entireresponse'),
+      make_option("--username", help=_t("User to doAs."),
+                  action="store", default="admin", dest='username'),
+      make_option("--verbose", help=_t("Verbose."),
+                  action="store_true", default=False, dest='verbose'),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--service", help=_t("Comma separated services to test, all, httpfs, solr, oozie, rm, jhs, sparkhs."),
+                    action="store", default='all', dest='service'),
+        parser.add_argument("--testname", help=_t("Test for a given service, must only include one service name."),
+                    action="store", default=None, dest='testname'),
+        parser.add_argument("--testoptions", help=_t("Comma separated list of options for test. IE: oozie_job=0000778-190820133637006-oozie-oozi-C,getlogs=true"),
+                    action="store", default=None, dest='testoptions'),
+        parser.add_argument("--showcurl", help=_t("Show curl commands."),
+                    action="store_true", default=False, dest='showcurl'),
+        parser.add_argument("--response", help=_t("Show entire REST response."),
+                    action="store_true", default=False, dest='entireresponse'),
+        parser.add_argument("--username", help=_t("User to doAs."),
+                    action="store", default="admin", dest='username'),
+        parser.add_argument("--verbose", help=_t("Verbose."),
+                    action="store_true", default=False, dest='verbose')
+    else:
+      logging.warn(str(e))
+      sys.exit(1)
+
+  def handle(self, *args, **options):
+    test_options = {}
+    test_options['TIME_ZONE'] = TIME_ZONE.get()
+    test_options['DOAS'] = options['username']
+    test_options['NOW'] = current_milli_time()
+    test_options['NOWLESSMIN'] = test_options['NOW'] - 60000
+    if options['testoptions'] is not None:
+      for test_option in options['testoptions'].split(','):
+        option, option_value = test_option.split('=')
+        test_options[option.upper()] = option_value
+
+    test_services = options['service'].split(',')
+    supported_services = ['all', 'httpfs', 'solr', 'oozie', 'rm', 'jhs', 'sparkhs']
+    allowed_tests = {}
+    allowed_tests['httpfs'] = {}
+    allowed_tests['httpfs']['USERHOME'] = None
+
+    allowed_tests['jhs'] = {}
+    allowed_tests['jhs']['FINISHED'] = None
+
+    allowed_tests['oozie'] = {}
+    allowed_tests['oozie']['STATUS'] = None
+    allowed_tests['oozie']['CONFIGURATION'] = None
+    allowed_tests['oozie']['WORKFLOWS'] = None
+    allowed_tests['oozie']['COORDS'] = None
+    allowed_tests['oozie']['WORKFLOW'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
+    allowed_tests['oozie']['WORKFLOWLOG'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
+    allowed_tests['oozie']['WORKFLOWDEF'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
+    allowed_tests['oozie']['COORD'] = "oozie_id=0000001-190820133637006-oozie-oozi-C"
+
+    allowed_tests['rm'] = {}
+    allowed_tests['rm']['CLUSTERINFO'] = None
+
+    allowed_tests['solr'] = {}
+    allowed_tests['solr']['JMX'] = None
+
+    if options['testname'] is not None:
+      if len(test_services) > 1 or "all" in test_services:
+        logging.warn("When using --testname you must only submit one service name and you must not use all")
+        sys.exit(1)
+
+      if options['testname'] not in allowed_tests[options['service'].lower()].keys():
+        logging.warn("--testname %s not found in allowed_tests for service %s" % (options['testname'], options['service']))
+        logging.warn("Allowed tests for service:")
+        for test in allowed_tests[options['service'].lower()].keys():
+          if allowed_tests[options['service'].lower()][test] is None:
+            testoptions = "NONE"
+          else:
+            testoptions = allowed_tests[options['service'].lower()][test]
+          logging.warn("testname: %s : testoptions: %s" % (test, testoptions))
+        sys.exit(1)
+
+
+    if not any(elem in test_services for elem in supported_services):
+      logging.warn("Your service list does not contain a supported service: %s" % options['service'])
+      logging.warn("Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs")
+      logging.warn("Format: httpfs,solr,oozie")
+      sys.exit(1)
+
+    if not all(elem in supported_services for elem in test_services):
+      logging.warn("Your service list contains an unsupported service: %s" % options['service'])
+      logging.warn("Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs")
+      logging.warn("Format: httpfs,solr,oozie")
+      sys.exit(1)
+
+    if options['service'] == 'sparkhs':
+      logging.warn("Spark History Server not supported yet")
+      sys.exit(1)
+
+    logging.info("TEST: %s" % str(test_options['NOW']))
+    logging.info("Running REST API Tests on Services: %s" % options['service'])
+    curl = Curl(verbose=options['verbose'])
+
+    available_services = {}
+
+    #Add Solr
+    add_service_test(available_services, options=options, service_name="Solr", testname="JMX",
+                     suburl='jmx', method='GET', teststring='solr.solrxml.location', test_options=test_options)
+
+    #Add Oozie
+    if options['testname'] is None or options['testname'].upper() == "STATUS":
+        add_service_test(available_services, options=options, service_name="Oozie", testname="STATUS",
+                       suburl='v1/admin/status?timezone=TIME_ZONE&user.name=hue&doAs=DOAS', method='GET',
+                       teststring='{"systemMode":"NORMAL"}', test_options=test_options)
+
+    elif options['testname'].upper() == 'CONFIGURATION':
+        add_service_test(available_services, options=options, service_name="Oozie", testname="CONFIGURATION",
+                       suburl='v2/admin/configuration?timezone=TIME_ZONE&user.name=hue&doAs=DOAS', method='GET',
+                       teststring='{"oozie.email.smtp.auth', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOWS':
+        add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOWS",
+                       suburl='v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset=1&timezone=TIME_ZONE&jobtype=wf', method='GET',
+                       teststring='"workflows":[', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOW':
+        add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOW",
+                       suburl='v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS', method='GET',
+                       teststring='{"appName":', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOWLOG':
+        add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOWLOG",
+                       suburl='v2/job/OOZIE_ID?timezone=TIME_ZONE&show=log&user.name=hue&logfilter=&doAs=DOAS', method='GET',
+                       teststring='org.apache.oozie.service.JPAService: SERVER', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOWDEF':
+        add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOWDEF",
+                       suburl='v2/job/OOZIE_ID?timezone=TIME_ZONE&show=definition&user.name=hue&logfilter=&doAs=DOAS', method='GET',
+                       teststring='xmlns="uri', test_options=test_options)
+
+    elif options['testname'].upper() == 'COORDS':
+        add_service_test(available_services, options=options, service_name="Oozie", testname="COORDS",
+                       suburl='v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset=1&timezone=TIME_ZONE&jobtype=coord', method='GET',
+                       teststring='"coordinatorjobs":[', test_options=test_options)
+
+    elif options['testname'].upper() == 'COORD':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="COORD",
+                       suburl='v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS', method='GET',
+                       teststring='{"appName":', test_options=test_options)
+
+    #Add HTTPFS
+    add_service_test(available_services, options=options, service_name="Httpfs", testname="USERHOME",
+                     suburl='user/DOAS?op=GETFILESTATUS&user.name=hue&DOAS=%s', method='GET',
+                     teststring='"type":"DIRECTORY"', test_options=test_options)
+
+    #Add RM
+    add_service_test(available_services, options=options, service_name="RM", testname="CLUSTERINFO",
+                     suburl='ws/v1/cluster/info', method='GET', teststring='"clusterInfo"', test_options=test_options)
+
+    #Add JHS
+    add_service_test(available_services, options=options, service_name="JHS", testname="FINISHED",
+                     suburl='ws/v1/history/mapreduce/jobs?finishedTimeBegin=NOWLESSMIN&finishedTimeEnd=NOW', method='GET',
+                     teststring='{"jobs"', test_options=test_options)
+
+    for service in available_services:
+      for service_test in available_services[service]['tests']:
+        logging.info("Running %s %s Test:" % (service, service_test))
+        start_time = time.time()
+        response = curl.do_curl_available_services(available_services[service]['tests'][service_test])
+        returned_in = (time.time() - start_time) * 1000
+        if available_services[service]['tests'][service_test]['test'] in response:
+          logging.info("TEST: %s %s: Passed in %dms: %s found in response" % (service, service_test, returned_in, available_services[service]['tests'][service_test]['test']))
+          if options['entireresponse']:
+            logging.info("TEST: %s %s: Response: %s" % (service, service_test, response))
+        else:
+          logging.info("TEST: %s %s: Failed in %dms: Response: %s" % (service, service_test, returned_in, response))
+
+    log_file = log_dir + '/backend_test_curl.log'
+    print ""
+    print "Tests completed, view logs here: %s" % log_file
+    print "Report:"
+    cmd = 'grep -A1000 "%s" %s | grep "TEST:" | sed "s/.*INFO.*TEST:/  TEST:/g"' % (str(test_options['NOW']), log_file)
+    grep_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    grep_response = grep_process.communicate()[0]
+    print "%s" % grep_response
+    print ""
+    print "OS Repro Commands are:"
+    cmd = 'grep -A1000 "%s" %s | grep "OSRUN:" | sed "s/.*INFO.*OSRUN:/  /g"' % (str(test_options['NOW']), log_file)
+    grep_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    grep_response = grep_process.communicate()[0]
+    print "%s" % grep_response

+ 68 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/c6_test_command.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+import datetime
+import re
+import logging
+
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from django.contrib.auth.models import User
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for running queries from Hue log with database_logging queries
+    """
+
+    try:
+        from optparse import make_option
+        option_list = BaseCommand.option_list + (
+            make_option("--username", help=_t("User to delete case sensitive. "),
+                        action="store"),
+        )
+
+    except AttributeError, e:
+        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+        if baseoption_test:
+            def add_arguments(self, parser):
+                parser.add_argument("--username", help=_t("User to delete case sensitive."),
+                                    action="store")
+        else:
+            LOG.exception(str(e))
+            sys.exit(1)
+
+    def handle(self, *args, **options):
+        LOG.warn("Deleting user: %s" % options['username'])
+
+        usernames = [ options['username'] ]
+        try:
+            User.objects.filter(username__in=usernames).delete()
+        except Exception as e:
+            LOG.warn("EXCEPTION: deleting user %s failed: %s" % (options['username'], e))
+
+
+        LOG.debug("Done deleting user:  %s" % options['username'] )

+ 81 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/change_owner_of_docs.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+import datetime
+import re
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from django.contrib.auth.models import User
+from desktop.models import Document2
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for changing ownership of docs
+    """
+
+    try:
+        from optparse import make_option
+        option_list = BaseCommand.option_list + (
+            make_option("--olduser", help=_t("User who's docs need to change ownership. "),
+                        action="store"),
+            make_option("--newuser", help=_t("User who will own the docs. "),
+                        action="store"),
+        )
+
+    except AttributeError, e:
+        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+        if baseoption_test:
+            def add_arguments(self, parser):
+                parser.add_argument("--olduser", help=_t("User who's docs need to change ownership. "),
+                            action="store"),
+                parser.add_argument("--newuser", help=_t("User who will own the docs. "),
+                            action="store")
+
+        else:
+            LOG.exception(str(e))
+            sys.exit(1)
+
+    def handle(self, *args, **options):
+        LOG.warn("Changing ownership of all docs owned by %s to %s" % (options['olduser'], options['newuser']))
+
+        if not options['olduser']:
+            LOG.exception("--olduser option required")
+            sys.exit(1)
+
+        if not options['newuser']:
+            LOG.exception("--newuser option required")
+            sys.exit(1)
+
+        try:
+            newuser = User.objects.get(username = options['newuser'])
+            olduser = User.objects.get(username = options['olduser'])
+            docs = Document2.objects.filter(owner=olduser)
+            Document2.objects.filter(owner=olduser).update(owner=newuser)
+
+        except Exception as e:
+            LOG.warn("EXCEPTION: Changing ownership of %s's docs to %s failed: %s" % (options['olduser'], options['newuser'], e))

+ 164 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/db_query_test.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+import datetime
+import re
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from django.db.backends.oracle.base import Oracle_datetime
+from django.db import connection
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+  """
+  Handler for running queries from Hue log with database_logging queries
+  """
+
+  try:
+      from optparse import make_option
+      option_list = BaseCommand.option_list + (
+          make_option("--read-log-file", help=_t("Log file to scan for queries to be run "
+                                                 "from database_logging = true."),
+                      action="store",
+                      default='/var/log/hue/runcpserver.log'),
+          make_option("--start-url", help=_t("NOT WORKING: Hue URL to search for in the logs as a starting"
+                                             "point, uses the most recent instance. Requires --end-url"),
+                      action="store"),
+          make_option("--end-url", help=_t("NOT WORKING: Hue URL to search for in the logs as an ending"
+                                           "point, uses the most recent instance. Requires --start-url"),
+                      action="store"),
+          make_option("--start-time", help=_t("Start time to search for queries in log, format:"
+                                              '%d/%b/%Y %H:%M:%S IE: 01/Jan/2018 00:00:00: This is'
+                                              'standard Hue log format'),
+                      action="store",
+                      default=(datetime.datetime.now() - datetime.timedelta(minutes=2))),
+          make_option("--end-time", help=_t("End time to search for queries in log, format:"
+                                            '%d/%b/%Y %H:%M:%S IE: 01/Jan/2018 00:00:00: This is'
+                                            'standard Hue log format'),
+                      action="store",
+                      default=(datetime.datetime.now())),
+      )
+
+  except AttributeError, e:
+      baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+      if baseoption_test:
+          def add_arguments(self, parser):
+              parser.add_argument("--read-log-file", help=_t("Log file to scan for queries to be run "
+                                                 "from database_logging = true."),
+                      action="store",
+                      default='/var/log/hue/runcpserver.log'),
+              parser.add_argument("--start-url", help=_t("NOT WORKING: Hue URL to search for in the logs as a starting"
+                                             "point, uses the most recent instance. Requires --end-url"),
+                      action="store"),
+              parser.add_argument("--end-url", help=_t("NOT WORKING: Hue URL to search for in the logs as an ending"
+                                           "point, uses the most recent instance. Requires --start-url"),
+                      action="store"),
+              parser.add_argument("--start-time", help=_t("Start time to search for queries in log, format:"
+                                              '%d/%b/%Y %H:%M:%S IE: 01/Jan/2018 00:00:00: This is'
+                                              'standard Hue log format'),
+                      action="store",
+                      default=(datetime.datetime.now() - datetime.timedelta(minutes=2))),
+              parser.add_argument("--end-time", help=_t("End time to search for queries in log, format:"
+                                            '%d/%b/%Y %H:%M:%S IE: 01/Jan/2018 00:00:00: This is'
+                                            'standard Hue log format'),
+                      action="store",
+                      default=(datetime.datetime.now()))
+      else:
+          LOG.exception(str(e))
+          sys.exit(1)
+
+  def handle(self, *args, **options):
+    LOG.warn("Running database queries in file: %s: starting from: %s" % (options['read_log_file'], options['start_time']))
+
+    start = time.time()
+
+    oracleDatetimeRegex = re.compile(r"Oracle_datetime\([0-9,\ ]*\)")
+    paramsFixRegex = re.compile(r",\ ")
+    trueFixRegex = re.compile(r"True")
+    falseFixRegex = re.compile(r"False")
+
+    count = 1
+    with open(options['read_log_file'], 'rU') as f:
+      for line in f:
+         if "QUERY" in line:
+           junk, dateTemp = line.split('[')
+           dateTemp = dateTemp.split(" ")
+           if isinstance(options['start_time'], basestring):
+             start_time = datetime.datetime.strptime(options['start_time'], "%d/%b/%Y %H:%M:%S")
+           else:
+             start_time = options['start_time']
+           if isinstance(options['end_time'], basestring):
+             end_time = datetime.datetime.strptime(options['end_time'], "%d/%b/%Y %H:%M:%S")
+           else:
+             end_time = options['end_time']
+           log_time = datetime.datetime.strptime(dateTemp[0] + " " + dateTemp[1], "%d/%b/%Y %H:%M:%S")
+           if start_time <= log_time <= end_time:
+             queryStart = time.time()
+             line = oracleDatetimeRegex.sub("PLACEHOLDER", line)
+             junk, query = line.split("QUERY = u'")
+             query, param_base = query.split("' - PARAMS = (")
+             params, junk = param_base.split(");")
+
+             if params.endswith(','):
+               params = params[:-1]
+
+             params = paramsFixRegex.sub(",", params)
+             params = params.split(',')
+
+             loopCount = len(params) - 1
+             while(loopCount >= 0):
+               i = loopCount
+               loopCount = loopCount - 1
+               updateArgsRegex = re.compile(r":arg%d" % i)
+               if params[i] == "PLACEHOLDER":
+                 query = updateArgsRegex.sub("'%s'" % Oracle_datetime.from_datetime(datetime.datetime.now()), query)
+               else:
+                 if "oracle" in desktop.conf.DATABASE.ENGINE.get():
+                   params[i] = trueFixRegex.sub("1", params[i], re.IGNORECASE)
+                   params[i] = falseFixRegex.sub("0", params[i], re.IGNORECASE)
+               query = updateArgsRegex.sub(params[i], query)
+
+             cursor = connection.cursor()
+             cursor.execute(query)
+             try:
+               row = cursor.fetchone()
+             except:
+               LOG.warn("EXCEPTION: fetchone failed for query: %s" % query)
+
+#              FETCH MANY MAY BE NEEDED
+#              rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)
+#              while rows:
+#                  rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)
+
+             count = count + 1
+             queryEnd = time.time()
+             queryElapsed = (queryEnd - queryStart)
+             LOG.debug("Query time elapsed (seconds): %s: query: %s" % (queryElapsed, query))
+             LOG.debug("")
+
+    end = time.time()
+    elapsed = (end - start)
+    LOG.debug("Total queries: %s: time elapsed (seconds): %.2f" % (count, elapsed))

+ 67 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/delete_user.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+import datetime
+import re
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from django.contrib.auth.models import User
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for deleting users
+    """
+
+    try:
+        from optparse import make_option
+        option_list = BaseCommand.option_list + (
+            make_option("--username", help=_t("User to delete case sensitive. "),
+                        action="store"),
+        )
+
+    except AttributeError, e:
+        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+        if baseoption_test:
+            def add_arguments(self, parser):
+                parser.add_argument("--username", help=_t("User to delete case sensitive."),
+                                    action="store")
+        else:
+            LOG.exception(str(e))
+            sys.exit(1)
+
+    def handle(self, *args, **options):
+        LOG.warn("Deleting user: %s" % options['username'])
+
+        usernames = [ options['username'] ]
+        try:
+            User.objects.filter(username__in=usernames).delete()
+        except Exception as e:
+            LOG.warn("EXCEPTION: deleting user %s failed: %s" % (options['username'], e))
+
+
+        LOG.debug("Done deleting user: %s" % options['username'] )

+ 178 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/estimate_concurrent_users.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import sys
+import logging
+import heapq
+import datetime
+import time
+import subprocess
+from collections import OrderedDict
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+import desktop.conf
+from desktop.conf import TIME_ZONE
+from search.conf import SOLR_URL, SECURITY_ENABLED as SOLR_SECURITY_ENABLED
+from liboozie.conf import OOZIE_URL, SECURITY_ENABLED as OOZIE_SECURITY_ENABLED
+from hadoop import conf as hdfs_conf
+from hadoop import cluster
+
+DEFAULT_LOG_DIR = 'logs'
+log_dir = os.getenv("DESKTOP_LOG_DIR", DEFAULT_LOG_DIR)
+
+class Command(BaseCommand):
+  """
+  Handler for renaming duplicate User objects
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--today", help=_t("Estimate users for today."),
+                  action="store_true", default=False, dest='today'),
+      make_option("--logdir", help=_t("Specify directory to process access logs."),
+                  action="store", default=log_dir, dest='logdir'),
+      make_option("--increment", help=_t("Increments to count users, hour, min10, day"),
+                  action="store", default="day", dest='increment'),
+      make_option("--date", help=_t("Estimate users for date.  In form of YYYY-MM-DD"),
+                  action="store", default=False, dest='date'),
+      make_option("--last10", help=_t("Process logs for last 10 minutes."),
+                  action="store_true", default=False, dest='last10'),
+      make_option("--last1h", help=_t("Process logs for last hour."),
+                  action="store_true", default=False, dest='last1h'),
+      make_option("--includejb", help=_t("Include Jobbrowser entries."),
+                  action="store_true", default=False, dest='includejb'),
+      make_option("--verbose", help=_t("Verbose."),
+                  action="store_true", default=False, dest='verbose'),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--today", help=_t("Estimate users for today."),
+                    action="store_true", default=False, dest='today'),
+        parser.add_argument("--logdir", help=_t("Specify directory to process access logs."),
+                    action="store", default=log_dir, dest='logdir'),
+        parser.add_argument("--increment", help=_t("Increments to count users, hour, min10, day"),
+                    action="store", default="day", dest='increment'),
+        parser.add_argument("--date", help=_t("Estimate users for date.  In form of YYYY-MM-DD"),
+                    action="store", default=False, dest='date'),
+        parser.add_argument("--last10", help=_t("Process logs for last 10 minutes."),
+                    action="store_true", default=False, dest='last10'),
+        parser.add_argument("--last1h", help=_t("Process logs for last hour."),
+                    action="store_true", default=False, dest='last1h'),
+        parser.add_argument("--includejb", help=_t("Include Jobbrowser entries."),
+                    action="store_true", default=False, dest='includejb'),
+        parser.add_argument("--verbose", help=_t("Verbose."),
+                    action="store_true", default=False, dest='verbose')
+    else:
+      logging.warn(str(e))
+      sys.exit(1)
+
+  def handle(self, *args, **options):
+    if options['date']:
+      now = datetime.datetime.strptime(options['date'], '%Y-%m-%d')
+    else:
+      now = datetime.datetime.now()
+      minus10 = now - datetime.timedelta(minutes=10)
+      minus1h = now - datetime.timedelta(minutes=60)
+
+    date = now - datetime.timedelta(days=1999)
+    previous_date = now - datetime.timedelta(days=2000)
+    totalconcurrent = 0
+    userlist = []
+    numlist = []
+
+    regex = re.compile(
+      # Example line
+      # [20/Jun/2017 04:40:07 -0700] DEBUG    172.31.112.36 -anon- - "HEAD /desktop/debug/is_alive HTTP/1.1"
+      r'\['
+      r'(?P<date>'
+      r'\d{2}/\w{3}/\d{4} '  # Parse Date in form of '25/Oct/2015'
+      r'\d{2}:\d{2}:\d{2}'  # Parse Time in form of '12:34:56'
+      r') '
+      r'[-+]?\d{4}'  # Ignore the timezone
+      r'\] '
+      r'(?P<level>\w+) +'
+      r'(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) '
+      r'(?P<user>\w+) '
+      r'\S+ "'  # Ignore unknown
+      r'(?P<method>\w+) '
+      r'(?P<url>\S+) '
+    )
+
+    for filename in sorted(os.listdir(options['logdir']), reverse=True):
+      if not filename.startswith("access"):
+        continue  # Only process access log files
+
+      for line in open(options['logdir'] + "/" + filename).xreadlines():
+        if not line.startswith("["):
+          continue  # Only process lines that start with a date
+
+        # Make sure this log entry is a user access
+        m = regex.match(line)
+        if m:
+          previous_date = date
+          date = datetime.datetime.strptime(m.group('date'), '%d/%b/%Y %H:%M:%S')
+
+          if not options['includejb']:
+            if re.match(m.group('url'), '/jobbrowser/jobs/$'):
+              continue
+
+          if options['today']:
+            if \
+                    date.year != now.year or \
+                            date.month != now.month or \
+                            date.day != now.day:
+              continue
+
+          if options['last10']:
+            # Skip anything older than 10 mins ago
+            if date < minus10:
+              continue
+
+          if options['last1h']:
+            # Skip anything older than 1 hour ago
+            if date < minus1h:
+              continue
+
+          user = m.group('user')
+
+          if previous_date.day == date.day:
+            if not user == "-anon-":
+              userlist.append(user)
+          else:
+            newuserlist = list(OrderedDict.fromkeys(userlist))
+            userlist = []
+            totalconcurrent = len(newuserlist)
+            numlist.append(totalconcurrent)
+
+    newuserlist = list(OrderedDict.fromkeys(userlist))
+    totalconcurrent = len(newuserlist)
+    numlist.append(totalconcurrent)
+    # Sort the list and remove any unique values
+    numlist = sorted(set(numlist))
+    # Print the top 10 most concurrent counts
+    logging.warn("largest: %s" % heapq.nlargest(10, numlist))
+    # print "newuserlist: %s" % newuserlist
+    # print "userlist: %s" % userlist

+ 149 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/hue_desktop_document_cleanup.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+from importlib import import_module
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from beeswax.models import SavedQuery
+from beeswax.models import Session
+from datetime import date, timedelta
+from oozie.models import Workflow
+from django.db.utils import DatabaseError
+import desktop.conf
+from desktop.models import Document2
+import logging
+import logging.handlers
+
+
+import desktop.conf
+
+logging.basicConfig()
+LOG = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+    """
+    Handler for purging old Query History, Workflow documents and Session data
+    """
+
+    try:
+        from optparse import make_option
+        option_list = BaseCommand.option_list + (
+            make_option("--keep-days", help=_t("Number of days of history data to keep."),
+                    action="store",
+                    type=int,
+                    default=30),
+        )
+
+    except AttributeError, e:
+        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+        if baseoption_test:
+            def add_arguments(self, parser):
+                parser.add_argument("--keep-days", help=_t("Number of days of history data to keep."),
+                    action="store",
+                    type=int,
+                    default=30)
+        else:
+            LOG.exception(str(e))
+            sys.exit(1)
+
+
+    def objectCleanup(self, objClass, filterType, filterValue, dateField):
+        errorCount = 0
+        checkCount = 0
+        resets = 0
+        deleteRecords = self.deleteRecordsBase
+
+        totalObjects = objClass.objects.filter(**{ '%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj, })\
+                                                .values_list("id", flat=True)
+        LOG.info("Looping through %s objects. %s objects to be deleted." % (objClass.__name__, totalObjects.count()))
+        while totalObjects.count():
+            if deleteRecords < 30 and resets < self.resetMax:
+                checkCount += 1
+            if checkCount == self.resetCount:
+                deleteRecords = self.deleteRecordsBase
+                resets += 1
+                checkCount = 0
+            LOG.info("%s objects left: %s" % (objClass.__name__, totalObjects.count()))
+            deleteObjects = objClass.objects.filter(**{ '%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj, })\
+                                                    .values_list("id", flat=True)[:deleteRecords]
+            try:
+                objClass.objects.filter(pk__in=list(deleteObjects)).delete()
+                errorCount = 0
+            except DatabaseError, e:
+                LOG.info("Non Fatal Exception: %s: %s" % (e.__class__.__name__, e))
+                errorCount += 1
+                if errorCount > 9 and deleteRecords == 1:
+                    raise
+                if deleteRecords > 100:
+                    deleteRecords = max(deleteRecords - 100, 1)
+                else:
+                    deleteRecords = max(deleteRecords - 10, 1)
+                LOG.info("Decreasing max delete records to: %s" % deleteRecords)
+            totalObjects = objClass.objects.filter(**{'%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj, })\
+                                                    .values_list("id", flat=True)
+
+
+    def handle(self, *args, **options):
+
+
+        self.keepDays = options['keep_days']
+        self.timeDeltaObj = date.today() - timedelta(days=self.keepDays)
+        self.resetCount = 15
+        self.resetMax = 5
+        self.deleteRecordsBase = 999  #number of documents to delete in a batch
+                                      #to avoid Non Fatal Exception: DatabaseError: too many SQL variables
+
+        LOG.info("Cleaning up anything in the Hue tables django_session, oozie*, desktop* and beeswax* older than %s old" % self.keepDays)
+
+        start = time.time()
+
+
+        #Clean out Hive / Impala Query History
+        self.objectCleanup(SavedQuery, 'is_auto', True, 'mtime')
+
+        #Clear out old Hive/Impala sessions
+        self.objectCleanup(Session, 'status_code__gte', -10000, 'last_used')
+
+        #Clean out Trashed Workflows
+        self.objectCleanup(Workflow, 'is_trashed', True, 'last_modified')
+
+        #Clean out Workflows without a name
+        self.objectCleanup(Workflow, 'name', '', 'last_modified')
+
+        #Clean out history Doc2 objects
+        self.objectCleanup(Document2, 'is_history', True, 'last_modified')
+
+        #Clean out expired sessions
+        LOG.debug("Cleaning out expired sessions from django_session table")
+
+        engine = import_module(settings.SESSION_ENGINE)
+        try:
+            engine.SessionStore.clear_expired()
+        except NotImplementedError:
+            LOG.error("Session engine '%s' doesn't support clearing "
+                            "expired sessions.\n" % settings.SESSION_ENGINE)
+
+
+        end = time.time()
+        elapsed = (end - start)
+        LOG.debug("Total time elapsed (seconds): %.2f" % elapsed)

+ 73 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/list_groups.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+import datetime
+import re
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from django.contrib.auth.models import User, Group
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for listing groups and groups associated with a user
+    """
+
+    try:
+        from optparse import make_option
+        option_list = BaseCommand.option_list + (
+            make_option("--username", help=_t("Groups this user belongs to . "),
+                        action="store", default=None),
+        )
+
+    except AttributeError, e:
+        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+        if baseoption_test:
+            def add_arguments(self, parser):
+                parser.add_argument("--username", help=_t("Groups this user belongs to."),
+                                    action="store", default=None)
+        else:
+            LOG.exception(str(e))
+            sys.exit(1)
+
+    def handle(self, *args, **options):
+        LOG.info("Listing Hue groups")
+        try:
+          if options['username'] != None:
+            LOG.info("Listing groups for %s" % options['username'])
+            user = User.objects.get(username = options['username'])
+            groups = user.groups.all()
+            for group in groups:
+              print group.name
+          else:
+            LOG.info("Listing all groups")
+            groups = Group.objects.all()
+            for group in groups:
+              print group.name
+
+        except Exception as e:
+            LOG.warn("EXCEPTION: Listing groups failed, %s" % e)

+ 66 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/promote_to_superuser.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+import datetime
+import re
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from django.contrib.auth.models import User
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for promoting a user to superuser
+    """
+
+    try:
+        from optparse import make_option
+        option_list = BaseCommand.option_list + (
+            make_option("--username", help=_t("User to delete case sensitive. "),
+                        action="store"),
+        )
+
+    except AttributeError, e:
+        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+        if baseoption_test:
+            def add_arguments(self, parser):
+                parser.add_argument("--username", help=_t("User to delete case sensitive."),
+                                    action="store")
+        else:
+            LOG.exception(str(e))
+            sys.exit(1)
+
+    def handle(self, *args, **options):
+        LOG.warn("Promoting user %s to superuser" % options['username'])
+
+        try:
+            new_super = User.objects.get(username = options['username'])
+            new_super.is_superuser = True
+            new_super.save()
+
+        except Exception as e:
+            LOG.warn("EXCEPTION: promoting user %s to superuser failed: %s" % (options['username'], e))

+ 92 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/remove_doc2_without_content_object.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+from django.db import models
+from django.contrib.contenttypes.models import ContentType
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+import desktop.conf
+from desktop.models import Document, Document2
+from django.contrib.auth.models import User
+import desktop.conf
+
+import logging
+import logging.handlers
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for purging cleaning up doc2 objets missing content objects
+    """
+
+    def handle(self, *args, **options):
+
+        LOG.info("Cleaning up Doc2 objects missing content object")
+
+        model_class = Document2
+        extra = "workflow2"
+        qfilter = None
+
+        idlist = []
+
+        for user in User.objects.filter():
+            LOG.info("Idlist: %s" % idlist)
+            LOG.info("User: user: %s" % user.username)
+            docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')
+            duplicated_records = Document.objects.values('name', 'owner').annotate(name_count=models.Count('name')).filter(name_count__gt=1, owner = user)
+            names = []
+            for obj in duplicated_records:
+                names.append(obj["name"])
+
+            duplicate_doc_ids = []
+            doclist = Document.objects.values('id', 'name', 'owner').filter(owner = user)
+            for obj in doclist:
+                if obj["name"] in names:
+                    duplicate_doc_ids.append(obj["id"])
+
+            LOG.info("Docs: docs: %s" % docs)
+
+            if model_class is not None:
+                ct = ContentType.objects.get_for_model(model_class)
+                docs = docs.filter(content_type=ct)
+
+            if extra is not None:
+                docs = docs.filter(extra=extra)
+
+            if qfilter is not None:
+                docs = docs.filter(qfilter)
+
+            LOG.info("Grabbing only duplicates")
+            docs = docs.filter(id__in=duplicate_doc_ids)
+            for d in docs:
+                try:
+                    if d.content_object is None:
+                        LOG.info("Adding document id %s with name %s and owner %s to idlist" % (d.id, d.name, user.username))
+                        idlist.append(d.id)
+                except:
+                    pass
+
+
+        deldocs = Document.objects.filter(id__in=idlist)
+        LOG.info("Docs to delete are: %s" % deldocs)
+        deldocs.delete()

+ 59 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/remove_duplicate_user_preferences.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from desktop.models import UserPreferences
+from django.db import models, transaction
+from django.contrib.auth.models import User
+
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for deleting duplicate UserPreference objects
+    """
+
+    def handle(self, *args, **options):
+        LOG.warn("Deleting ducpliate UserPreference objects")
+
+        for user in User.objects.filter():
+            duplicated_records = UserPreferences.objects \
+               .values('user', 'key') \
+               .annotate(key_count=models.Count('key')) \
+               .filter(key_count__gt=1, user = user)
+            # Delete all but the first document.
+            for record in duplicated_records:
+                preferences = UserPreferences.objects \
+                    .values_list('id', flat=True) \
+                    .filter(
+                        user = user,
+                        key = record['key'],
+                    )[1:]
+                preferences = list(preferences)
+                LOG.warn("Deleting UserPreferences duplicate ids: %s" % preferences)
+                UserPreferences.objects.filter(id__in=preferences).delete()
+
+
+        transaction.commit()

+ 112 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/remove_orphaned_docs.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import uuid
+
+from importlib import import_module
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from datetime import date, timedelta
+from django.db.utils import DatabaseError
+import desktop.conf
+from desktop.models import Directory, Document, Document2
+from django.contrib.auth.models import User
+from desktop.auth.backend import find_or_create_user, rewrite_user, ensure_has_a_group
+from useradmin.models import get_profile, get_default_user_group, UserProfile
+from notebook.connectors.base import get_api, Notebook
+from oozie.models2 import Workflow
+import logging
+import logging.handlers
+
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+  """
+  Handler for moving orphaned docs
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--keep-days", help=_t("Number of days of history data to keep."),
+          action="store",
+          type=int,
+          default=30),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--keep-days", help=_t("Number of days of history data to keep."),
+          action="store",
+          type=int,
+          default=30)
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+  def handle(self, *args, **options):
+
+    LOG.info("Removing any orphaned docs")
+
+    start = time.time()
+
+    totalUsers = User.objects.filter().values_list("id", flat=True)
+    totalDocs = Document2.objects.exclude(owner_id__in=totalUsers)
+    docstorage_id = "docstorage" + str(uuid.uuid4())
+    docstorage_id = docstorage_id[:30]
+    LOG.info("Creating new owner for all orphaned docs: %s" % docstorage_id)
+    docstorage = find_or_create_user(docstorage_id)
+    docstorage = rewrite_user(docstorage)
+    userprofile = get_profile(docstorage)
+    userprofile.first_login = False
+    userprofile.save()
+    ensure_has_a_group(docstorage)
+    new_home_dir = Document2.objects.create_user_directories(docstorage)
+
+    for doc in totalDocs:
+      if not doc.type == "directory":
+        new_dir_name = "recover-" + str(doc.owner_id)
+        new_sub_dir = Directory.objects.create(name=new_dir_name, owner=docstorage, parent_directory=new_home_dir)
+        doc1 = doc.doc.get()
+        doc.owner = docstorage
+        doc1.owner = docstorage
+        doc.parent_directory = new_sub_dir
+        doc.save()
+        doc1.save()
+        Document.objects.sync()
+        LOG.info("Migrating orphaned doc: %s : %s : %s : %s : to orphaned doc owner: %s" % (doc.name, doc.type, doc.owner_id, doc.parent_directory, docstorage_id))
+
+    for doc in totalDocs:
+      if doc.type == "directory":
+        LOG.info("Deleting orphaned directory: %s : %s : %s" % (doc.name, doc.type, doc.owner_id))
+        doc.delete()
+
+
+    end = time.time()
+    elapsed = (end - start)
+    LOG.info("Total time elapsed (seconds): %.2f" % elapsed)

+ 155 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/rename_duplicate_users.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+import datetime
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+from django.contrib.auth.models import User
+from django.db import models, transaction
+from django.contrib.auth.models import User
+
+
+import desktop.conf
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+  """
+  Handler for renaming duplicate User objects
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--renamecase", help=_t("Rename permanent user to be all lowercase, uppercase or NONE."),
+                  action="store", default="lowercase", dest='renamecase'),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--olduser", help=_t("Rename permanent user to be all lowercase, uppercase or NONE."),
+                            action="store", default="lowercase", dest='renamecase')
+
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+
+  def rename_user(self, rename_user=None, keep_user=None, count=None):
+    new_username = rename_user + "renamed"
+    LOG.warn("%s is newer than %s, renaming %s to %s" % (rename_user, keep_user, rename_user, new_username))
+    self.users_dict[self.username][count]["username"] = new_username
+    user_mod = User.objects.get(username=rename_user)
+    user_mod.username = new_username
+    user_mod.save()
+
+
+  def change_user_case(self, username=None, newcase=None):
+    tmp_username = username + "tmp"
+    if newcase == "lowercase":
+      new_username = username.lower()
+    elif newcase == "uppercase":
+      new_username = username.upper()
+    else:
+      new_username = username
+
+    if "renamed" not in username and username != new_username:
+      LOG.warn("Changing user case, renaming %s to %s" % (username, new_username))
+      user_mod = User.objects.get(username=username)
+      user_mod.username = new_username
+      user_mod.save()
+
+
+  def log_users_list(self):
+    users_unsorted = []
+    for user in User.objects.filter():
+      users_unsorted.append(user.username)
+
+    users_sorted = sorted(users_unsorted, key=lambda s: s.lower())
+    count = 0
+    while count < len(users_sorted):
+      LOG.warn("%s" % users_sorted[count])
+      count = count + 1
+
+
+  def handle(self, *args, **options):
+    LOG.warn("Deleting duplicate User objects")
+
+    self.users_dict = {}
+    self.username = None
+
+    LOG.warn("users list before renames")
+    self.log_users_list()
+
+
+    for user in User.objects.filter():
+      user_list = [{'username': user.username, 'date_joined': user.date_joined, 'date_joined_readable': user.date_joined.strftime('%Y-%m-%d %H:%M:%S%z')}]
+      self.users_dict[user.username.lower()] = user_list
+      for usercompare in User.objects.filter():
+        if usercompare.id != user.id and usercompare.username.lower() == user.username.lower():
+          self.users_dict[user.username.lower()].append({'username': usercompare.username, 'date_joined': usercompare.date_joined, 'date_joined_readable': usercompare.date_joined.strftime('%Y-%m-%d %H:%M:%S%z')})
+
+    for username in self.users_dict.keys():
+      self.username = username
+      count = 0
+      oldest_user = None
+      oldest_date = None
+      while count < len(self.users_dict[self.username]):
+        current_dict = self.users_dict[self.username]
+        if oldest_user is None:
+          username1 = current_dict[count]['username']
+          date1 = current_dict[count]['date_joined']
+          username2 = current_dict[count + 1]['username']
+          date2 = current_dict[count + 1]['date_joined']
+          if date1 < date2:
+            oldest_user = username1
+            oldest_date = date1
+            oldest_count = count
+            self.rename_user(rename_user=username2, keep_user=oldest_user, count=count + 1)
+          else:
+            oldest_user = username2
+            oldest_date = date2
+            oldest_count = count + 1
+            self.rename_user(rename_user=username1, keep_user=oldest_user, count=count)
+        else:
+          username2 = current_dict[count]['username']
+          date2 = current_dict[count]['date_joined']
+          if username2.lower() == self.username and username2 != oldest_user:
+            if oldest_date < date2:
+              self.rename_user(rename_user=username2, keep_user=oldest_user, count=count)
+            else:
+              self.rename_user(rename_user=oldest_user, keep_user=username2, count=oldest_count)
+              oldest_user = username2
+              oldest_date = date2
+              oldest_count = count
+
+        count = count + 1
+
+    LOG.warn("renaming all users to be %s" % options['renamecase'])
+    for user in User.objects.filter():
+      self.change_user_case(username=user.username, newcase=options['renamecase'])
+
+    LOG.warn("users list after renames")
+    self.log_users_list()
+
+    transaction.commit()

+ 133 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/run_hive_impala_query.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import logging
+import datetime
+import time
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+from django.contrib.auth.models import User
+
+import desktop.conf
+
+logging.basicConfig()
+LOG = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+  """
+  Handler for renaming duplicate User objects
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--hive", help=_t("Run Hive query."),
+                  action="store_true", default=False, dest='runhive'),
+      make_option("--impala", help=_t("Run Impala query."),
+                  action="store_true", default=True, dest='runimpala'),
+      make_option("--username", help=_t("User to run query as."),
+                  action="store", default="admin", dest='username'),
+      make_option("--query", help=_t("Query to run."),
+                  action="store", default="select * from default.sample_07;", dest='query'),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--hive", help=_t("Run Hive query."),
+                    action="store_true", default=False, dest='runhive'),
+        parser.add_argument("--impala", help=_t("Run Impala query."),
+                    action="store_true", default=True, dest='runimpala'),
+        parser.add_argument("--username", help=_t("User to run query as."),
+                    action="store", default="admin", dest='username'),
+        parser.add_argument("--query", help=_t("Query to run."),
+                    action="store", default="select * from default.sample_07;", dest='query')
+
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+
+  def handle(self, *args, **options):
+    hue, created = User.objects.get_or_create(username=options['username'])
+    if options['runhive']:
+      query_backend = 'hive'
+      from beeswax.server import dbms
+      from beeswax.conf import HIVE_SERVER_HOST
+      SERVER_HOST = HIVE_SERVER_HOST
+      db = dbms.get(hue)
+    else:
+      query_backend = 'impala'
+      from impala import conf
+      from impala import dbms
+      from beeswax.server import dbms as beeswax_dbms
+      SERVER_HOST = conf.SERVER_HOST.get()
+      query_server = dbms.get_query_server_config()
+      db = beeswax_dbms.get(hue, query_server=query_server)
+
+    LOG.info("QUERY_BACKEND: %s" % query_backend)
+    LOG.info("QUERY_USER: %s" % options['username'])
+    LOG.info("QUERY: %s" % options['query'])
+    LOG.info("QUERY_HOST: %s" % SERVER_HOST)
+
+    start = time.time()
+
+    db.get_tables()
+
+    executequery = options['query']
+    query = db.execute_statement(executequery)
+
+    LOG.info(db.get_log(query.get_handle()))
+
+    while True:
+      ret = db.get_state(query.get_handle())
+      try:
+        LOG.info("ret.key: %s" % ret.key)
+        LOG.info("ret: %s" % ret)
+        if ret.key != 'running':
+          break
+      except AttributeError:
+#    submitted = 0
+#    running = 1
+#    available = 2
+#    failed = 3
+#    expired = 4
+        LOG.info("ret: %s" % ret.value)
+        if ret.value != 1:
+          break
+        pass
+      time.sleep(1)
+      LOG.info("Waiting for query execution")
+
+    result = db.fetch(query.get_handle())
+
+    i = 0
+    for row in result.rows():
+      LOG.info("row: %s" % row)
+      if i > 100:
+        break
+      i += 1
+
+    LOG.info(db.get_log(query.get_handle()))
+    end = time.time()
+    elapsed = (end - start) / 60
+    LOG.info("Time elapsed (minutes): %.2f" % elapsed)

+ 88 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/set_default_editor.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import logging
+import datetime
+import time
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+from django.contrib.auth.models import User
+from desktop.models import set_user_preferences
+
+import desktop.conf
+
+logging.basicConfig()
+LOG = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+  """
+  Handler for renaming duplicate User objects
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--hive", help=_t("Set Hive as default."),
+                  action="store_true", default=False, dest='sethive'),
+      make_option("--impala", help=_t("Set Impala as default."),
+                  action="store_true", default=False, dest='setimpala'),
+      make_option("--username", help=_t("User to set."),
+                  action="store", default="all", dest='username'),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--hive", help=_t("Set Hive as default."),
+                    action="store_true", default=False, dest='sethive'),
+        parser.add_argument("--impala", help=_t("Set Impala as default."),
+                    action="store_true", default=False, dest='setimpala'),
+        parser.add_argument("--username", help=_t("User to set."),
+                    action="store", default="all", dest='username'),
+
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+
+  def handle(self, *args, **options):
+    key = "default_app"
+    set_props = None
+    if options['sethive']:
+      set_props = '{"app":"editor","interpreter":"hive"}'
+      editor = "hive"
+    if options['setimpala']:
+      set_props = '{"app":"editor","interpreter":"impala"}'
+      editor = "impala"
+    if set_props is None:
+      set_props = '{"app":"editor","interpreter":"impala"}'
+      editor = "impala"
+
+    if options['username'] != "all":
+      LOG.info("Setting default interpreter to %s for user %s" % (editor, options['username']))
+      user = User.objects.get(username = options['username'])
+      set_user_preferences(user, key, set_props)
+
+    else:
+      for user in User.objects.filter():
+        LOG.info("Setting default interpreter to %s for user %s" % (editor, options['username']))
+        set_user_preferences(user, key, set_props)

+ 152 - 0
tools/ops/script_runner/lib/custom_commands/management/commands/share_all_workflows.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
+import desktop.conf
+from desktop.models import Document2
+from django.contrib.auth.models import User, Group
+import desktop.conf
+
+import logging
+import logging.handlers
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Handler for sharing workflows
+    """
+
+    try:
+        from optparse import make_option
+        option_list = BaseCommand.option_list + (
+            make_option("--shareusers", help=_t("Comma separated list of users to share all workflows with."),
+                        action="store"),
+            make_option("--sharegroups", help=_t("Comma separated list of groups to share all workflows with."),
+                        action="store"),
+            make_option("--owner", help=_t("Give permissions to only workflows owned by this user."),
+                        action="store"),
+            make_option("--permissions", help=_t("Comma separated list of permissions for the users and groups."
+                                                 "read, write or read,write"), action="store"),
+        )
+
+    except AttributeError, e:
+        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+        if baseoption_test:
+            def add_arguments(self, parser):
+                parser.add_argument("--shareusers", help=_t("Comma separated list of users to share all workflows with."),
+                        action="store"),
+                parser.add_argument("--sharegroups", help=_t("Comma separated list of groups to share all workflows with."),
+                        action="store"),
+                parser.add_argument("--owner", help=_t("Give permissions to only workflows owned by this user."),
+                        action="store"),
+                parser.add_argument("--permissions", help=_t("Comma separated list of permissions for the users and groups."
+                                                 "read, write or read,write"), action="store")
+        else:
+            LOG.exception(str(e))
+            sys.exit(1)
+
+    def handle(self, *args, **options):
+
+        if not options['shareusers'] and not options['sharegroups']:
+            LOG.warn("You must set either shareusers or sharegroups or both")
+            sys.exit(1)
+
+        if not options['permissions']:
+            LOG.warn("permissions option required either read, write or read,write")
+            sys.exit(1)
+
+        if options['shareusers']:
+            users = options['shareusers'].split(",")
+        else:
+            users = []
+
+        if options['sharegroups']:
+            groups = options['sharegroups'].split(",")
+        else:
+            groups = []
+
+        perms = options['permissions'].split(",")
+
+        LOG.info("Setting permissions %s on all workflows for users: %s" % (perms, users))
+        LOG.info("Setting permissions %s on all workflows for groups: %s" % (perms, groups))
+
+        shareusers = User.objects.filter(username__in=users)
+        sharegroups = Group.objects.filter(name__in=groups)
+
+        doc_types = ['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2']
+        workflow_owner = User.objects.get(username = options['owner'])
+
+        if options['owner']:
+            LOG.info("Only setting permissions for workflows owned by %s" % options['owner'])
+            oozie_docs = Document2.objects.filter(type__in=doc_types, owner = workflow_owner)
+        else:
+            oozie_docs = Document2.objects.filter(type__in=doc_types)
+
+        for perm in perms:
+            if perm in ['read', 'write']:
+                for oozie_doc in oozie_docs:
+                    owner = User.objects.get(id = oozie_doc.owner_id)
+                    read_perms = oozie_doc.to_dict()['perms']['read']
+                    write_perms = oozie_doc.to_dict()['perms']['write']
+
+                    read_users = []
+                    write_users = []
+                    read_groups = []
+                    write_groups = []
+
+                    for user in read_perms['users']:
+                        read_users.append(user['id'])
+
+                    for group in read_perms['groups']:
+                        read_groups.append(group['id'])
+
+                    for user in write_perms['users']:
+                        write_users.append(user['id'])
+
+                    for group in write_perms['groups']:
+                        write_groups.append(group['id'])
+
+                    for user in shareusers:
+                        if perm == 'read':
+                            read_users.append(user.id)
+
+                        if perm == 'write':
+                            write_users.append(user.id)
+
+                    for group in sharegroups:
+                        if perm == 'read':
+                            read_groups.append(group.id)
+
+                        if perm == 'write':
+                            write_groups.append(group.id)
+
+                    if perm == 'read':
+                        users = User.objects.in_bulk(read_users)
+                        groups = Group.objects.in_bulk(read_groups)
+
+                    if perm == 'write':
+                        users = User.objects.in_bulk(write_users)
+                        groups = Group.objects.in_bulk(write_groups)
+
+                    LOG.warn("Setting %s on %s for users: %s : groups: %s" % (perm, oozie_doc.name, users, groups))
+                    oozie_doc.share(owner, name=perm, users=users, groups=groups)

+ 21 - 0
tools/ops/script_runner/lib/custom_commands/settings.py

@@ -0,0 +1,21 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DJANGO_APPS = [ "custom_commands" ]
+NICE_NAME = "Custom Commands"
+REQUIRES_HADOOP = False
+MENU_INDEX = 60
+

+ 245 - 0
tools/ops/script_runner/lib/customdumpdata.py

@@ -0,0 +1,245 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import NoArgsCommand, CommandError, OutputWrapper
+from django.core import serializers
+from django.db import router, DEFAULT_DB_ALIAS
+from django.utils.datastructures import SortedDict
+from django.contrib.auth.models import User
+
+from optparse import make_option
+import logging
+import sys
+
+#class Command(BaseCommand):
+#    option_list = BaseCommand.option_list + (
+#        make_option('--format', default='json', dest='format',
+#            help='Specifies the output serialization format for fixtures.'),
+#        make_option('--indent', default=None, dest='indent', type='int',
+#            help='Specifies the indent level to use when pretty-printing output'),
+#        make_option('--database', action='store', dest='database',
+#            default=DEFAULT_DB_ALIAS, help='Nominates a specific database to dump '
+#                'fixtures from. Defaults to the "default" database.'),
+#        make_option('-e', '--exclude', dest='exclude',action='append', default=[],
+#            help='An appname or appname.ModelName to exclude (use multiple --exclude to exclude multiple apps/models).'),
+#        make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
+#            help='Yay CHRIS Use natural keys if they are available.'),
+#        make_option('-a', '--all', action='store_true', dest='use_base_manager', default=False,
+#            help="Use Django's base manager to dump all models stored in the database, including those that would otherwise be filtered or modified by a custom manager."),
+#        make_option('--pks', dest='primary_keys', help="Only dump objects with "
+#            "given primary keys. Accepts a comma seperated list of keys. "
+#            "This option will only work when you specify one model."),
+#    )
+#    help = ("Output the contents of the database as a fixture of the given "
+#            "format (using each model's default manager unless --all is "
+#            "specified).")
+#    args = '[appname appname.ModelName ...]'
+
+#    def handle(self, *app_labels, **options):
+class Command(NoArgsCommand):
+  def handle_noargs(self, *app_labels, **options):
+        from django.db.models import get_app, get_apps, get_model
+
+        format = options.get('format')
+        indent = options.get('indent')
+        using = options.get('database')
+        excludes = options.get('exclude')
+        show_traceback = options.get('traceback')
+        use_natural_keys = options.get('use_natural_keys')
+        use_base_manager = options.get('use_base_manager')
+        pks = options.get('primary_keys')
+        user = options.get('user')
+        userid = user.id
+
+        stdout = OutputWrapper(options.get('stdout', sys.stdout))
+
+        if pks:
+            primary_keys = pks.split(',')
+        else:
+            primary_keys = []
+
+        excluded_apps = set()
+        excluded_models = set()
+        if excludes:
+            for exclude in excludes:
+                if '.' in exclude:
+                    app_label, model_name = exclude.split('.', 1)
+                    model_obj = get_model(app_label, model_name)
+                    if not model_obj:
+                        raise CommandError('Unknown model in excludes: %s' % exclude)
+                    excluded_models.add(model_obj)
+                else:
+                    try:
+                        app_obj = get_app(exclude)
+                        excluded_apps.add(app_obj)
+                    except ImproperlyConfigured:
+                        raise CommandError('Unknown app in excludes: %s' % exclude)
+
+        if len(app_labels) == 0:
+            if primary_keys:
+                raise CommandError("You can only use --pks option with one model")
+            app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps)
+        else:
+            if len(app_labels) > 1 and primary_keys:
+                raise CommandError("You can only use --pks option with one model")
+            app_list = SortedDict()
+            for label in app_labels:
+                try:
+                    app_label, model_label = label.split('.')
+                    try:
+                        app = get_app(app_label)
+                    except ImproperlyConfigured:
+                        raise CommandError("Unknown application: %s" % app_label)
+                    if app in excluded_apps:
+                        continue
+                    model = get_model(app_label, model_label)
+                    if model is None:
+                        raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
+
+                    if app in app_list.keys():
+                        if app_list[app] and model not in app_list[app]:
+                            app_list[app].append(model)
+                    else:
+                        app_list[app] = [model]
+                except ValueError:
+                    if primary_keys:
+                        raise CommandError("You can only use --pks option with one model")
+                    # This is just an app - no model qualifier
+                    app_label = label
+                    try:
+                        app = get_app(app_label)
+                    except ImproperlyConfigured:
+                        raise CommandError("Unknown application: %s" % app_label)
+                    if app in excluded_apps:
+                        continue
+                    app_list[app] = None
+
+        # Check that the serialization format exists; this is a shortcut to
+        # avoid collating all the objects and _then_ failing.
+        if format not in serializers.get_public_serializer_formats():
+            try:
+                serializers.get_serializer(format)
+            except serializers.SerializerDoesNotExist:
+                pass
+
+            raise CommandError("Unknown serialization format: %s" % format)
+
+        def get_objects():
+            # Collate the objects to be serialized.
+            for model in sort_dependencies(app_list.items()):
+                if model in excluded_models:
+                    continue
+                if not model._meta.proxy and router.allow_syncdb(using, model):
+                    if use_base_manager:
+                        objects = model._base_manager
+                    else:
+                        objects = model._default_manager
+
+                    queryset = objects.using(using).order_by(model._meta.pk.name)
+                    if primary_keys:
+                        queryset = queryset.filter(pk__in=primary_keys)
+                    queryset = queryset.filter(owner_id=userid)
+                    for obj in queryset.iterator():
+                        yield obj
+
+        try:
+            stdout.ending = None
+#            self.stdout.ending = None
+            serializers.serialize(format, get_objects(), indent=indent,
+                    use_natural_keys=use_natural_keys, stream=stdout)
+            stdout
+#            serializers.serialize(format, get_objects(), indent=indent,
+#                    use_natural_keys=use_natural_keys, stream=self.stdout)
+        except Exception as e:
+            if show_traceback:
+                raise
+            raise CommandError("Unable to serialize database: %s" % e)
+
+def sort_dependencies(app_list):
+    """Sort a list of app,modellist pairs into a single list of models.
+
+    The single list of models is sorted so that any model with a natural key
+    is serialized before a normal model, and any model with a natural key
+    dependency has it's dependencies serialized first.
+    """
+    from django.db.models import get_model, get_models
+    # Process the list of models, and get the list of dependencies
+    model_dependencies = []
+    models = set()
+    for app, model_list in app_list:
+        if model_list is None:
+            model_list = get_models(app)
+
+        for model in model_list:
+            models.add(model)
+            # Add any explicitly defined dependencies
+            if hasattr(model, 'natural_key'):
+                deps = getattr(model.natural_key, 'dependencies', [])
+                if deps:
+                    deps = [get_model(*d.split('.')) for d in deps]
+            else:
+                deps = []
+
+            # Now add a dependency for any FK or M2M relation with
+            # a model that defines a natural key
+            for field in model._meta.fields:
+                if hasattr(field.rel, 'to'):
+                    rel_model = field.rel.to
+                    if hasattr(rel_model, 'natural_key') and rel_model != model:
+                        deps.append(rel_model)
+            for field in model._meta.many_to_many:
+                rel_model = field.rel.to
+                if hasattr(rel_model, 'natural_key') and rel_model != model:
+                    deps.append(rel_model)
+            model_dependencies.append((model, deps))
+
+    model_dependencies.reverse()
+    # Now sort the models to ensure that dependencies are met. This
+    # is done by repeatedly iterating over the input list of models.
+    # If all the dependencies of a given model are in the final list,
+    # that model is promoted to the end of the final list. This process
+    # continues until the input list is empty, or we do a full iteration
+    # over the input models without promoting a model to the final list.
+    # If we do a full iteration without a promotion, that means there are
+    # circular dependencies in the list.
+    model_list = []
+    while model_dependencies:
+        skipped = []
+        changed = False
+        while model_dependencies:
+            model, deps = model_dependencies.pop()
+
+            # If all of the models in the dependency list are either already
+            # on the final model list, or not on the original serialization list,
+            # then we've found another model with all it's dependencies satisfied.
+            found = True
+            for candidate in ((d not in models or d in model_list) for d in deps):
+                if not candidate:
+                    found = False
+            if found:
+                model_list.append(model)
+                changed = True
+            else:
+                skipped.append((model, deps))
+        if not changed:
+            raise CommandError("Can't resolve dependencies for %s in serialized app list." %
+                ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
+                for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
+            )
+        model_dependencies = skipped
+
+    return model_list

+ 97 - 0
tools/ops/script_runner/lib/doc2_utils.py

@@ -0,0 +1,97 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import re
+import logging
+import difflib
+
+from desktop.models import Document2
+#DOC2_NAME_INVALID_CHARS = "[<>/{}[\]~`u'\xe9'u'\xfa'u'\xf3'u'\xf1'u'\xed']"
+DOC2_NAME_INVALID_CHARS = "[<>/{}\[\]]"
+
+LOG = logging.getLogger(__name__)
+
+def removeInvalidChars(fixString):
+  fixString = re.sub(r'[^\x00-\x7f]',r'', fixString)
+  return re.sub(DOC2_NAME_INVALID_CHARS, '', fixString)
+
+
+def findMatchingQuery(user, id, name, query, include_history=False, all=False, values=False):
+#Returns list of matching queries.  If all = False
+#returns at first found for speed
+  name = removeInvalidChars(name)
+#  LOG.info("finding queries that match name: %s" % name)
+  documents = getSavedQueries(user=user, name=name, include_history=include_history)
+  matchdocs = []
+  matchvalues = []
+   
+  for doc in documents:
+    if all == True or not matchdocs:
+      matchdata = json.loads(doc.data)
+      matchname = removeInvalidChars(doc.name)
+#      LOG.info("found name: matchname: %s" % matchname)
+      if 'snippets' in matchdata:
+        matchquery = matchdata['snippets'][0]['statement_raw']
+        if re.match(name, matchname) and id != doc.id:
+#          LOG.info("Query name: %s and matchname: %s are similar" % (name, matchname))
+#          LOG.info("Comparing queries:")
+          if query == matchquery:
+#            LOG.info("MATCHED QUERY: name: %s: id: %s" % (name, id))
+            matchdocs.append(doc) 
+            matchvalues.append(doc.id)
+
+  if values == False:
+#    LOG.info("returning %s matching docs" % len(matchdocs))
+    return matchdocs
+  else:
+#    LOG.info("returning %s matching doc ids" % len(matchdocs))
+    return matchvalues
+
+
+def getSavedQueries(user, name=None, include_history=False):
+#mimic api call to get saved queries
+  perms = 'both'
+  include_trashed = False
+  flatten = True
+  if name:
+#    LOG.info("getting queries that match name: %s" % name)
+    if include_history:
+      documents = Document2.objects.filter(name__contains=name, owner=user, type__in=['query-hive', 'query-impala'])
+#      documents = Document2.objects.filter(name__iregex=r'%s.*' %name, owner=user, type__in=['query-hive', 'query-impala'])
+    else:
+      documents = Document2.objects.filter(name__contains=name, owner=user, type__in=['query-hive', 'query-impala'], is_history=include_history)
+#      documents = Document2.objects.filter(name__iregex=r'%s.*' %name, owner=user, type__in=['query-hive', 'query-impala'], is_history=include_history)
+  else:
+#    LOG.info("getting all queries")
+    if include_history:
+      documents = Document2.objects.filter(
+        user=user,
+        include_trashed=include_trashed,
+        type__in=['query-hive', 'query-impala']
+      )
+    else:
+      documents = Document2.objects.filter(
+        user=user,
+        include_history=include_history,
+        include_trashed=include_trashed,
+        type__in=['query-hive', 'query-impala']
+      )
+
+#  LOG.info("returning queries, total count: %s" % len(documents))
+  return documents
+
+

+ 109 - 0
tools/ops/script_runner/lib/doc_count_util.py

@@ -0,0 +1,109 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import time
+import re
+import sys
+
+from django.db import transaction
+
+from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission
+
+LOG = logging.getLogger(__name__)
+
+class DocumentCounts(object):
+  """
+  Given a user, prints the count of doc1 and doc2 documents 
+  """
+
+  def __init__(self, user, differ=False):
+    LOG.info("Sai")
+    self.user = user
+    self.differ = differ
+    self.home_dir = Document2.objects.create_user_directories(self.user)
+    sys.stdout = open('counts.txt', 'a')
+    
+
+  def printCounts(self):
+    LOG.info("HI")
+    # Convert SavedQuery documents
+    try:
+      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
+
+      docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
+      self.saved_query_doc1 = len(docs)
+
+      docs = Document2.objects.filter(owner=self.user, type__startswith='query-').filter(is_history=False)
+      self.saved_query_doc2 = len(docs)
+    except:
+      LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
+      pass
+
+    # Convert SQL Query history documents
+    try:
+      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
+
+      docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
+      self.query_history_doc1 = len(docs)
+
+      docs = Document2.objects.filter(owner=self.user,type__startswith='query-').filter(is_history=True)
+      self.query_history_doc2 = len(docs)
+
+    except:
+      LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
+      pass
+    
+    if self.differ:
+      if self.saved_query_doc2 != self.saved_query_doc1 or self.query_history_doc2 != self.query_history_doc1:
+        print("%-20s  %s/%s \t  %s/%s" %(self.user.username, self.saved_query_doc2, self.saved_query_doc1, self.query_history_doc2, self.query_history_doc1))
+    else: 
+      print("%-20s  %s/%s \t  %s/%s" %(self.user.username, self.saved_query_doc2, self.saved_query_doc1, self.query_history_doc2, self.query_history_doc1))
+    sys.stdout.close()
+
+
+  def _get_unconverted_docs(self, content_type, with_history=False):
+    docs = Document.objects.get_docs(self.user, content_type).filter(owner=self.user)
+
+    tags = [
+      DocumentTag.objects.get_trash_tag(user=self.user), # No trashed docs
+      DocumentTag.objects.get_example_tag(user=self.user), # No examples
+#      self.imported_tag # No already imported docs
+    ]
+
+    if not with_history:
+      tags.append(DocumentTag.objects.get_history_tag(user=self.user)) # No history yet
+
+    return docs.exclude(tags__in=tags)
+
+
+  def _get_parent_directory(self, document):
+    """
+    Returns the parent directory object that should be used for a given document. If the document is tagged with a
+        project name (non-RESERVED DocumentTag), a Directory object with the first project tag found is returned.
+        Otherwise, the owner's home directory is returned.
+    """
+    parent_dir = self.home_dir
+    project_tags = document.tags.exclude(tag__in=DocumentTag.RESERVED)
+    if project_tags.exists():
+      first_tag = project_tags[0]
+      parent_dir, created = Directory.objects.get_or_create(
+          owner=self.user,
+          name=first_tag.tag,
+          parent_directory=self.home_dir
+      )
+    return parent_dir

+ 353 - 0
tools/ops/script_runner/lib/hue_adapters.py

@@ -0,0 +1,353 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright (c) 2015 Cloudera, Inc. All rights reserved.
+
+import logging
+import time
+
+from cmf.monitor import schema
+from cmf.monitor.abstract_monitor import CollectionError
+from cmf.monitor.constants import WEB_METRICS_COLLECTION_GOOD, \
+  DEFAULT_MONITOR_LOG_RATE, WEB_METRICS_COLLECTION_COMMUNICATION_FAILURE
+from cmf.monitor.generic import AbstractMetricCollector
+from cmf.monitor.generic.adapter import Adapter
+from cmf.monitor.generic.service_defined_metrics import ServiceDefinedMetrics
+from cmf.monitor.generic.utils import SplittingSourceProcessor, visit_json, \
+  SimpleMetricsExtractor
+from cmf.throttling_logger import ThrottlingLogger
+from url_util import head_request_with_timeout
+
+logging.basicConfig()
+LOG = logging.getLogger('HueServerAdapter')
+THROTTLED_LOG = ThrottlingLogger(LOG, DEFAULT_MONITOR_LOG_RATE)
+
+class _HueIsAliveCollector(AbstractMetricCollector):
+  """
+  A collector that generates web-metric collection metrics from the hue is alive
+  end-point.
+  """
+
+  def __init__(self, adapter):
+    AbstractMetricCollector.__init__(self, adapter)
+    if not isinstance(adapter, HueServerAdapter):
+      raise Exception("Unsupported adapter type")
+
+  def update_with_conf(self, conf):
+    LOG.info("hue_adapters: _HueIsAliveCollector: update_with_conf: metrics_uri: %s" % self._adapter._get_is_alive_url(conf))
+    self._metrics_uri = self._adapter._get_is_alive_url(conf)
+
+  def collect_and_parse(self, conf):
+    start = time.time()
+    try:
+      LOG.info("hue_adapters: _HueIsAliveCollector: _call_is_alive: metrics_uri: %s" % self._metrics_uri)
+      self._call_is_alive(self._metrics_uri)
+      result = WEB_METRICS_COLLECTION_GOOD
+    except Exception, ex:
+      THROTTLED_LOG.exception("Error calling is alive at '%s'" %
+                              (self._metrics_uri,))
+      result = CollectionError(WEB_METRICS_COLLECTION_COMMUNICATION_FAILURE)
+    now = time.time()
+    self._metrics_tuple = result, now, int((now - start) * 1000)
+
+  def _add_metrics(self, service_version, update, collected_metrics, duration):
+    update.add_metric(schema.WEB_METRICS_COLLECTION_DURATION, duration)
+    if isinstance(collected_metrics, CollectionError):
+      update.add_metric(schema.WEB_METRICS_COLLECTION_STATUS,
+                        collected_metrics.code)
+    else:
+      update.add_metric(schema.WEB_METRICS_COLLECTION_STATUS,
+                        collected_metrics)
+    return True
+
+  def is_ready_to_report(self, conf, pid):
+    if pid is None:
+      raise Exception("Pid cannot be None")
+
+    metrics_url = self._adapter._get_is_alive_url(conf)
+    if metrics_url is None:
+      return AbstractMetricCollector._COLLECTOR_NOT_SUPPORTED
+    try:
+      self._call_is_alive(metrics_url, 0.5)
+      return True
+    except Exception, ex:
+      pass
+    return False
+
+  def _call_is_alive(self, is_alive_url, timeout=None):
+    """
+    We need this function to make testing eaiser.
+    """
+    head_request_with_timeout(is_alive_url, timeout=timeout)
+
+class HueServerAdapter(Adapter):
+  """
+  An adapter that collects and generates metrics for the HUE_SERVER. The adapter
+  collects metrics from a sample file generated by hue. It also generates
+  web-metric-collection metrics using an is-alive endpoint exposed by hue.
+  Note that the web-metric-collection metrics generation is done as part of the
+  sample file metric collection. The reason is that the hue is-alive end-point
+  accepts HTTP HEAD requests and we can't use the regular web metric collector
+  to generate these metics.
+  """
+
+  _HUE_SERVER_METRICS_SAMPLE_FILE_KEY = "location"
+  _HUE_SERVER_HTTP_HOST_KEY = "http_host"
+  _HUE_SERVER_HTTP_PORT_KEY = "http_port"
+  _HUE_SERVER_SSL_ENABLED_KEY = "ssl_enable"
+  _SERVICE_RELEASE = 'service_release'
+
+  def __init__(self, safety_valve):
+    Adapter.__init__(self, "HUE", "HUE_SERVER", safety_valve)
+    self._metrics = None
+    self._is_alive_collector = _HueIsAliveCollector(self)
+
+  def read_service_defined_metrics(self, path):
+    if path is None:
+      raise Exception("A path is required!")
+    self._metrics = ServiceDefinedMetrics(path, SplittingSourceProcessor('::'))
+
+  def get_metrics_file(self, conf):
+    if conf is None:
+      raise Exception("A configuration is required!")
+
+    try:
+      return conf.get(
+        self.section,
+        HueServerAdapter._HUE_SERVER_METRICS_SAMPLE_FILE_KEY)
+    except:
+      LOG.exception("Failed to retrieve %s from monitoring configuration file" %
+                    HueServerAdapter._HUE_SERVER_METRICS_SAMPLE_FILE_KEY)
+
+  def _get_is_alive_url(self, conf):
+    if conf is None:
+      raise Exception("A configuration is required!")
+
+    if not self._is_alive_supported(conf):
+      return None
+
+    try:
+      host = conf.get(self.section, HueServerAdapter._HUE_SERVER_HTTP_HOST_KEY)
+      if host is None:
+        LOG.error("%s entry missing from monitoring configuration file" %
+                  (HueServerAdapter._HUE_SERVER_HTTP_HOST_KEY, ))
+        return None
+      port = conf.getint(self.section,
+                         HueServerAdapter._HUE_SERVER_HTTP_PORT_KEY)
+      if port is None:
+        LOG.error("%s entry missing from monitoring configuration file" %
+                  (HueServerAdapter._HUE_SERVER_HTTP_PORT_KEY, ))
+        return None
+      if conf.getboolean_with_default(
+          self.section,
+          HueServerAdapter._HUE_SERVER_SSL_ENABLED_KEY,
+          False):
+        url_format = "https://%s:%s/desktop/debug/is_alive"
+      else:
+        url_format = "http://%s:%s/desktop/debug/is_alive"
+      return url_format % (host, port)
+    except:
+      LOG.exception("Failed to read monitoring configuration file")
+      return None
+
+  def parse_metrics_from_file(self, conf, json):
+    if json is None:
+      raise Exception("a json sample is required")
+    if self._metrics is None:
+      raise Exception("No metrics have been loaded!")
+
+    role_extractor = SimpleMetricsExtractor(
+      self._metrics.get_sources(self._role_type))
+    visit_json(json, [role_extractor])
+    return role_extractor.metrics
+
+  def add_sample_file_metrics(self, version, update, metrics, accessors):
+    for metric_id, value in metrics.iteritems():
+      update.add_metric(metric_id, value)
+
+  def get_metrics_sample(self, version):
+    return _HUE_SERVER_METRICS_SAMPLE
+
+  def _is_alive_supported(self, conf):
+    return True
+
+  def get_adapter_specific_collectors(self):
+    return [self._is_alive_collector]
+
+_HUE_SERVER_METRICS_SAMPLE="""
+{
+  "desktop.auth.oauth.authentication-time": {
+    "1m_rate": 0,
+    "999_percentile": 0,
+    "15m_rate": 0,
+    "99_percentile": 1234567,
+    "mean_rate": 0,
+    "max": 0,
+    "sum": 0,
+    "min": 0,
+    "5m_rate": 0,
+    "count": 0,
+    "75_percentile": 0,
+    "std_dev": 0,
+    "95_percentile": 0,
+    "avg": 0
+  },
+  "desktop.auth.saml2.authentication-time": {
+    "1m_rate": 0,
+    "999_percentile": 0,
+    "15m_rate": 0,
+    "99_percentile": 0,
+    "mean_rate": 0,
+    "max": 0,
+    "sum": 0,
+    "min": 0,
+    "5m_rate": 0,
+    "count": 0,
+    "75_percentile": 0,
+    "std_dev": 0,
+    "95_percentile": 0,
+    "avg": 0
+  },
+  "python.threads.count": {
+    "value": 42.5
+  },
+  "desktop.users.logged-in.count": {
+    "count": 1
+  },
+  "python.gc.referrers.count": {
+    "value": 0
+  },
+  "desktop.requests.exceptions.count": {
+    "count": 7
+  },
+  "python.multiprocessing.active": {
+    "value": 0
+  },
+  "desktop.users.count": {
+    "value": 2
+  },
+  "python.gc.referents.count": {
+    "value": 0
+  },
+  "python.threads.active": {
+    "value": 52
+  },
+  "python.threads.daemon": {
+    "value": 1
+  },
+  "desktop.auth.pam.authentication-time": {
+    "1m_rate": 0,
+    "999_percentile": 0,
+    "15m_rate": 0,
+    "99_percentile": 0,
+    "mean_rate": 0,
+    "max": 0,
+    "sum": 0,
+    "min": 0,
+    "5m_rate": 0,
+    "count": 0,
+    "75_percentile": 0,
+    "std_dev": 0,
+    "95_percentile": 0,
+    "avg": 0
+  },
+  "desktop.auth.spnego.authentication-time": {
+    "1m_rate": 0,
+    "999_percentile": 0,
+    "15m_rate": 0,
+    "99_percentile": 0,
+    "mean_rate": 0,
+    "max": 0,
+    "sum": 0,
+    "min": 0,
+    "5m_rate": 0,
+    "count": 0,
+    "75_percentile": 0,
+    "std_dev": 0,
+    "95_percentile": 0,
+    "avg": 0
+  },
+  "desktop.auth.ldap.authentication-time": {
+    "1m_rate": 0,
+    "999_percentile": 0,
+    "15m_rate": 0,
+    "99_percentile": 0,
+    "mean_rate": 0,
+    "max": 0,
+    "sum": 0,
+    "min": 0,
+    "5m_rate": 0,
+    "count": 0,
+    "75_percentile": 0,
+    "std_dev": 0,
+    "95_percentile": 0,
+    "avg": 0
+  },
+  "desktop.requests.aggregate-response-time": {
+    "1m_rate": 2.5683487079247715e-101,
+    "999_percentile": 11.331326007843018,
+    "15m_rate": 4.33344429167876e-8,
+    "99_percentile": 11.331326007843018,
+    "mean_rate": 0.0005057717284166405,
+    "max": 11.331326007843018,
+    "sum": 15.07522988319397,
+    "min": 0.007016897201538086,
+    "5m_rate": 2.0635466335636803e-21,
+    "count": 7,
+    "75_percentile": 2.34737491607666,
+    "std_dev": 4.1352759732033935,
+    "95_percentile": 11.331326007843018,
+    "avg": 2.15360426902771
+  },
+  "python.gc.objects.count": {
+    "value": 218408
+  },
+  "python.multiprocessing.count": {
+    "value": 0
+  },
+  "desktop.auth.openid.authentication-time": {
+    "1m_rate": 0,
+    "999_percentile": 0,
+    "15m_rate": 0,
+    "99_percentile": 0,
+    "mean_rate": 0,
+    "max": 0,
+    "sum": 0,
+    "min": 0,
+    "5m_rate": 0,
+    "count": 0,
+    "75_percentile": 0,
+    "std_dev": 0,
+    "95_percentile": 0,
+    "avg": 0
+  },
+  "python.gc.collection.count2": {
+    "value": 10
+  },
+  "python.gc.collection.count0": {
+    "value": 10
+  },
+  "python.gc.collection.count1": {
+    "value": 10
+  },
+  "python.multiprocessing.daemon": {
+    "value": 0
+  },
+  "desktop.requests.active.count": {
+    "count": 0
+  }
+}
+"""

+ 365 - 0
tools/ops/script_runner/lib/hue_converters.py

@@ -0,0 +1,365 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import time
+import re
+
+from django.db import transaction
+
+from desktop.lib.exceptions_renderable import PopupException
+from django.core.exceptions import FieldError
+from desktop.models import Document, DocumentPermission, DocumentTag, Document2, Directory, Document2Permission, FilesystemException
+from notebook.models import import_saved_beeswax_query
+from doc2_utils import findMatchingQuery, removeInvalidChars
+
+LOG = logging.getLogger(__name__)
+
+class DocumentConverterHueScripts(object):
+  """
+  Given a user, converts any existing Document objects to Document2 objects
+  """
+
+  def __init__(self, user, allowdupes=False, startqueryname=None, startuser=None, processdocs=None):
+    self.user = user
+    self.allowdupes = allowdupes
+    self.startqueryname = startqueryname
+    self.startuser = startuser
+    if (self.startqueryname or self.startuser) and not processdocs:
+      self.processdocs = False
+    else:
+      self.processdocs = True
+    # If user does not have a home directory, we need to create one and import any orphan documents to it
+    try:
+      self.home_dir = Document2.objects.create_user_directories(self.user)
+    except FilesystemException, e:
+      LOG.warn("User: %s failed: Exception: %s" % (self.user, e))
+      raise
+    self.imported_tag = DocumentTag.objects.get_imported2_tag(user=self.user)
+    self.imported_docs = []
+
+
+  def convertfailed(self):
+    # Convert SavedQuery documents
+    try:
+      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
+  
+      docs = self._get_unconverted_docs(SavedQuery).filter(extra__in=[HQL, IMPALA, RDBMS])
+      for doc in docs:
+        if doc.content_object:
+          id_temp = doc.to_dict()
+          id = id_temp['id']
+          notebook = import_saved_beeswax_query(doc.content_object)
+          data = notebook.get_data()
+          name = data['name']
+          query = data['snippets'][0]['statement_raw']
+          if re.match(self.startqueryname, name) and not self.startuser:
+            self.processdocs = True
+          if self.processdocs:
+            matchdocs = findMatchingQuery(user=self.user, id=id, name=name, query=query, include_history=False)
+            if not matchdocs or self.allowdupes:
+              try:
+                if doc.is_historic():
+                  data['isSaved'] = False
+
+                doc2 = self._create_doc2(
+                    document=doc,
+                    doctype=data['type'],
+                    name=data['name'],
+                    description=data['description'],
+                    data=notebook.get_json()
+                )
+
+                if doc.is_historic():
+                  doc2.is_history = False
+
+                self.imported_docs.append(doc2)
+              
+              except:
+                pass
+
+    except ImportError:
+      LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
+      pass
+
+    # Convert SQL Query history documents
+    try:
+      from beeswax.models import SavedQuery, HQL, IMPALA, RDBMS
+
+      docs = self._get_unconverted_docs(SavedQuery, with_history=True).filter(extra__in=[HQL, IMPALA, RDBMS]).order_by('-last_modified')
+
+      for doc in docs:
+  	if not doc.content_object:
+          LOG.error("Content object is missing")
+        elif doc.content_object:
+          id_temp = doc.to_dict()
+          id = id_temp['id']
+          notebook = import_saved_beeswax_query(doc.content_object)
+          data = notebook.get_data()
+          name = data['name']
+          query = data['snippets'][0]['statement_raw']
+          if re.match(self.startqueryname, name) and not self.startuser:
+            self.processdocs = True
+          if self.processdocs:
+            try:
+              data['isSaved'] = False
+              data['snippets'][0]['lastExecuted'] = time.mktime(doc.last_modified.timetuple()) * 1000
+
+              doc2 = self._historify(data, self.user)
+              doc2.last_modified = doc.last_modified
+
+              # save() updates the last_modified to current time. Resetting it using update()
+              doc2.save()
+              Document2.objects.filter(id=doc2.id).update(last_modified=doc.last_modified)
+ 
+              self.imported_docs.append(doc2)
+  
+                # Tag for not re-importing
+              Document.objects.link(
+                doc2,
+                owner=doc2.owner,
+                name=doc2.name,
+                description=doc2.description,
+                extra=doc.extra
+              )
+  
+              try:
+                doc.add_tag(self.imported_tag)
+              except IntegrityError, e:
+                LOG.exception("Failed to add imported_tag to doc %s with error %s" % (doc2.name, e))
+                pass
+
+              doc.save()
+
+            except:
+              LOG.exception("Doc name: %s" % (doc.name))
+              pass  
+         
+    except ImportError, e:
+      LOG.info('Cannot convert Saved Query documents: beeswax app is not installed')
+      pass
+
+    # Convert Job Designer documents
+    try:
+      from oozie.models import Workflow
+
+      # TODO: Change this logic to actually embed the workflow data in Doc2 instead of linking to old job design
+      docs = self._get_unconverted_docs(Workflow)
+      for doc in docs:
+        try:
+          if doc.content_object:
+            data = doc.content_object.data_dict
+            data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
+            doc2 = self._create_doc2(
+                document=doc,
+                doctype='link-workflow',
+                description=doc.description,
+                data=json.dumps(data)
+            )
+            self.imported_docs.append(doc2)
+        except Exception, e:
+          self.failed_docs.append(doc)
+          LOG.exception('Failed to import Job Designer document id: %d' % doc.id)
+    except ImportError, e:
+      LOG.warn('Cannot convert Job Designer documents: oozie app is not installed')
+
+
+    # Convert PigScript documents
+    try:
+      from pig.models import PigScript
+
+      # TODO: Change this logic to actually embed the pig data in Doc2 instead of linking to old pig script
+      docs = self._get_unconverted_docs(PigScript)
+      for doc in docs:
+        try:
+          if doc.content_object:
+            data = doc.content_object.dict
+            data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
+            doc2 = self._create_doc2(
+                document=doc,
+                doctype='link-pigscript',
+                description=doc.description,
+                data=json.dumps(data)
+            )
+            self.imported_docs.append(doc2)
+        except Exception, e:
+          self.failed_docs.append(doc)
+          LOG.exception('Failed to import Pig document id: %d' % doc.id)
+    except ImportError, e:
+      LOG.warn('Cannot convert Pig documents: pig app is not installed')
+
+    # Add converted docs to root directory
+    if self.imported_docs:
+      LOG.info('Successfully imported %d documents' % len(self.imported_docs))
+
+    # Set is_trashed field for old documents with is_trashed=None
+    try:
+      docs = Document2.objects.filter(owner=self.user, is_trashed=None)
+      for doc in docs:
+        try:
+          if doc.path and doc.path != '/.Trash':
+            doc_last_modified = doc.last_modified
+            doc.is_trashed = doc.path.startswith('/.Trash')
+            doc.save()
+
+            # save() updates the last_modified to current time. Resetting it using update()
+            Document2.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
+        except Exception, e:
+          LOG.exception("Failed to set is_trashed field with exception: %s" % e)
+    except FieldError, e:
+      LOG.info("Skipping is_trashed as does not exist in this version") 
+
+    return self.processdocs
+
+
+  def _get_unconverted_docs(self, content_type, with_history=False):
+    docs = Document.objects.get_docs(self.user, content_type).filter(owner=self.user)
+
+    tags = [
+      DocumentTag.objects.get_trash_tag(user=self.user), # No trashed docs
+      DocumentTag.objects.get_example_tag(user=self.user), # No examples
+#      self.imported_tag # No already imported docs
+    ]
+
+    if not with_history:
+      tags.append(DocumentTag.objects.get_history_tag(user=self.user)) # No history yet
+
+    return docs.exclude(tags__in=tags)
+
+
+  def _get_parent_directory(self, document):
+    """
+    Returns the parent directory object that should be used for a given document. If the document is tagged with a
+        project name (non-RESERVED DocumentTag), a Directory object with the first project tag found is returned.
+        Otherwise, the owner's home directory is returned.
+    """
+    parent_dir = self.home_dir
+    project_tags = document.tags.exclude(tag__in=DocumentTag.RESERVED)
+    if project_tags.exists():
+      first_tag = project_tags[0]
+      parent_dir, created = Directory.objects.get_or_create(
+          owner=self.user,
+          name=first_tag.tag,
+          parent_directory=self.home_dir
+      )
+    return parent_dir
+
+
+  def _sync_permissions(self, document, document2):
+    """
+    Syncs (creates) Document2Permissions based on the DocumentPermissions found for a given document.
+    """
+    doc_permissions = DocumentPermission.objects.filter(doc=document)
+    for perm in doc_permissions:
+      try:
+        doc2_permission, created = Document2Permission.objects.get_or_create(doc=document2, perms=perm.perms)
+        if perm.users:
+          doc2_permission.users.add(*perm.users.all())
+        if perm.groups:
+          doc2_permission.groups.add(*perm.groups.all())
+      except:
+        pass
+
+
+  def _create_doc2(self, document, doctype, name=None, description=None, data=None):
+    try:
+      with transaction.atomic():
+        name = name if name else document.name
+        name = removeInvalidChars(name)
+
+        document2 = Document2.objects.create(
+          owner=self.user,
+          parent_directory=self._get_parent_directory(document),
+          name=name,
+          type=doctype,
+          description=description,
+          data=data
+        )
+        self._sync_permissions(document, document2)
+
+        # Create a doc1 copy and link it for backwards compatibility
+        Document.objects.link(
+          document2,
+          owner=document2.owner,
+          name=document2.name,
+          description=document2.description,
+          extra=document.extra
+        )
+
+        # save() updates the last_modified to current time. Resetting it using update()
+        Document2.objects.filter(id=document2.id).update(last_modified=document.last_modified)
+
+        document.add_tag(self.imported_tag)
+        document.save()
+        return document2
+    except Exception, e:
+      raise PopupException(_("Failed to convert Document object: %s") % e)
+
+
+  def _historify(self, notebook, user):
+    query_type = notebook['type']
+    name = notebook['name'] if (notebook['name'] and notebook['name'].strip() != '') else DEFAULT_HISTORY_NAME
+    name = removeInvalidChars(name)
+
+    try:
+      history_doc = Document2.objects.create(
+        name=name,
+        type=query_type,
+        owner=user,
+        is_history=True,
+        is_managed=notebook.get('isManaged') == True
+      )
+    except TypeError:
+      history_doc = Document2.objects.create(
+        name=name,
+        type=query_type,
+        owner=user,
+        is_history=True,
+      )
+
+    # Link history of saved query
+    if notebook['isSaved']:
+      parent_doc = Document2.objects.get(uuid=notebook.get('parentSavedQueryUuid') or notebook['uuid']) # From previous history query or initial saved query
+      notebook['parentSavedQueryUuid'] = parent_doc.uuid
+      history_doc.dependencies.add(parent_doc)
+
+    Document.objects.link(
+      history_doc,
+      name=history_doc.name,
+      owner=history_doc.owner,
+      description=history_doc.description,
+      extra=query_type
+    )
+
+    notebook['uuid'] = history_doc.uuid
+    history_doc.update_data(notebook)
+    history_doc.search = self._get_statement(notebook)
+    history_doc.save()
+
+    return history_doc
+
+
+  def _get_statement(self, notebook):
+    statement = ''
+    if notebook['snippets'] and len(notebook['snippets']) > 0:
+      try:
+        statement = notebook['snippets'][0]['result']['handle']['statement']
+        if type(statement) == dict:  # Old format
+          statement = notebook['snippets'][0]['statement_raw']
+      except KeyError:  # Old format
+        statement = notebook['snippets'][0]['statement_raw']
+    return statement

+ 75 - 0
tools/ops/script_runner/lib/hue_curl.py

@@ -0,0 +1,75 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import logging
+import datetime
+import time
+import subprocess
+
+from cm_environment import check_security
+from hue_shared import which
+
+#logging.basicConfig()
+#logging = logging.getLogger(__name__)
+
+class Curl(object):
+
+  def __init__(self, verbose=False):
+    self.curl = which('curl')
+    if self.curl is None:
+      logging.exception("curl is required, please install and rerun")
+      sys.exit(1)
+
+    # We will change to handle certs later
+    self.basecmd = self.curl + ' -k'
+    logging.info("Checking security status")
+    self.security_enabled = check_security()
+    self.verbose = verbose
+
+    if self.security_enabled:
+      self.basecmd = self.basecmd + ' --negotiate -u :'
+
+    if self.verbose:
+      self.basecmd = self.basecmd + ' -v'
+    else:
+      self.basecmd = self.basecmd + ' -s'
+
+  def do_curl(self, url, method='GET', follow=False, args=None):
+
+    cmd = self.basecmd + ' -X ' + method
+    if follow:
+      cmd = cmd + ' -L'
+
+    if args is not None:
+      cmd = cmd + ' ' + args
+
+    cmd = cmd + ' \'' + url + '\''
+    logging.info("OSRUN: %s" % cmd)
+    curl_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    curl_response = curl_process.communicate()[0]
+    curl_ret = curl_process.returncode
+    if curl_ret > 0:
+      logging.exception("Curl failed to run succesfully: %s" % curl_response)
+    return curl_response
+
+
+  def do_curl_available_services(self, service_test):
+    url = service_test['url']
+    method = service_test['method']
+    response = self.do_curl(url, method=method)
+    return response

+ 24 - 0
tools/ops/script_runner/lib/hue_shared.py

@@ -0,0 +1,24 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+def which(file_name):
+  for path in os.environ["PATH"].split(os.pathsep):
+    full_path = os.path.join(path, file_name)
+    if os.path.exists(full_path) and os.access(full_path, os.X_OK):
+      return full_path
+  return None

+ 108 - 0
tools/ops/script_runner/lib/log.conf

@@ -0,0 +1,108 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+########################################
+# Definition for the different objects
+# - FOR DEVELOPMENT ONLY -
+#
+# Directories where log files are kept must already exist.
+# That's why we pick /tmp.
+#
+# The loggers are configured to write to the log files ONLY.
+# Developers may set the DESKTOP_DEBUG environment variable to
+# enable stderr logging output.
+########################################
+
+[logger_root]
+handlers=logfile,errorlog
+
+[logger_access]
+handlers=accesslog
+qualname=access
+
+[logger_django_auth_ldap]
+handlers=accesslog
+qualname=django_auth_ldap
+
+[logger_kazoo_client]
+level=INFO
+handlers=errorlog
+qualname=kazoo.client
+
+[logger_djangosaml2]
+level=INFO
+handlers=errorlog
+qualname=djangosaml2
+
+[logger_requests_packages_urllib3_connectionpool]
+level=DEBUG
+handlers=errorlog
+qualname=requests.packages.urllib3.connectionpool
+
+[logger_django_db]
+level=DEBUG
+handlers=errorlog
+qualname=django.db.backends
+
+[handler_stderr]
+class=StreamHandler
+formatter=default
+level=DEBUG
+args=(sys.stderr,)
+
+[handler_accesslog]
+class=handlers.RotatingFileHandler
+level=INFO
+propagate=True
+formatter=access
+args=('%LOG_DIR%/%PROC_NAME%.log', 'a', 1000000, 3)
+
+[handler_errorlog]
+class=handlers.RotatingFileHandler
+level=ERROR
+formatter=default
+args=('%LOG_DIR%/%PROC_NAME%.log', 'a', 1000000, 3)
+
+[handler_logfile]
+class=handlers.RotatingFileHandler
+# Choices are DEBUG, INFO, WARNING, ERROR, CRITICAL
+level=DEBUG
+formatter=default
+args=('%LOG_DIR%/%PROC_NAME%.log', 'a', 1000000, 3)
+
+[formatter_default]
+class=desktop.log.formatter.Formatter
+format=[%(asctime)s] %(module)-12s %(levelname)-8s %(message)s
+datefmt=%d/%b/%Y %H:%M:%S %z
+
+[formatter_access]
+class=desktop.log.formatter.Formatter
+format=[%(asctime)s] %(levelname)-8s %(message)s
+datefmt=%d/%b/%Y %H:%M:%S %z
+
+
+########################################
+# A summary of loggers, handlers and formatters
+########################################
+
+[loggers]
+keys=root,access,django_auth_ldap,kazoo_client,requests_packages_urllib3_connectionpool,djangosaml2,django_db
+
+[handlers]
+keys=stderr,logfile,accesslog,errorlog
+
+[formatters]
+keys=default,access

+ 226 - 0
tools/ops/script_runner/lib/log/__init__.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import logging.config
+import os
+import os.path
+import re
+import sys
+
+from cStringIO import StringIO
+from logging import FileHandler
+from logging.handlers import RotatingFileHandler
+
+from desktop.lib.paths import get_desktop_root
+from desktop.log import formatter
+from desktop.log.formatter import MessageOnlyFormatter
+
+
+DEFAULT_LOG_DIR = 'logs'
+LOG_FORMAT = '[%(asctime)s] %(module)-12s %(levelname)-8s %(message)s'
+DATE_FORMAT = '%d/%b/%Y %H:%M:%S %z'
+FORCE_DEBUG = False
+CONF_RE = re.compile('%LOG_DIR%|%PROC_NAME%')
+
+_log_dir = None
+lib_dir = os.path.dirname(os.path.realpath(__file__)) + "/../"
+
+
+def _read_log_conf(proc_name, log_dir):
+  """
+  _read_log_conf(proc_name, log_dir) -> StringIO or None
+
+  This method also replaces the %LOG_DIR% and %PROC_NAME% occurrences.
+  """
+  def _repl(match):
+    if match.group(0) == '%LOG_DIR%':
+      return log_dir
+    elif match.group(0) == '%PROC_NAME%':
+      return proc_name
+
+  log_conf = lib_dir + 'log.conf'
+
+  if not os.path.isfile(log_conf):
+    return None
+
+  try:
+    raw = file(log_conf).read()
+    sio = StringIO(CONF_RE.sub(_repl, raw))
+    return sio
+  except IOError, ex:
+    print >> sys.stderr, "ERROR: Failed to open %s: %s" % (log_conf, ex)
+    return None
+
+
+def _find_console_stream_handler(logger):
+  """Find a StreamHandler that is attached to the logger that prints to the console."""
+  for handler in logger.handlers:
+    if isinstance(handler, logging.StreamHandler) and handler.stream in (sys.stderr, sys.stdout):
+      return logger
+  return None
+
+
+class AuditHandler(RotatingFileHandler):
+  pass
+
+
+def get_audit_logger():
+  from desktop.conf import AUDIT_EVENT_LOG_DIR, AUDIT_LOG_MAX_FILE_SIZE
+
+  audit_logger = logging.getLogger('audit')
+  if not filter(lambda hclass: isinstance(hclass, AuditHandler), audit_logger.handlers): # Don't add handler twice
+    size, unit = int(AUDIT_LOG_MAX_FILE_SIZE.get()[:-2]), AUDIT_LOG_MAX_FILE_SIZE.get()[-2:]
+    maxBytes = size * 1024 ** (1 if unit == 'KB' else 2 if unit == 'MB' else 3)
+
+    audit_handler = AuditHandler(AUDIT_EVENT_LOG_DIR.get(), maxBytes=maxBytes, backupCount=50)
+    audit_handler.setFormatter(MessageOnlyFormatter())
+    audit_logger.addHandler(audit_handler)
+
+  return audit_logger
+
+
+def chown_log_dir(uid, gid):
+  """
+  chown all files in the log dir to this user and group.
+  Should only be called after loggic has been setup.
+  Return success
+  """
+  if _log_dir is None:
+    return False
+
+  try:
+    os.chown(_log_dir, uid, gid)
+    for entry in os.listdir(_log_dir):
+      os.chown(os.path.join(_log_dir, entry), uid, gid)
+    return True
+  except OSError, ex:
+    print >> sys.stderr, 'Failed to chown log directory %s: ex' % (_log_dir, ex)
+    return False
+
+
+def basic_logging(proc_name, log_dir=None):
+  """
+  Configure logging for the program ``proc_name``:
+    - Apply log.conf in the config directory.
+    - If DESKTOP_LOGLEVEL environment variable is specified, the root console
+      handler (stdout/stderr) is set to that level. If there is no console handler,
+      a new one is created.
+    - Defining the environment variable DESKTOP_DEBUG is the same as setting
+      DESKTOP_LOGLEVEL=DEBUG.
+
+  The ``log_dir`` will replace the %LOG_DIR% in log.conf. If not specified, we look
+  for the DESTKOP_LOG_DIR environment variable, and then default to the DEFAULT_LOG_DIR.
+
+  This removes all previously installed logging handlers.
+  """
+  global FORCE_DEBUG
+
+  # Setup log_dir
+  if not log_dir:
+    log_dir = os.getenv("DESKTOP_LOG_DIR", DEFAULT_LOG_DIR)
+  if not os.path.exists(log_dir):
+    try:
+      os.makedirs(log_dir)
+    except OSError, err:
+      print >> sys.stderr, 'Failed to create log directory "%s": %s' % (log_dir, err)
+      raise err
+
+  # Remember where our log directory is
+  global _log_dir
+  _log_dir = log_dir
+
+  log_conf = _read_log_conf(proc_name, log_dir)
+
+  if log_conf is not None:
+    logging.config.fileConfig(log_conf)
+    root_logger = logging.getLogger()
+  else:
+    # Get rid of any preinstalled/default handlers
+    root_logger = logging.getLogger()
+    for h in root_logger.handlers:
+      root_logger.removeHandler(h)
+
+  # always keep DEBUG at the root, since we'll filter in the
+  # handlers themselves - this allows the /logs endpoint
+  # to always have all logs.
+  root_logger.setLevel(logging.DEBUG)
+
+  # Handle env variables
+  env_loglevel = os.getenv("DESKTOP_LOGLEVEL")
+  env_debug = os.getenv('DESKTOP_DEBUG') or FORCE_DEBUG
+  if env_debug:
+    env_loglevel = 'DEBUG'
+
+  if env_loglevel:
+    try:
+      lvl = getattr(logging, env_loglevel.upper())
+    except AttributeError:
+      raise Exception("Invalid log level in DESKTOP_LOGLEVEL: %s" % (env_loglevel,))
+
+    # Set the StreamHandler to the level (create one if necessary)
+    handler = _find_console_stream_handler(root_logger)
+    if not handler:
+      handler = logging.StreamHandler()
+      handler.setFormatter(logging.Formatter(LOG_FORMAT, DATE_FORMAT))
+      root_logger.addHandler(handler)
+    handler.setLevel(lvl)
+
+    # Set all loggers but error.log to the same logging level
+    error_handler = logging.getLogger('handler_logfile')
+    for h in root_logger.handlers:
+      if isinstance(h, (FileHandler, RotatingFileHandler)) and h != error_handler:
+        h.setLevel(lvl)
+
+
+def fancy_logging():
+  """Configure logging into a buffer for /logs endpoint."""
+  from log_buffer import FixedBufferHandler
+
+  BUFFER_SIZE = 500 * 200 # This is the size in characters, not bytes. Targets about 500 rows.
+  buffer_handler = FixedBufferHandler(BUFFER_SIZE)
+  _formatter = formatter.Formatter(LOG_FORMAT, DATE_FORMAT)
+
+  # We always want to catch all messages in our error report buffer
+  buffer_handler.setLevel(logging.DEBUG)
+  buffer_handler.setFormatter(_formatter)
+  root_logger = logging.getLogger()
+  root_logger.addHandler(buffer_handler)
+
+
+def get_all_debug():
+  global FORCE_DEBUG
+
+  return FORCE_DEBUG
+
+
+def set_all_debug():
+  from desktop.settings import ENV_HUE_PROCESS_NAME # Circular dependency
+  global FORCE_DEBUG
+
+  FORCE_DEBUG = True
+  basic_logging(os.environ[ENV_HUE_PROCESS_NAME])
+  fancy_logging()
+
+
+def reset_all_debug():
+  from desktop.settings import ENV_HUE_PROCESS_NAME # Circular dependency
+  global FORCE_DEBUG
+
+  FORCE_DEBUG = False
+  basic_logging(os.environ[ENV_HUE_PROCESS_NAME])
+  fancy_logging()

+ 168 - 0
tools/ops/script_runner/lib/log/access.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Decorators and methods related to access log.
+This assumes a single-threaded server.
+"""
+
+import logging
+import re
+import threading
+import time
+
+import desktop.conf
+
+ACCESS_LOG = logging.getLogger('access')
+
+def access_log_level(lvl):
+  """Decorator to set the access log level of a view function."""
+  if lvl not in (logging.DEBUG, logging.WARN, logging.ERROR, logging.CRITICAL, logging.FATAL):
+    raise ValueError('%s is not a valid logging level' % (lvl,))
+
+  def deco_view(func):
+    func.access_log_level = lvl
+    return func
+  return deco_view
+
+
+#
+# Keep most recent per user per app per view access info
+#
+# This is a dictionary (indexed by user)
+# of dictionary (indexed by app)
+# of dictionary (indexed by path)
+# of list (of AccessInfo) sorted by time most recent first
+#
+recent_access_map = { }
+_recent_access_map_lk = threading.Lock()
+_per_user_lk = { }      # Indexed by username
+
+# Store a map of usernames and a dictionary of
+# their IP addresses and last access times
+last_access_map = { }
+
+# Max number of records per user per view to keep
+_USER_ACCESS_HISTORY_SIZE = desktop.conf.USER_ACCESS_HISTORY_SIZE.get()
+
+class AccessInfo(dict):
+  """
+  Represents details on a user access.
+
+  In addition to the attributes specified in __init__, it may contain
+  ``msg`` -- A message associated with the access
+  ``app`` -- The top level package name of the view function, which
+             need NOT be a valid Desktop application name
+  """
+  def __init__(self, request):
+    self['username'] = request.user.username or '-anon-'
+    if request.META.has_key('HTTP_X_FORWARDED_FOR'):
+      self['remote_ip'] = request.META.get('HTTP_X_FORWARDED_FOR', '-')
+    else:
+      self['remote_ip'] = request.META.get('REMOTE_ADDR', '-')
+    self['method'] = request.method
+    self['path'] = request.path
+    self['proto'] = request.META.get('SERVER_PROTOCOL', '-')
+    self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
+    self['time'] = time.time()
+
+  def log(self, level, msg=None):
+    if msg is not None:
+      self['msg'] = msg
+      ACCESS_LOG.log(level,
+                     '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s" -- %(msg)s' %
+                     self)
+    else:
+      ACCESS_LOG.log(level,
+                     '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"' % self)
+
+  def add_to_access_history(self, app):
+    """Record this user access to the recent access map"""
+    self['app'] = app
+    user = self['username']
+    path = self['path']
+    try:
+      app_dict = recent_access_map[user]
+    except KeyError:
+      # Hold the global lock when modifying recent_access_map
+      _recent_access_map_lk.acquire()
+      try:
+        app_dict = { }
+        _per_user_lk[user] = threading.Lock()
+        recent_access_map[user] = app_dict
+      finally:
+        _recent_access_map_lk.release()
+
+    # Hold the per user lock when modifying adding the access record.
+    # We could further break down the locking granularity but that seems silly.
+    user_lk = _per_user_lk[user]
+    user_lk.acquire()
+    try:
+      try:
+        path_dict = app_dict[app]
+      except KeyError:
+        path_dict = { }
+        app_dict[app] = path_dict
+
+      try:
+        view_access_list = path_dict[path]
+      except KeyError:
+        view_access_list = [ ]
+        path_dict[path] = view_access_list
+
+      # Most recent first
+      view_access_list.insert(0, self)
+      if len(view_access_list) > _USER_ACCESS_HISTORY_SIZE:
+        view_access_list.pop()
+
+      # Update the IP address and last access time of the user
+      last_access_map[user] = {'ip':self['remote_ip'],
+                               'time':self['time']}
+    finally:
+      user_lk.release()
+
+
+_MODULE_RE = re.compile('[^.]*')
+
+def log_page_hit(request, view_func, level=None):
+  """Log the request to the access log"""
+  if level is None:
+    level = logging.INFO
+  ai = AccessInfo(request)
+  ai.log(level)
+  # Find the app
+  app_re_match = _MODULE_RE.match(view_func.__module__)
+  app = app_re_match and app_re_match.group(0) or '-'
+  ai.add_to_access_history(app)
+
+
+def access_log(request, msg=None, level=None):
+  """
+  access_log(request, msg=None, level=None) -> None
+
+  Write to the access log. This could be a page hit, or general auditing information.
+  """
+  if level is None:
+    level = logging.INFO
+  ai = AccessInfo(request)
+  ai.log(level, msg)
+
+
+def access_warn(request, msg=None):
+  """Write to access log with a WARN log level"""
+  ai = AccessInfo(request)
+  ai.log(logging.WARN, msg)

+ 50 - 0
tools/ops/script_runner/lib/log/formatter.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from pytz import UnknownTimeZoneError, datetime, timezone
+
+class Formatter(logging.Formatter):
+  def formatTime(self, record, datefmt=None):
+    try:
+      tz = timezone(os.environ['TZ'])
+    except (KeyError, UnknownTimeZoneError):
+      tz = None
+
+    try:
+      ct = datetime.datetime.fromtimestamp(record.created, tz=tz)
+    except (OverflowError, TypeError, ValueError):
+      # Fallback to original.
+      return super(Formatter, self).formatTime(record, datefmt=datefmt)
+
+    if datefmt:
+      s = ct.strftime(datefmt)
+    else:
+      t = ct.strftime("%Y-%m-%d %H:%M:%S")
+      s = "%s,%03d" % (t, record.msecs)
+    return s
+
+
+class MessageOnlyFormatter(logging.Formatter):
+  def formatTime(self, record, datefmt=None):
+    return ''
+
+  def format(self, record):
+    return record.getMessage()
+

+ 78 - 0
tools/ops/script_runner/lib/log/log_buffer.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+We would like to keep the last X characters
+of log message around for us to view in case of emergency.
+
+This log handler lets us do that.
+"""
+
+import logging, collections
+
+class FixedBuffer(object):
+  """
+  The what: a buffer that maintains a fixed-size sliding window on
+  the log history. As messages come in, old messages get pushed out.
+
+  The plan: use a deque to keep a list of messages by reference (so
+  minimal copying required). If the total size in characters exceeds
+  some maximum, pop off messages until we get below the max, and then
+  pad back up with the last maxsize-size characters of the most recently
+  removed message to bring us back up to the maximum.
+
+  Net cost is eventually one string copy per insert and a linear amount of
+  reference manipulation. Benefit is the ability to save a slice through
+  the really big messages (although huge messages are rare) rather than
+  lose them completely when they get popped.
+  """
+  def __init__(self, maxsize=50000):
+    """
+    maxsize is in characters, not bytes.
+    """
+    self.buffer = collections.deque()
+    self.maxsize = maxsize
+    self.size = 0
+
+  def insert(self, message):
+    self.size += len(message)
+    self.buffer.append(message)
+    if self.size > self.maxsize:
+      while self.size > self.maxsize:
+        last = self.buffer.popleft()
+        self.size -= len(last)
+      # Prepend only as many characters of the outgoing string
+      # as we can fit in the buffer
+      self.buffer.appendleft(last[-(self.maxsize-self.size):])
+      self.size = self.maxsize
+
+  def __str__(self):
+    return '\n'.join([m for m in self.buffer])
+
+  def __iter__(self):
+    return iter(self.buffer)
+
+class FixedBufferHandler(logging.Handler):
+  """
+  Super simple log handler.
+  """
+  def __init__(self,buffer_size=50000):
+    logging.Handler.__init__(self)
+    self.buf = FixedBuffer(buffer_size)
+
+  def emit(self,record):
+    self.buf.insert(self.format(record))

+ 41 - 0
tools/ops/script_runner/lib/log/log_buffer_test.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# A couple of test cases for the log buffer
+#
+
+import log_buffer
+import logging
+import unittest
+
+class TestLogBuffer(unittest.TestCase):
+  def test_logger(self):
+    logger = logging.getLogger()
+    handler = log_buffer.FixedBufferHandler()
+    logger.addHandler(handler)
+    msg = "My test logging message"
+    logger.warn(msg)
+    self.assertEquals(msg, str(handler.buf))
+
+  def test_overflow(self):
+    buffer = log_buffer.FixedBuffer(maxsize=10)
+    buffer.insert("0123456789")
+    buffer.insert("abcde")
+    self.assertEquals("56789\nabcde", str(buffer))
+
+if __name__ == '__main__':
+  unittest.main()

+ 95 - 0
tools/ops/script_runner/lib/log/tests.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import tempfile
+
+from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal
+
+from desktop.log import get_audit_logger, AuditHandler
+from desktop.conf import AUDIT_EVENT_LOG_DIR, AUDIT_LOG_MAX_FILE_SIZE
+
+
+def test_one_audit():
+  with tempfile.NamedTemporaryFile("w+t") as log_tmp:
+
+    # KB
+    reset = [
+        AUDIT_EVENT_LOG_DIR.set_for_testing(log_tmp.name),
+        AUDIT_LOG_MAX_FILE_SIZE.set_for_testing('25KB')
+    ]
+
+    audit_logger = get_audit_logger()
+    audit_handler = audit_logger.handlers[0]
+
+    assert_equal(25 * 1024 ** 1, audit_handler.maxBytes)
+    assert_equal(len(audit_logger.handlers), 1, audit_logger.handlers)
+    assert_true(isinstance(audit_handler, AuditHandler), audit_logger.handlers)
+
+    audit_logger = get_audit_logger()
+    assert_equal(len(audit_logger.handlers), 1, audit_logger.handlers) # Not adding handler twice
+
+    # Cleanup
+    audit_logger.removeHandler(audit_handler)
+
+    for r in reset:
+      r()
+
+    # MB
+    reset = [
+        AUDIT_EVENT_LOG_DIR.set_for_testing(log_tmp.name),
+        AUDIT_LOG_MAX_FILE_SIZE.set_for_testing('25MB')
+    ]
+
+    audit_logger = get_audit_logger()
+    audit_handler = audit_logger.handlers[0]
+
+    assert_equal(25 * 1024 ** 2, audit_handler.maxBytes)
+    assert_equal(len(audit_logger.handlers), 1, audit_logger.handlers)
+    assert_true(isinstance(audit_handler, AuditHandler), audit_logger.handlers)
+
+    audit_logger = get_audit_logger()
+    assert_equal(len(audit_logger.handlers), 1, audit_logger.handlers) # Not adding handler twice
+
+    # Cleanup
+    audit_logger.removeHandler(audit_handler)
+
+    for r in reset:
+      r()
+
+    # GB
+    reset = [
+        AUDIT_EVENT_LOG_DIR.set_for_testing(log_tmp.name),
+        AUDIT_LOG_MAX_FILE_SIZE.set_for_testing('25GB')
+    ]
+
+    audit_logger = get_audit_logger()
+    audit_handler = audit_logger.handlers[0]
+
+    assert_equal(25 * 1024 ** 3, audit_handler.maxBytes)
+    assert_equal(len(audit_logger.handlers), 1, audit_logger.handlers)
+    assert_true(isinstance(audit_handler, AuditHandler), audit_logger.handlers)
+
+    audit_logger = get_audit_logger()
+    assert_equal(len(audit_logger.handlers), 1, audit_logger.handlers) # Not adding handler twice
+
+    # Cleanup
+    audit_logger.removeHandler(audit_handler)
+
+    for r in reset:
+      r()

+ 234 - 0
tools/ops/script_runner/lib/url_util.py

@@ -0,0 +1,234 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright (c) 2011-2013 Cloudera, Inc. All rights reserved.
+
+import httplib
+import logging
+import socket
+import sys
+import time
+import urllib2
+from urllib2_kerberos import HTTPKerberosAuthHandler
+
+from M2Crypto import httpslib
+from M2Crypto import SSL
+from M2Crypto import m2
+
+logging.basicConfig()
+LOG = logging.getLogger(__name__)
+
+# urlopen_with_timeout.
+#
+# The optional secure_http_service_name parameter allows callers to connect to
+# secure HTTP servers via the urllib2_kerberos library. We have a modified
+# version of the HTTPKerberosAuthHandler code which takes the Kerberos service
+# name rather than construct the name using the HTTP request host. We always add
+# the HTTPKerberosAuthHandler to urllib2 opener handlers because it has no effect
+# if security is not actually enabled.
+#
+# The optional username and pasword parameters similarly handle setting up HTTP
+# digest authentication. Again, this has no effect if HTTP digest authentication
+# is not in use on the connection.
+#
+# The cafile, capath and max_cert_depth control the SSL certificate verification
+# behavior. https://www.openssl.org/docs/ssl/SSL_CTX_load_verify_locations.html
+# explains the semantics of the parameters. Passing none for both means that
+# no verification of the server certification (including the server's hostname)
+# will be performed.
+def urlopen_with_timeout(url,
+                         data=None,
+                         timeout=None,
+                         secure_http_service_name=None,
+                         username=None,
+                         password=None,
+                         cafile=None,
+                         capath=None,
+                         max_cert_depth=9):
+
+  openers = []
+  openers.append(_make_https_handler(cafile,
+                                     capath,
+                                     max_cert_depth))
+
+  openers.append(HTTPKerberosAuthHandler(secure_http_service_name))
+
+  full_url = url
+  if isinstance(url, urllib2.Request):
+    full_url = url.get_full_url()
+  openers.append(_make_http_digest_auth_handler(full_url, username, password))
+
+  LOG.info("url_util: urlopen_with_timeout: full_url: %s" % full_url)
+  if sys.version_info < (2, 6):
+    # The timeout parameter to urlopen was introduced in Python 2.6.
+    # To workaround it in older versions of python, we copy, with
+    # minor modification, httplib.HTTPConnection, and hook it all
+    # up.
+    openers.append(_make_timeout_handler(timeout))
+    opener = urllib2.build_opener(*openers)
+    LOG.info("url_util: urlopen_with_timeout: sys.version_inf < (2, 6): opener: %s" % opener)
+    return opener.open(url, data)
+  else:
+    openers.append(_make_timeout_handler(timeout))
+    opener = urllib2.build_opener(*openers)
+    LOG.info("url_util: urlopen_with_timeout: sys.version_inf > (2, 6): opener: %s" % opener)
+    return opener.open(url, data, timeout)
+
+def head_request_with_timeout(url,
+                              data=None,
+                              timeout=None,
+                              secure_http_service_name=None,
+                              username=None,
+                              password=None,
+                              cafile=None,
+                              capath=None,
+                              max_cert_depth=9):
+
+  class HeadRequest(urllib2.Request):
+    def get_method(self):
+      return "HEAD"
+
+  if isinstance(url, urllib2.Request):
+    raise Exception("Unsupported url type: urllib2.Request.")
+
+  LOG.info("url_util: head_request_with_timeout: url: %s: timeout: %s" % (url, timeout))
+  return urlopen_with_timeout(HeadRequest(url),
+                              data,
+                              timeout,
+                              secure_http_service_name,
+                              username,
+                              password,
+                              cafile,
+                              capath,
+                              max_cert_depth)
+
+def _make_timeout_handler(timeout):
+  # Create these two helper classes fresh each time, since
+  # timeout needs to be in the closure.
+
+  class TimeoutHTTPConnection(httplib.HTTPConnection):
+    def connect(self):
+      """Connect to the host and port specified in __init__."""
+      msg = "getaddrinfo returns an empty list"
+      for res in socket.getaddrinfo(self.host, self.port, 0,
+                      socket.SOCK_STREAM):
+        af, socktype, proto, canonname, sa = res
+        try:
+          self.sock = socket.socket(af, socktype, proto)
+          if timeout is not None:
+            self.sock.settimeout(timeout)
+          if self.debuglevel > 0:
+            LOG.info("connect: (%s, %s)" % (self.host, self.port))
+          self.sock.connect(sa)
+        except socket.error, msg:
+          if self.debuglevel > 0:
+            LOG.info('connect fail:', (self.host, self.port))
+          if self.sock:
+            self.sock.close()
+          self.sock = None
+          continue
+        break
+      if not self.sock:
+        raise socket.error, msg
+
+  class TimeoutHTTPHandler(urllib2.HTTPHandler):
+    http_request = urllib2.AbstractHTTPHandler.do_request_
+    def http_open(self, req):
+      return self.do_open(TimeoutHTTPConnection, req)
+
+  return TimeoutHTTPHandler
+
+def _make_http_digest_auth_handler(url, username, password):
+  password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
+  password_manager.add_password(None, # realm
+                                url,
+                                username,
+                                password)
+  return urllib2.HTTPDigestAuthHandler(password_manager)
+
+def _make_https_handler(cafile=None,
+                        capath=None,
+                        max_cert_depth=9):
+  class HTTPSConnection(httpslib.HTTPSConnection):
+    """
+    A class that extends the default HTTPSConnection to ensure two things:
+    1) Enforce tlsv1 protocol for all ssl connection. Some older pythons
+       (e.g., sles11, probably all versions <= 2.6) attempt SSLv23 handshake
+       that is rejected by newer web servers. See OPSAPS-32192 for an example.
+    2) Force validation if cafile/capath is supplied.
+    """
+
+    def __init__(self, host, port=None, **ssl):
+      # Specifying sslv23 enables the following ssl versions:
+      # SSLv3, SSLv23, TLSv1, TLSv1.1, and TLSv1.2. We will explicitly exclude
+      # SSLv3 and SSLv2 below. This mimics what is done by create_default_context
+      # on newer python versions (python >= 2.7).
+      ctx = SSL.Context('sslv23')
+      # SSL_OP_ALL turns on all workarounds for known bugs. See
+      # https://www.openssl.org/docs/manmaster/ssl/SSL_CTX_set_options.html for
+      # a full list of these workarounds. I believe that we don't really need
+      # any of these workarounds, but, this is default in later pythons and is
+      # future looking.
+      ctx.set_options(m2.SSL_OP_ALL | m2.SSL_OP_NO_SSLv2 | m2.SSL_OP_NO_SSLv3)
+
+      if cafile is not None or capath is not None:
+        ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert,
+                       max_cert_depth)
+        ctx.load_verify_info(cafile=cafile, capath=capath)
+        self._postConnectionCheck = True
+      else:
+        ctx.set_verify(SSL.verify_none, max_cert_depth)
+        self._postConnectionCheck = False
+      httpslib.HTTPSConnection.__init__(self, host, port, ssl_context=ctx)
+
+    def connect(self):
+      # This is a bit ugly but we need to override the connect method in order
+      # to disable hostname verification. This is buried deep inside M2Crypto
+      # and the only way to disable it is to disable post connection checks on
+      # the socket itself.
+      self.sock = SSL.Connection(self.ssl_ctx)
+      if self.session:
+        self.sock.set_session(self.session)
+
+      if not self._postConnectionCheck:
+        self.sock.postConnectionCheck = None
+
+      self.sock.connect((self.host, self.port))
+
+  class HTTPSHandler(urllib2.HTTPSHandler):
+
+    def https_open(self, req):
+      return self.do_open(HTTPSConnection, req)
+
+  return HTTPSHandler()
+
+def urlopen_with_retry_on_authentication_errors(function,
+                                                retries,
+                                                sleeptime):
+  # See OPSAPS-28469: we retry on 401 errors on the presumption that we
+  # are hitting a race with the kinit from the kt_renewer.
+  attempt = 1
+  while True:
+    try:
+      return function()
+    except urllib2.HTTPError, err:
+      if err.code == 401 and attempt <= retries:
+        LOG.exception("Autentication error on attempt %d. Retrying after "
+                      "sleeping %f seconds." % (attempt, sleeptime))
+        time.sleep(sleeptime)
+        attempt += 1
+      else:
+        raise

+ 273 - 0
tools/ops/script_runner/patches/5.12_all_logging.patch

@@ -0,0 +1,273 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+diff --git a/desktop/core/src/desktop/conf.py b/desktop/core/src/desktop/conf.py
+index 152291f402..0bcaa70e04 100644
+--- a/desktop/core/src/desktop/conf.py
++++ b/desktop/core/src/desktop/conf.py
+@@ -373,6 +373,18 @@ COLLECT_USAGE = Config(
+   type=coerce_bool,
+   default=True)
+
++REST_RESPONSE_SIZE = Config(
++  key="rest_response_size",
++  help=_("Number of characters the rest api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
++THRIFT_RESPONSE_SIZE = Config(
++  key="thrift_response_size",
++  help=_("Number of characters the thrift api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
+ LEAFLET_TILE_LAYER = Config(
+   key="leaflet_tile_layer",
+   help=_("Tile layer server URL for the Leaflet map charts. Read more on http://leafletjs.com/reference.html#tilelayer. Make sure you add the tile domain to the img-src section of the 'secure_content_security_policy' configuration parameter as well."),
+@@ -1279,6 +1291,17 @@ MEMORY_PROFILER = Config(
+   default=False)
+
+
++def get_instrumentation_default():
++  """If django_debug_mode is True, this is automatically enabled"""
++  return DJANGO_DEBUG_MODE.get()
++
++INSTRUMENTATION = Config(
++  key='instrumentation',
++  help=_('Enable or disable instrumentation. If django_debug_mode is True, this is automatically enabled.'),
++  type=coerce_bool,
++  dynamic_default=get_instrumentation_default)
++
++
+ AUDIT_EVENT_LOG_DIR = Config(
+   key="audit_event_log_dir",
+   help=_("The directory where to store the auditing logs. Auditing is disable if the value is empty."),
+diff --git a/desktop/core/src/desktop/lib/rest/resource.py b/desktop/core/src/desktop/lib/rest/resource.py
+index 15200940d7..bf8fd88d44 100644
+--- a/desktop/core/src/desktop/lib/rest/resource.py
++++ b/desktop/core/src/desktop/lib/rest/resource.py
+@@ -16,9 +16,15 @@
+
+ import logging
+ import posixpath
++import time
++
++from django.utils.encoding import iri_to_uri, smart_str
++from django.utils.http import urlencode
+
+ from desktop.lib.i18n import smart_unicode
+
++from desktop import conf
++
+ LOG = logging.getLogger(__name__)
+
+
+@@ -69,6 +75,7 @@ class Resource(object):
+     @return: Raw body or JSON dictionary (if response content type is JSON).
+     """
+     path = self._join_uri(relpath)
++    start_time = time.time()
+     resp = self._client.execute(method,
+                                 path,
+                                 params=params,
+@@ -80,11 +87,23 @@ class Resource(object):
+                                 clear_cookies=clear_cookies)
+
+     if self._client.logger.isEnabledFor(logging.DEBUG):
+-      self._client.logger.debug(
+-          "%s Got response: %s%s" %
+-          (method,
+-           smart_unicode(resp.content[:1000], errors='replace'),
+-           len(resp.content) > 1000 and "..." or ""))
++      log_length = conf.REST_RESPONSE_SIZE.get() != -1 and conf.REST_RESPONSE_SIZE.get() # We want to output duration without content
++      duration = time.time() - start_time
++      message = '%s %s %s%s%s %s%s returned in %dms %s %s %s%s' % (
++        method,
++        type(self._client._session.auth) if self._client._session and self._client._session.auth else None,
++        self._client._base_url,
++        smart_str(path),
++        iri_to_uri('?' + urlencode(params)) if params else '',
++        smart_unicode(data, errors='replace')[:log_length] if data else "",
++        log_length and len(data) > log_length and "..." or "" if data else "",
++        (duration * 1000),
++        resp.status_code if resp else 0,
++        len(resp.content) if resp else 0,
++        smart_unicode(resp.content[:log_length], errors='replace') if resp else "",
++        log_length and len(resp.content) > log_length and "..." or "" if resp else ""
++      )
++      self._client.logger.debug("%s" % message)
+
+     return self._format_response(resp)
+
+diff --git a/desktop/core/src/desktop/lib/thrift_util.py b/desktop/core/src/desktop/lib/thrift_util.py
+index ad8b69288f..7688974fb2 100644
+--- a/desktop/core/src/desktop/lib/thrift_util.py
++++ b/desktop/core/src/desktop/lib/thrift_util.py
+@@ -36,6 +36,7 @@ from thrift.protocol.TMultiplexedProtocol import TMultiplexedProtocol
+ from django.conf import settings
+ from django.utils.translation import ugettext as _
+ from desktop.conf import SASL_MAX_BUFFER
++from desktop import conf
+
+ from desktop.lib.python_util import create_synchronous_io_multiplexer
+ from desktop.lib.thrift_.http_client import THttpClient
+@@ -440,7 +441,7 @@ class SuperClient(object):
+           log_msg = _unpack_guid_secret_in_handle(repr(ret))
+
+           # Truncate log message, increase output in DEBUG mode
+-          log_limit = 2000 if settings.DEBUG else 1000
++          log_limit = conf.THRIFT_RESPONSE_SIZE.get() if settings.DEBUG else 1000
+           log_msg = log_msg[:log_limit] + (log_msg[log_limit:] and '...')
+
+           duration = time.time() - st
+diff --git a/desktop/core/src/desktop/log/access.py b/desktop/core/src/desktop/log/access.py
+index bc7a4e82da..f668935b5b 100644
+--- a/desktop/core/src/desktop/log/access.py
++++ b/desktop/core/src/desktop/log/access.py
+@@ -22,6 +22,8 @@ This assumes a single-threaded server.
+
+ import logging
+ import re
++import resource
++import sys
+ import threading
+ import time
+
+@@ -79,16 +81,37 @@ class AccessInfo(dict):
+     self['proto'] = request.META.get('SERVER_PROTOCOL', '-')
+     self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
+     self['time'] = time.time()
++    self['duration'] = None
++    self['memory'] = None
++
++  def memory_usage_resource(self):
++    """
++      This is a lightweight way to get the total peak memory as
++       doing the diffing before/after request with guppy was too inconsistent and memory intensive.
++      """
++    rusage_denom = 1024
++    if sys.platform == 'darwin':
++      rusage_denom = rusage_denom * 1024
++    # get peak memory usage, bytes on OSX, Kilobytes on Linux
++    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
++
++  def log(self, level, msg=None, start_time=None):
++    is_instrumentation = desktop.conf.INSTRUMENTATION.get()
++    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None and is_instrumentation else ''
++    self['memory'] = ' (mem: %dmb)' % self.memory_usage_resource() if is_instrumentation else ''
+
+-  def log(self, level, msg=None):
+     if msg is not None:
+       self['msg'] = msg
+-      ACCESS_LOG.log(level,
+-                     '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s" -- %(msg)s' %
+-                     self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s-- %(msg)s' % self)
+     else:
+-      ACCESS_LOG.log(level,
+-                     '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"' % self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s' % self)
++
++    if is_instrumentation:
++      import gc
++      gc.collect()
++      for o in gc.garbage:
++        for r in gc.get_referrers(o):
++          ACCESS_LOG.log(level, 'ref for %r: %r' % (o, r))
+
+   def add_to_access_history(self, app):
+     """Record this user access to the recent access map"""
+@@ -138,16 +161,18 @@ class AccessInfo(dict):
+
+ _MODULE_RE = re.compile('[^.]*')
+
+-def log_page_hit(request, view_func, level=None):
++def log_page_hit(request, view_func, level=None, start_time=None):
+   """Log the request to the access log"""
+   if level is None:
+     level = logging.INFO
+   ai = AccessInfo(request)
+-  ai.log(level)
++  ai.log(level, start_time=start_time)
++
++  # Disabled for now as not used
+   # Find the app
+-  app_re_match = _MODULE_RE.match(view_func.__module__)
+-  app = app_re_match and app_re_match.group(0) or '-'
+-  ai.add_to_access_history(app)
++#   app_re_match = _MODULE_RE.match(view_func.__module__)
++#   app = app_re_match and app_re_match.group(0) or '-'
++#   ai.add_to_access_history(app)
+
+
+ def access_log(request, msg=None, level=None):
+diff --git a/desktop/core/src/desktop/middleware.py b/desktop/core/src/desktop/middleware.py
+index 9741790afc..0cc3d36052 100644
+--- a/desktop/core/src/desktop/middleware.py
++++ b/desktop/core/src/desktop/middleware.py
+@@ -273,6 +273,8 @@ class LoginAndPermissionMiddleware(object):
+     which tells us the log level. The downside is that we don't have the status code,
+     which isn't useful for status logging anyways.
+     """
++    request.ts = time.time()
++    request.view_func = view_func
+     access_log_level = getattr(view_func, 'access_log_level', None)
+     # First, skip views not requiring login
+
+@@ -315,7 +317,8 @@ class LoginAndPermissionMiddleware(object):
+         return PopupException(
+             _("You do not have permission to access the %(app_name)s application.") % {'app_name': app_accessed.capitalize()}, error_code=401).response(request)
+       else:
+-        log_page_hit(request, view_func, level=access_log_level)
++        if not hasattr(request, 'view_func'):
++          log_page_hit(request, view_func, level=access_log_level)
+         return None
+
+     logging.info("Redirecting to login page: %s", request.get_full_path())
+@@ -330,6 +333,11 @@ class LoginAndPermissionMiddleware(object):
+     else:
+       return HttpResponseRedirect("%s?%s=%s" % (settings.LOGIN_URL, REDIRECT_FIELD_NAME, urlquote(request.get_full_path())))
+
++  def process_response(self, request, response):
++    if hasattr(request, 'ts') and hasattr(request, 'view_func'):
++      log_page_hit(request, request.view_func, level=logging.INFO, start_time=request.ts)
++    return response
++
+
+ class JsonMessage(object):
+   def __init__(self, **kwargs):
+diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py
+index 2dafdd9e2a..e0aa364681 100644
+--- a/desktop/core/src/desktop/settings.py
++++ b/desktop/core/src/desktop/settings.py
+@@ -20,6 +20,7 @@
+ # Local customizations are done by symlinking a file
+ # as local_settings.py.
+
++import gc
+ import logging
+ import os
+ import pkg_resources
+@@ -497,6 +498,9 @@ if desktop.conf.MEMORY_PROFILER.get():
+   MEMORY_PROFILER = hpy()
+   MEMORY_PROFILER.setrelheap()
+
++# Instrumentation
++if desktop.conf.INSTRUMENTATION.get():
++  gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_OBJECTS)
+
+ if not desktop.conf.DATABASE_LOGGING.get():
+   def disable_database_logging():

+ 273 - 0
tools/ops/script_runner/patches/5.13_all_logging.patch

@@ -0,0 +1,273 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+diff --git a/desktop/core/src/desktop/conf.py b/desktop/core/src/desktop/conf.py
+index 152291f402..0bcaa70e04 100644
+--- a/desktop/core/src/desktop/conf.py
++++ b/desktop/core/src/desktop/conf.py
+@@ -373,6 +373,18 @@ COLLECT_USAGE = Config(
+   type=coerce_bool,
+   default=True)
+
++REST_RESPONSE_SIZE = Config(
++  key="rest_response_size",
++  help=_("Number of characters the rest api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
++THRIFT_RESPONSE_SIZE = Config(
++  key="thrift_response_size",
++  help=_("Number of characters the thrift api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
+ LEAFLET_TILE_LAYER = Config(
+   key="leaflet_tile_layer",
+   help=_("Tile layer server URL for the Leaflet map charts. Read more on http://leafletjs.com/reference.html#tilelayer. Make sure you add the tile domain to the img-src section of the 'secure_content_security_policy' configuration parameter as well."),
+@@ -1279,6 +1291,17 @@ MEMORY_PROFILER = Config(
+   default=False)
+
+
++def get_instrumentation_default():
++  """If django_debug_mode is True, this is automatically enabled"""
++  return DJANGO_DEBUG_MODE.get()
++
++INSTRUMENTATION = Config(
++  key='instrumentation',
++  help=_('Enable or disable instrumentation. If django_debug_mode is True, this is automatically enabled.'),
++  type=coerce_bool,
++  dynamic_default=get_instrumentation_default)
++
++
+ AUDIT_EVENT_LOG_DIR = Config(
+   key="audit_event_log_dir",
+   help=_("The directory where to store the auditing logs. Auditing is disable if the value is empty."),
+diff --git a/desktop/core/src/desktop/lib/rest/resource.py b/desktop/core/src/desktop/lib/rest/resource.py
+index 15200940d7..bf8fd88d44 100644
+--- a/desktop/core/src/desktop/lib/rest/resource.py
++++ b/desktop/core/src/desktop/lib/rest/resource.py
+@@ -16,9 +16,15 @@
+
+ import logging
+ import posixpath
++import time
++
++from django.utils.encoding import iri_to_uri, smart_str
++from django.utils.http import urlencode
+
+ from desktop.lib.i18n import smart_unicode
+
++from desktop import conf
++
+ LOG = logging.getLogger(__name__)
+
+
+@@ -69,6 +75,7 @@ class Resource(object):
+     @return: Raw body or JSON dictionary (if response content type is JSON).
+     """
+     path = self._join_uri(relpath)
++    start_time = time.time()
+     resp = self._client.execute(method,
+                                 path,
+                                 params=params,
+@@ -80,11 +87,23 @@ class Resource(object):
+                                 clear_cookies=clear_cookies)
+
+     if self._client.logger.isEnabledFor(logging.DEBUG):
+-      self._client.logger.debug(
+-          "%s Got response: %s%s" %
+-          (method,
+-           smart_unicode(resp.content[:1000], errors='replace'),
+-           len(resp.content) > 1000 and "..." or ""))
++      log_length = conf.REST_RESPONSE_SIZE.get() != -1 and conf.REST_RESPONSE_SIZE.get() # We want to output duration without content
++      duration = time.time() - start_time
++      message = '%s %s %s%s%s %s%s returned in %dms %s %s %s%s' % (
++        method,
++        type(self._client._session.auth) if self._client._session and self._client._session.auth else None,
++        self._client._base_url,
++        smart_str(path),
++        iri_to_uri('?' + urlencode(params)) if params else '',
++        smart_unicode(data, errors='replace')[:log_length] if data else "",
++        log_length and len(data) > log_length and "..." or "" if data else "",
++        (duration * 1000),
++        resp.status_code if resp else 0,
++        len(resp.content) if resp else 0,
++        smart_unicode(resp.content[:log_length], errors='replace') if resp else "",
++        log_length and len(resp.content) > log_length and "..." or "" if resp else ""
++      )
++      self._client.logger.debug("%s" % message)
+
+     return self._format_response(resp)
+
+diff --git a/desktop/core/src/desktop/lib/thrift_util.py b/desktop/core/src/desktop/lib/thrift_util.py
+index ad8b69288f..7688974fb2 100644
+--- a/desktop/core/src/desktop/lib/thrift_util.py
++++ b/desktop/core/src/desktop/lib/thrift_util.py
+@@ -36,6 +36,7 @@ from thrift.protocol.TMultiplexedProtocol import TMultiplexedProtocol
+ from django.conf import settings
+ from django.utils.translation import ugettext as _
+ from desktop.conf import SASL_MAX_BUFFER
++from desktop import conf
+
+ from desktop.lib.python_util import create_synchronous_io_multiplexer
+ from desktop.lib.thrift_.http_client import THttpClient
+@@ -440,7 +441,7 @@ class SuperClient(object):
+           log_msg = _unpack_guid_secret_in_handle(repr(ret))
+
+           # Truncate log message, increase output in DEBUG mode
+-          log_limit = 2000 if settings.DEBUG else 1000
++          log_limit = conf.THRIFT_RESPONSE_SIZE.get() if settings.DEBUG else 1000
+           log_msg = log_msg[:log_limit] + (log_msg[log_limit:] and '...')
+
+           duration = time.time() - st
+diff --git a/desktop/core/src/desktop/log/access.py b/desktop/core/src/desktop/log/access.py
+index bc7a4e82da..f668935b5b 100644
+--- a/desktop/core/src/desktop/log/access.py
++++ b/desktop/core/src/desktop/log/access.py
+@@ -22,6 +22,8 @@ This assumes a single-threaded server.
+
+ import logging
+ import re
++import resource
++import sys
+ import threading
+ import time
+
+@@ -79,16 +81,37 @@ class AccessInfo(dict):
+     self['proto'] = request.META.get('SERVER_PROTOCOL', '-')
+     self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
+     self['time'] = time.time()
++    self['duration'] = None
++    self['memory'] = None
++
++  def memory_usage_resource(self):
++    """
++      This is a lightweight way to get the total peak memory as
++       doing the diffing before/after request with guppy was too inconsistent and memory intensive.
++      """
++    rusage_denom = 1024
++    if sys.platform == 'darwin':
++      rusage_denom = rusage_denom * 1024
++    # get peak memory usage, bytes on OSX, Kilobytes on Linux
++    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
++
++  def log(self, level, msg=None, start_time=None):
++    is_instrumentation = desktop.conf.INSTRUMENTATION.get()
++    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None and is_instrumentation else ''
++    self['memory'] = ' (mem: %dmb)' % self.memory_usage_resource() if is_instrumentation else ''
+
+-  def log(self, level, msg=None):
+     if msg is not None:
+       self['msg'] = msg
+-      ACCESS_LOG.log(level,
+-                     '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s" -- %(msg)s' %
+-                     self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s-- %(msg)s' % self)
+     else:
+-      ACCESS_LOG.log(level,
+-                     '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"' % self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s' % self)
++
++    if is_instrumentation:
++      import gc
++      gc.collect()
++      for o in gc.garbage:
++        for r in gc.get_referrers(o):
++          ACCESS_LOG.log(level, 'ref for %r: %r' % (o, r))
+
+   def add_to_access_history(self, app):
+     """Record this user access to the recent access map"""
+@@ -138,16 +161,18 @@ class AccessInfo(dict):
+
+ _MODULE_RE = re.compile('[^.]*')
+
+-def log_page_hit(request, view_func, level=None):
++def log_page_hit(request, view_func, level=None, start_time=None):
+   """Log the request to the access log"""
+   if level is None:
+     level = logging.INFO
+   ai = AccessInfo(request)
+-  ai.log(level)
++  ai.log(level, start_time=start_time)
++
++  # Disabled for now as not used
+   # Find the app
+-  app_re_match = _MODULE_RE.match(view_func.__module__)
+-  app = app_re_match and app_re_match.group(0) or '-'
+-  ai.add_to_access_history(app)
++#   app_re_match = _MODULE_RE.match(view_func.__module__)
++#   app = app_re_match and app_re_match.group(0) or '-'
++#   ai.add_to_access_history(app)
+
+
+ def access_log(request, msg=None, level=None):
+diff --git a/desktop/core/src/desktop/middleware.py b/desktop/core/src/desktop/middleware.py
+index 9741790afc..0cc3d36052 100644
+--- a/desktop/core/src/desktop/middleware.py
++++ b/desktop/core/src/desktop/middleware.py
+@@ -273,6 +273,8 @@ class LoginAndPermissionMiddleware(object):
+     which tells us the log level. The downside is that we don't have the status code,
+     which isn't useful for status logging anyways.
+     """
++    request.ts = time.time()
++    request.view_func = view_func
+     access_log_level = getattr(view_func, 'access_log_level', None)
+     # First, skip views not requiring login
+
+@@ -315,7 +317,8 @@ class LoginAndPermissionMiddleware(object):
+         return PopupException(
+             _("You do not have permission to access the %(app_name)s application.") % {'app_name': app_accessed.capitalize()}, error_code=401).response(request)
+       else:
+-        log_page_hit(request, view_func, level=access_log_level)
++        if not hasattr(request, 'view_func'):
++          log_page_hit(request, view_func, level=access_log_level)
+         return None
+
+     logging.info("Redirecting to login page: %s", request.get_full_path())
+@@ -330,6 +333,11 @@ class LoginAndPermissionMiddleware(object):
+     else:
+       return HttpResponseRedirect("%s?%s=%s" % (settings.LOGIN_URL, REDIRECT_FIELD_NAME, urlquote(request.get_full_path())))
+
++  def process_response(self, request, response):
++    if hasattr(request, 'ts') and hasattr(request, 'view_func'):
++      log_page_hit(request, request.view_func, level=logging.INFO, start_time=request.ts)
++    return response
++
+
+ class JsonMessage(object):
+   def __init__(self, **kwargs):
+diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py
+index 2dafdd9e2a..e0aa364681 100644
+--- a/desktop/core/src/desktop/settings.py
++++ b/desktop/core/src/desktop/settings.py
+@@ -20,6 +20,7 @@
+ # Local customizations are done by symlinking a file
+ # as local_settings.py.
+
++import gc
+ import logging
+ import os
+ import pkg_resources
+@@ -497,6 +498,9 @@ if desktop.conf.MEMORY_PROFILER.get():
+   MEMORY_PROFILER = hpy()
+   MEMORY_PROFILER.setrelheap()
+
++# Instrumentation
++if desktop.conf.INSTRUMENTATION.get():
++  gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_OBJECTS)
+
+ if not desktop.conf.DATABASE_LOGGING.get():
+   def disable_database_logging():

+ 239 - 0
tools/ops/script_runner/patches/5.14_all_logging.patch

@@ -0,0 +1,239 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+diff --git a/apps/filebrowser/src/filebrowser/settings.py b/apps/filebrowser/src/filebrowser/settings.py
+index cb7a8c7c3d..9a783965a8 100644
+--- a/apps/filebrowser/src/filebrowser/settings.py
++++ b/apps/filebrowser/src/filebrowser/settings.py
+@@ -22,7 +22,7 @@ ICON = "filebrowser/art/icon_filebrowser_48.png"
+ MENU_INDEX = 20
+
+ from aws.s3.s3fs import PERMISSION_ACTION_S3
+-from azure.adls.webhdfs import PERMISSION_ACTION_ADLS
++PERMISSION_ACTION_ADLS = "adls_access"
+
+
+ PERMISSION_ACTIONS = (
+diff --git a/desktop/core/src/desktop/conf.py b/desktop/core/src/desktop/conf.py
+index 429ee983a5..7d20ad4f25 100644
+--- a/desktop/core/src/desktop/conf.py
++++ b/desktop/core/src/desktop/conf.py
+@@ -379,6 +379,18 @@ COLLECT_USAGE = Config(
+   type=coerce_bool,
+   default=True)
+
++REST_RESPONSE_SIZE = Config(
++  key="rest_response_size",
++  help=_("Number of characters the rest api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
++THRIFT_RESPONSE_SIZE = Config(
++  key="thrift_response_size",
++  help=_("Number of characters the thrift api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
+ LEAFLET_TILE_LAYER = Config(
+   key="leaflet_tile_layer",
+   help=_("Tile layer server URL for the Leaflet map charts. Read more on http://leafletjs.com/reference.html#tilelayer. Make sure you add the tile domain to the img-src section of the 'secure_content_security_policy' configuration parameter as well."),
+@@ -1314,6 +1326,17 @@ MEMORY_PROFILER = Config(
+   default=False)
+
+
++def get_instrumentation_default():
++  """If django_debug_mode is True, this is automatically enabled"""
++  return DJANGO_DEBUG_MODE.get()
++
++INSTRUMENTATION = Config(
++  key='instrumentation',
++  help=_('Enable or disable instrumentation. If django_debug_mode is True, this is automatically enabled.'),
++  type=coerce_bool,
++  dynamic_default=get_instrumentation_default)
++
++
+ AUDIT_EVENT_LOG_DIR = Config(
+   key="audit_event_log_dir",
+   help=_("The directory where to store the auditing logs. Auditing is disable if the value is empty."),
+diff --git a/desktop/core/src/desktop/lib/rest/resource.py b/desktop/core/src/desktop/lib/rest/resource.py
+index aa2e01a993..20150f9a09 100644
+--- a/desktop/core/src/desktop/lib/rest/resource.py
++++ b/desktop/core/src/desktop/lib/rest/resource.py
+@@ -18,8 +18,12 @@ import logging
+ import posixpath
+ import time
+
++from django.utils.encoding import iri_to_uri, smart_str
++from django.utils.http import urlencode
++
+ from desktop.lib.i18n import smart_unicode
+
++from desktop import conf
+
+ LOG = logging.getLogger(__name__)
+
+@@ -82,15 +86,24 @@ class Resource(object):
+                                 urlencode=self._urlencode,
+                                 clear_cookies=clear_cookies)
+
+-    if log_response and self._client.logger.isEnabledFor(logging.DEBUG):
+-      self._client.logger.debug(
+-        "%s %s Got response%s: %s%s" % (
+-          method,
+-          smart_unicode(path, errors='ignore'),
+-           ' in %dms' % ((time.time() - start_time) * 1000),
+-           smart_unicode(resp.content[:1000], errors='replace'),
+-           len(resp.content) > 1000 and "..." or "")
++    if self._client.logger.isEnabledFor(logging.DEBUG):
++      log_length = conf.REST_RESPONSE_SIZE.get() != -1 and conf.REST_RESPONSE_SIZE.get() # We want to output duration without content
++      duration = time.time() - start_time
++      message = '%s %s %s%s%s %s%s returned in %dms %s %s %s%s' % (
++        method,
++        type(self._client._session.auth) if self._client._session and self._client._session.auth else None,
++        self._client._base_url,
++        smart_str(path),
++        iri_to_uri('?' + urlencode(params)) if params else '',
++        smart_unicode(data, errors='replace')[:log_length] if data else "",
++        log_length and len(data) > log_length and "..." or "" if data else "",
++        (duration * 1000),
++        resp.status_code if resp else 0,
++        len(resp.content) if resp else 0,
++        smart_unicode(resp.content[:log_length], errors='replace') if resp else "",
++        log_length and len(resp.content) > log_length and "..." or "" if resp else ""
+       )
++      self._client.logger.debug("%s" % message)
+
+     return self._format_response(resp)
+
+diff --git a/desktop/core/src/desktop/lib/thrift_util.py b/desktop/core/src/desktop/lib/thrift_util.py
+index b36a99d000..a1c40b5092 100644
+--- a/desktop/core/src/desktop/lib/thrift_util.py
++++ b/desktop/core/src/desktop/lib/thrift_util.py
+@@ -36,6 +36,7 @@ from thrift.protocol.TMultiplexedProtocol import TMultiplexedProtocol
+ from django.conf import settings
+ from django.utils.translation import ugettext as _
+ from desktop.conf import SASL_MAX_BUFFER, CHERRYPY_SERVER_THREADS
++from desktop import conf
+
+ from desktop.lib.python_util import create_synchronous_io_multiplexer
+ from desktop.lib.thrift_.http_client import THttpClient
+@@ -440,7 +441,7 @@ class SuperClient(object):
+           log_msg = _unpack_guid_secret_in_handle(repr(ret))
+
+           # Truncate log message, increase output in DEBUG mode
+-          log_limit = 2000 if settings.DEBUG else 1000
++          log_limit = conf.THRIFT_RESPONSE_SIZE.get() if settings.DEBUG else 1000
+           log_msg = log_msg[:log_limit] + (log_msg[log_limit:] and '...')
+
+           duration = time.time() - st
+diff --git a/desktop/core/src/desktop/log/access.py b/desktop/core/src/desktop/log/access.py
+index 679f9d1035..ad3c48805f 100644
+--- a/desktop/core/src/desktop/log/access.py
++++ b/desktop/core/src/desktop/log/access.py
+@@ -22,6 +22,8 @@ This assumes a single-threaded server.
+
+ import logging
+ import re
++import resource
++import sys
+ import threading
+ import time
+
+@@ -81,15 +83,36 @@ class AccessInfo(dict):
+     self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
+     self['time'] = time.time()
+     self['duration'] = None
++    self['memory'] = None
++
++  def memory_usage_resource(self):
++    """
++      This is a lightweight way to get the total peak memory as
++       doing the diffing before/after request with guppy was too inconsistent and memory intensive.
++      """
++    rusage_denom = 1024
++    if sys.platform == 'darwin':
++      rusage_denom = rusage_denom * 1024
++    # get peak memory usage, bytes on OSX, Kilobytes on Linux
++    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
+
+   def log(self, level, msg=None, start_time=None):
+-    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None else ''
++    is_instrumentation = desktop.conf.INSTRUMENTATION.get()
++    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None and is_instrumentation else ''
++    self['memory'] = ' (mem: %dmb)' % self.memory_usage_resource() if is_instrumentation else ''
+
+     if msg is not None:
+       self['msg'] = msg
+-      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s -- %(msg)s' % self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s-- %(msg)s' % self)
+     else:
+-      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s' % self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s' % self)
++
++    if is_instrumentation:
++      import gc
++      gc.collect()
++      for o in gc.garbage:
++        for r in gc.get_referrers(o):
++          ACCESS_LOG.log(level, 'ref for %r: %r' % (o, r))
+
+   def add_to_access_history(self, app):
+     """Record this user access to the recent access map"""
+diff --git a/desktop/core/src/desktop/middleware.py b/desktop/core/src/desktop/middleware.py
+index 0feaf5b545..b53df44ad2 100644
+--- a/desktop/core/src/desktop/middleware.py
++++ b/desktop/core/src/desktop/middleware.py
+@@ -317,7 +317,8 @@ class LoginAndPermissionMiddleware(object):
+         return PopupException(
+             _("You do not have permission to access the %(app_name)s application.") % {'app_name': app_accessed.capitalize()}, error_code=401).response(request)
+       else:
+-        log_page_hit(request, view_func, level=access_log_level)
++        if not hasattr(request, 'view_func'):
++          log_page_hit(request, view_func, level=access_log_level)
+         return None
+
+     logging.info("Redirecting to login page: %s", request.get_full_path())
+@@ -334,7 +335,7 @@ class LoginAndPermissionMiddleware(object):
+
+   def process_response(self, request, response):
+     if hasattr(request, 'ts') and hasattr(request, 'view_func'):
+-      log_page_hit(request, request.view_func, level=logging.DEBUG, start_time=request.ts)
++      log_page_hit(request, request.view_func, level=logging.INFO, start_time=request.ts)
+     return response
+
+
+diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py
+index a77673739d..7f83f68cc6 100644
+--- a/desktop/core/src/desktop/settings.py
++++ b/desktop/core/src/desktop/settings.py
+@@ -20,6 +20,7 @@
+ # Local customizations are done by symlinking a file
+ # as local_settings.py.
+
++import gc
+ import logging
+ import os
+ import pkg_resources
+@@ -506,6 +507,9 @@ if desktop.conf.MEMORY_PROFILER.get():
+   MEMORY_PROFILER = hpy()
+   MEMORY_PROFILER.setrelheap()
+
++# Instrumentation
++if desktop.conf.INSTRUMENTATION.get():
++  gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_OBJECTS)
+
+ if not desktop.conf.DATABASE_LOGGING.get():
+   def disable_database_logging():

+ 224 - 0
tools/ops/script_runner/patches/5.15_all_logging.patch

@@ -0,0 +1,224 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+diff --git a/apps/filebrowser/src/filebrowser/settings.py b/apps/filebrowser/src/filebrowser/settings.py
+index cb7a8c7c3d..9a783965a8 100644
+--- a/apps/filebrowser/src/filebrowser/settings.py
++++ b/apps/filebrowser/src/filebrowser/settings.py
+@@ -22,7 +22,7 @@ ICON = "filebrowser/art/icon_filebrowser_48.png"
+ MENU_INDEX = 20
+
+ from aws.s3.s3fs import PERMISSION_ACTION_S3
+-from azure.adls.webhdfs import PERMISSION_ACTION_ADLS
++PERMISSION_ACTION_ADLS = "adls_access"
+
+
+ PERMISSION_ACTIONS = (
+diff --git a/desktop/core/src/desktop/conf.py b/desktop/core/src/desktop/conf.py
+index 6a84e9ca28..6095fad1c6 100644
+--- a/desktop/core/src/desktop/conf.py
++++ b/desktop/core/src/desktop/conf.py
+@@ -397,6 +397,18 @@ COLLECT_USAGE = Config(
+   type=coerce_bool,
+   default=True)
+
++REST_RESPONSE_SIZE = Config(
++  key="rest_response_size",
++  help=_("Number of characters the rest api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
++THRIFT_RESPONSE_SIZE = Config(
++  key="thrift_response_size",
++  help=_("Number of characters the thrift api reponse calls to dump to the logs when debug is enabled."),
++  type=int,
++  default=1000)
++
+ LEAFLET_TILE_LAYER = Config(
+   key="leaflet_tile_layer",
+   help=_("Tile layer server URL for the Leaflet map charts. Read more on http://leafletjs.com/reference.html#tilelayer. Make sure you add the tile domain to the img-src section of the 'secure_content_security_policy' configuration parameter as well."),
+@@ -1338,6 +1350,17 @@ MEMORY_PROFILER = Config(
+   default=False)
+
+
++def get_instrumentation_default():
++  """If django_debug_mode is True, this is automatically enabled"""
++  return DJANGO_DEBUG_MODE.get()
++
++INSTRUMENTATION = Config(
++  key='instrumentation',
++  help=_('Enable or disable instrumentation. If django_debug_mode is True, this is automatically enabled.'),
++  type=coerce_bool,
++  dynamic_default=get_instrumentation_default)
++
++
+ AUDIT_EVENT_LOG_DIR = Config(
+   key="audit_event_log_dir",
+   help=_("The directory where to store the auditing logs. Auditing is disable if the value is empty."),
+diff --git a/desktop/core/src/desktop/lib/rest/resource.py b/desktop/core/src/desktop/lib/rest/resource.py
+index 04644a502b..0310611b38 100644
+--- a/desktop/core/src/desktop/lib/rest/resource.py
++++ b/desktop/core/src/desktop/lib/rest/resource.py
+@@ -18,9 +18,13 @@ import logging
+ import posixpath
+ import time
+
++from django.utils.encoding import iri_to_uri, smart_str
++from django.utils.http import urlencode
++
+ from desktop.lib.i18n import smart_unicode
+ from desktop.lib.apputil import WARN_LEVEL_CALL_DURATION_MS, INFO_LEVEL_CALL_DURATION_MS
+
++from desktop import conf
+
+ LOG = logging.getLogger(__name__)
+
+@@ -98,13 +102,20 @@ class Resource(object):
+
+     if log_response:
+       duration = time.time() - start_time
+-      message = "%s %s Got response%s: %s%s" % (
+-          method,
+-          smart_unicode(path, errors='ignore'),
+-          ' in %dms' % (duration * 1000),
+-          smart_unicode(resp.content[:1000], errors='replace'),
+-          len(resp.content) > 1000 and "..." or ""
+-      )
++      log_length = conf.REST_RESPONSE_SIZE.get() != -1 and conf.REST_RESPONSE_SIZE.get() # We want to output duration without content
++      message = '%s %s %s%s%s %s%s returned in %dms %s %s %s%s' % (
++        method,
++        type(self._client._session.auth) if self._client._session and self._client._session.auth else None,
++        self._client._base_url,
++        smart_str(path),
++        iri_to_uri('?' + urlencode(params)) if params else '',
++        smart_unicode(data, errors='replace')[:log_length] if data else "",
++        log_length and len(data) > log_length and "..." or "" if data else "",
++        (duration * 1000),
++        resp.status_code if resp else 0,
++        len(resp.content) if resp else 0,
++        smart_unicode(resp.content[:log_length], errors='replace') if resp else "",
++        log_length and len(resp.content) > log_length and "..." or "" if resp else "")
+       log_if_slow_call(duration=duration, message=message, logger=self._client.logger)
+
+     return resp
+@@ -199,4 +210,4 @@ def log_if_slow_call(duration, message, logger):
+   elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000:
+     logger.info('SLOW: %.2f - %s' % (duration, message))
+   else:
+-    logger.debug(message)
++    logging.debug(message)
+diff --git a/desktop/core/src/desktop/lib/thrift_util.py b/desktop/core/src/desktop/lib/thrift_util.py
+index d6ab2a3e74..2f6503161c 100644
+--- a/desktop/core/src/desktop/lib/thrift_util.py
++++ b/desktop/core/src/desktop/lib/thrift_util.py
+@@ -44,6 +44,7 @@ from desktop.lib.thrift_.TSSLSocketWithWildcardSAN import TSSLSocketWithWildcard
+ from desktop.lib.thrift_sasl import TSaslClientTransport
+ from desktop.lib.exceptions import StructuredException, StructuredThriftTransportException
+
++from desktop import conf
+
+ LOG = logging.getLogger(__name__)
+
+@@ -441,7 +442,7 @@ class SuperClient(object):
+           log_msg = _unpack_guid_secret_in_handle(repr(ret))
+
+           # Truncate log message, increase output in DEBUG mode
+-          log_limit = 2000 if settings.DEBUG else 1000
++          log_limit = conf.THRIFT_RESPONSE_SIZE.get() if settings.DEBUG else 1000
+           log_msg = log_msg[:log_limit] + (log_msg[log_limit:] and '...')
+
+           duration = time.time() - st
+@@ -755,4 +756,4 @@ def log_if_slow_call(duration, message):
+   elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000:
+     LOG.info('SLOW: %.2f - %s' % (duration, message))
+   else:
+-    LOG.debug(message)
++    logging.debug(message)
+diff --git a/desktop/core/src/desktop/log/access.py b/desktop/core/src/desktop/log/access.py
+index 679f9d1035..ad3c48805f 100644
+--- a/desktop/core/src/desktop/log/access.py
++++ b/desktop/core/src/desktop/log/access.py
+@@ -22,6 +22,8 @@ This assumes a single-threaded server.
+
+ import logging
+ import re
++import resource
++import sys
+ import threading
+ import time
+
+@@ -81,15 +83,36 @@ class AccessInfo(dict):
+     self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
+     self['time'] = time.time()
+     self['duration'] = None
++    self['memory'] = None
++
++  def memory_usage_resource(self):
++    """
++      This is a lightweight way to get the total peak memory as
++       doing the diffing before/after request with guppy was too inconsistent and memory intensive.
++      """
++    rusage_denom = 1024
++    if sys.platform == 'darwin':
++      rusage_denom = rusage_denom * 1024
++    # get peak memory usage, bytes on OSX, Kilobytes on Linux
++    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
+
+   def log(self, level, msg=None, start_time=None):
+-    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None else ''
++    is_instrumentation = desktop.conf.INSTRUMENTATION.get()
++    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None and is_instrumentation else ''
++    self['memory'] = ' (mem: %dmb)' % self.memory_usage_resource() if is_instrumentation else ''
+
+     if msg is not None:
+       self['msg'] = msg
+-      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s -- %(msg)s' % self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s-- %(msg)s' % self)
+     else:
+-      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s' % self)
++      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s%(memory)s' % self)
++
++    if is_instrumentation:
++      import gc
++      gc.collect()
++      for o in gc.garbage:
++        for r in gc.get_referrers(o):
++          ACCESS_LOG.log(level, 'ref for %r: %r' % (o, r))
+
+   def add_to_access_history(self, app):
+     """Record this user access to the recent access map"""
+diff --git a/desktop/core/src/desktop/settings.py b/desktop/core/src/desktop/settings.py
+index db89c9dfb0..62c26e2b07 100644
+--- a/desktop/core/src/desktop/settings.py
++++ b/desktop/core/src/desktop/settings.py
+@@ -20,6 +20,7 @@
+ # Local customizations are done by symlinking a file
+ # as local_settings.py.
+
++import gc
+ import logging
+ import os
+ import pkg_resources
+@@ -509,6 +510,9 @@ if desktop.conf.MEMORY_PROFILER.get():
+   MEMORY_PROFILER = hpy()
+   MEMORY_PROFILER.setrelheap()
+
++# Instrumentation
++if desktop.conf.INSTRUMENTATION.get():
++  gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_OBJECTS)
+
+ if not desktop.conf.DATABASE_LOGGING.get():
+   def disable_database_logging():

+ 36 - 0
tools/ops/script_runner/patches/fix_cm_config_file.patch

@@ -0,0 +1,36 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+diff --git a/desktop/core/src/desktop/manage_entry.py b/desktop/core/src/desktop/manage_entry.py
+index e02436ff73..e9971df789 100644
+--- a/desktop/core/src/desktop/manage_entry.py
++++ b/desktop/core/src/desktop/manage_entry.py
+@@ -41,6 +41,7 @@ def entry():
+   from django.core.management.base import BaseCommand
+
+   os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
++  cm_config_file = '/etc/cloudera-scm-agent/config.ini'
+
+   # What's the subcommand being run?
+   # This code uses the same logic from django.core.management to handle command args
+@@ -57,7 +58,6 @@ def entry():
+                         "import_ldap_group", "import_ldap_user", "sync_ldap_users_and_groups", "useradmin_sync_with_unix" ]
+     if subcommand in commands_req_db:
+       #Check if this is a CM managed cluster
+-      cm_config_file = '/etc/cloudera-scm-agent/config.ini'
+       if os.path.isfile(cm_config_file) and "--cm-managed" not in sys.argv:
+         if not "HUE_CONF_DIR" in os.environ:
+           print "ALERT: This appears to be a CM Managed environment"

+ 66 - 0
tools/ops/script_runner/queries.txt

@@ -0,0 +1,66 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+fetchone|SELECT COUNT(*) FROM "AUTH_USER"
+fetchone|SELECT COUNT(*) FROM "USERADMIN_USERPROFILE" WHERE "USERADMIN_USERPROFILE"."LAST_ACTIVITY" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss')
+fetchone|SELECT COUNT(*) FROM "AUTH_USER"
+fetchone|SELECT COUNT(*) FROM "USERADMIN_USERPROFILE" WHERE "USERADMIN_USERPROFILE"."LAST_ACTIVITY" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss')
+fetchone|SELECT COUNT(*) FROM "AUTH_USER"
+fetchone|SELECT COUNT(*) FROM "USERADMIN_USERPROFILE" WHERE "USERADMIN_USERPROFILE"."LAST_ACTIVITY" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss')
+fetchone|SELECT COUNT(*) FROM "AUTH_USER"
+fetchone|SELECT COUNT(*) FROM "USERADMIN_USERPROFILE" WHERE "USERADMIN_USERPROFILE"."LAST_ACTIVITY" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss')
+fetchmany|SELECT "AXES_ACCESSATTEMPT"."ID", "AXES_ACCESSATTEMPT"."USER_AGENT", "AXES_ACCESSATTEMPT"."IP_ADDRESS", "AXES_ACCESSATTEMPT"."USERNAME", "AXES_ACCESSATTEMPT"."TRUSTED", "AXES_ACCESSATTEMPT"."HTTP_ACCEPT", "AXES_ACCESSATTEMPT"."PATH_INFO", "AXES_ACCESSATTEMPT"."ATTEMPT_TIME", "AXES_ACCESSATTEMPT"."GET_DATA", "AXES_ACCESSATTEMPT"."POST_DATA", "AXES_ACCESSATTEMPT"."FAILURES_SINCE_START" FROM "AXES_ACCESSATTEMPT" WHERE ("AXES_ACCESSATTEMPT"."USERNAME" IS NULL AND "AXES_ACCESSATTEMPT"."IP_ADDRESS" = '172.18.18.72'  AND "AXES_ACCESSATTEMPT"."TRUSTED" = 1 ) ORDER BY "AXES_ACCESSATTEMPT"."ATTEMPT_TIME" DESC
+fetchmany|SELECT "AUTH_USER"."ID", "AUTH_USER"."PASSWORD", "AUTH_USER"."LAST_LOGIN", "AUTH_USER"."IS_SUPERUSER", "AUTH_USER"."USERNAME", "AUTH_USER"."FIRST_NAME", "AUTH_USER"."LAST_NAME", "AUTH_USER"."EMAIL", "AUTH_USER"."IS_STAFF", "AUTH_USER"."IS_ACTIVE", "AUTH_USER"."DATE_JOINED" FROM "AUTH_USER" WHERE "AUTH_USER"."USERNAME" IS NULL
+fetchmany|SELECT "AXES_ACCESSATTEMPT"."ID", "AXES_ACCESSATTEMPT"."USER_AGENT", "AXES_ACCESSATTEMPT"."IP_ADDRESS", "AXES_ACCESSATTEMPT"."USERNAME", "AXES_ACCESSATTEMPT"."TRUSTED", "AXES_ACCESSATTEMPT"."HTTP_ACCEPT", "AXES_ACCESSATTEMPT"."PATH_INFO", "AXES_ACCESSATTEMPT"."ATTEMPT_TIME", "AXES_ACCESSATTEMPT"."GET_DATA", "AXES_ACCESSATTEMPT"."POST_DATA", "AXES_ACCESSATTEMPT"."FAILURES_SINCE_START" FROM "AXES_ACCESSATTEMPT" WHERE ("AXES_ACCESSATTEMPT"."IP_ADDRESS" = '172.18.18.72'  AND "AXES_ACCESSATTEMPT"."TRUSTED" = 0 ) ORDER BY "AXES_ACCESSATTEMPT"."ATTEMPT_TIME" DESC
+fetchone|SELECT COUNT(*) FROM "AUTH_USER"
+fetchmany|SELECT "DESKTOP_SETTINGS"."ID", "DESKTOP_SETTINGS"."COLLECT_USAGE", "DESKTOP_SETTINGS"."TOURS_AND_TUTORIALS" FROM "DESKTOP_SETTINGS" WHERE "DESKTOP_SETTINGS"."ID" = 1
+fetchmany|SELECT "DESKTOP_SETTINGS"."ID", "DESKTOP_SETTINGS"."COLLECT_USAGE", "DESKTOP_SETTINGS"."TOURS_AND_TUTORIALS" FROM "DESKTOP_SETTINGS" WHERE "DESKTOP_SETTINGS"."ID" = 1
+fetchone|SELECT * FROM (SELECT ROWNUM AS "_RN", "_SUB".* FROM (SELECT (1) AS "A" FROM "DJANGO_SESSION" WHERE "DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la' ) "_SUB" WHERE ROWNUM <= 1) WHERE "_RN" > 0
+none|UPDATE "DJANGO_SESSION" SET "SESSION_DATA" = 'Y2VjOWIyZDE2NjBkN2Y3NjVhN2Q2YmNiZTU2ZWY5M2ZmODUxZDNlNjp7InRlc3Rjb29raWUiOiJ3b3JrZWQifQ==', "EXPIRE_DATE" = to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') WHERE "DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'
+none|INSERT INTO "DJANGO_SESSION" ("SESSION_KEY", "SESSION_DATA", "EXPIRE_DATE") SELECT 'piuyi8amnknxlzk3k6zp4k50seov07la', 'Y2VjOWIyZDE2NjBkN2Y3NjVhN2Q2YmNiZTU2ZWY5M2ZmODUxZDNlNjp7InRlc3Rjb29raWUiOiJ3b3JrZWQifQ==', to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') FROM DUAL
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT "DJANGO_SESSION"."SESSION_KEY", "DJANGO_SESSION"."SESSION_DATA", "DJANGO_SESSION"."EXPIRE_DATE" FROM "DJANGO_SESSION" WHERE ("DJANGO_SESSION"."SESSION_KEY" = 'piuyi8amnknxlzk3k6zp4k50seov07la'  AND "DJANGO_SESSION"."EXPIRE_DATE" > to_date('2016/Nov/18 05:31:42', 'yyyy/mon/dd hh24:mi:ss') )
+fetchmany|SELECT * FROM AUTH_USER

+ 134 - 0
tools/ops/script_runner/script_runner

@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, sys
+from os import walk
+import os.path
+import subprocess
+import logging
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+lib_dir = '%s/lib' % script_dir
+#sys.argv.append("--cm-managed")
+if 'NO_CM' in os.environ.keys():
+  if 'HUE_CONF_DIR' in os.environ.keys():
+    if not 'HUE_CONF_DIR_ORIG' in os.environ.keys():
+      os.environ['HUE_CONF_DIR_ORIG'] = os.environ.get('HUE_CONF_DIR')
+
+if not os.path.isdir(lib_dir):
+  print("The lib directory is missing.  Please download the entire package and not just the script_runner file")
+  print("git clone https://github.com/cmconner156/hue_scripts.git /opt/hue_scripts")
+  print("or")
+  print("cd /opt && rm -Rf hue_scripts && wget https://github.com/cmconner156/hue_scripts/archive/master.zip && unzip master.zip && mv hue_scripts-master hue_scripts && rm -f master.zip")
+  exit(1)
+
+if not os.environ["USER"] == "root":
+  print("This script must be run as root")
+  sys.exit(1)
+
+sys.path.insert(0, lib_dir)
+
+try:
+  from cm_environment import set_cm_environment, reload_with_cm_env
+except ImportError as e:
+  print("Unable to load cm_environment please send following to support:")
+  print("sys.path: %s" % sys.path)
+  print("All files in script_dir: %s" % script_dir)
+  files = []
+  for (dirpath, dirnames, filenames) in walk(script_dir):
+    for file in filenames:
+      if file == "script_runner" or "lib" in dirpath:
+        print("%s/%s" % (dirpath, file))
+        files.append(file)
+
+  exit(1)
+
+hue_config = set_cm_environment()
+hue_path = hue_config['hue_path']
+hue_bin_dir = hue_config['hue_bin_dir']
+HUE_CONF_DIR = hue_config['HUE_CONF_DIR']
+parcel_dir = hue_config['parcel_dir']
+parcel_name = hue_config['parcel_name']
+
+if not "SKIP_RELOAD" in os.environ.keys():
+  reload_with_cm_env()
+
+if 'NO_CM' in os.environ.keys():
+  if 'HUE_CONF_DIR_ORIG' in os.environ.keys():
+    HUE_CONF_DIR = os.environ.get('HUE_CONF_DIR_ORIG')
+    os.environ['HUE_CONF_DIR'] = HUE_CONF_DIR
+
+activate_file = hue_bin_dir + "/activate_this.py"
+hue_bin = hue_bin_dir + "/hue"
+
+import os; activate_this=os.path.join(os.path.dirname(os.path.realpath(activate_file)), 'activate_this.py'); exec(compile(open(activate_this).read(), activate_this, 'exec'), dict(__file__=activate_this)); del activate_this
+
+with open(hue_bin, 'rU') as f:
+  for line in f:
+     if "__requires__" in line:
+       desktop_ver = line.split("'")
+
+__requires__ = desktop_ver[1]
+
+import sys
+from pkg_resources import load_entry_point
+from django.conf import settings
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
+
+from custom_commands.settings import *
+settings.INSTALLED_APPS.append('custom_commands')
+
+from desktop.conf import DATABASE as desktop_database
+logging.info("Using the following config make sure it looks correct")
+
+try:
+  import platform
+  logging.info("Python Version: %s" % platform.python_version())
+  logging.info("OS Version: %s %s" % (platform.linux_distribution()[0], platform.linux_distribution()[1]))
+except:
+  pass
+logging.info("HUE_CONF_DIR: %s" % HUE_CONF_DIR)
+logging.info("parcel_dir: %s" % parcel_dir)
+logging.info("parcel_name: %s" % parcel_name)
+logging.info("hue_bin_dir: %s" % hue_bin_dir)
+logging.info("DB Engine: %s" % desktop_database.ENGINE.get())
+logging.info("DB Name: %s" % desktop_database.NAME.get())
+logging.info("DB User: %s" % desktop_database.USER.get())
+logging.info("DB Host: %s" % desktop_database.HOST.get())
+logging.info("DB Port: %s" % str(desktop_database.PORT.get()))
+if hue_config['LD_LIBRARY_PATH'] is not None:
+  logging.info("LD_LIBRARY_PATH: %s" % hue_config["LD_LIBRARY_PATH"])
+
+if __name__ == '__main__':
+#    sys.argv.remove("--cm-managed")
+    try:
+        sys.exit(
+            load_entry_point('desktop', 'console_scripts', 'hue')()
+        )
+    except UnboundLocalError, e:
+        logging.info("Command failed because of bug in older versions of CDH: %s" % e)
+        logging.info("Patching and re-running")
+        patch_file = "%s/patches/fix_cm_config_file.patch" % script_dir
+        patch_cmd = ["patch", "-p1"]
+        patch_file_in = open(patch_file)
+        os.chdir(hue_path)
+        subprocess.call(patch_cmd, stdin=patch_file_in)
+        os.chdir(script_dir)
+        sys.exit(
+            load_entry_point('desktop', 'console_scripts', 'hue')()
+        )