Kaynağa Gözat

[tools] Add latest set of op scripts

Christopher Conner 10 yıl önce
ebeveyn
işleme
ca8019edac

+ 29 - 0
tools/ops/bees_monitor.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+API_URL='localhost:7180'
+CLUSTER='Cluster 1 - CDH4'
+EMAIL_TO='root'
+
+curl -u admin "http://$API_URL/api/v2/clusters/$CLUSTER/services" > new_status
+diff new_status old_status > last_change
+if [ $? -ne 0 ]
+then
+cat last_change new_status | mail -s "Status of $CLUSTER has Changed" $EMAIL_TO
+fi
+
+mv new_status old_status

+ 64 - 0
tools/ops/create_userprofile_json.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HUE_BIN=/opt/cloudera/parcels/CDH/share/hue/build/env/bin/hue
+export JSON_FILE=/tmp/authuser.json
+export TXT_FILE=/tmp/authuser.txt
+export METHOD="EXTERNAL"
+
+NEW="false"
+echo "["
+while read -r LINE
+do
+
+   if [[ ${LINE} =~ '--' ]]
+   then
+     NEW="true"
+     echo "  {"
+     echo "    \"pk\": $ID,"
+     echo "    \"model\": \"useradmin.userprofile\","
+     echo "    \"fields\": {"
+     echo "      \"creation_method\": \"${METHOD}\","
+     echo "      \"user\": $ID,"
+     echo "      \"home_directory\": \"/user/$USERNAME\""
+       echo "  },"
+   fi
+   if [[ ${NEW} =~ "false" ]]
+   then
+     if [[ ${LINE} =~ "pk" ]]
+     then
+       ID=`echo ${LINE} | awk -F: '{print $2}' | awk -F, '{print $1}' | awk '{print $1}'`
+     fi 
+     if [[ ${LINE} =~ "username" ]]
+     then 
+       USERNAME=`echo ${LINE} | awk -F: '{print $2}' | awk -F, '{print $1}' | awk -F\" '{print $2}'`
+     fi
+   fi
+   NEW="false"
+
+done < <(cat ${TXT_FILE})
+
+
+echo "  {"
+echo "    \"pk\": $ID,"
+echo "    \"model\": \"useradmin.userprofile\","
+echo "    \"fields\": {"
+echo "      \"creation_method\": \"${METHOD}\","
+echo "      \"user\": $ID,"
+echo "      \"home_directory\": \"/user/$USERNAME\""
+echo "   }"
+echo "]"

+ 42 - 0
tools/ops/hbase_break_test.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HUE_SERVER="cdh46-1"
+HUE_PORT="8888"
+HUE_PASS_URL="${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_CREATE_URL="${HUE_SERVER}:${HUE_PORT}/jobsub/designs/hive/new"
+admin_user="cconner"
+admin_pass="password"
+
+
+curl -i -c ${admin_user}_cookie.txt -d "username=${admin_user}&password=${admin_pass}" "${HUE_PASS_URL}" > /dev/null
+
+
+while read -r LINE
+do
+
+   ACTION=`echo ${LINE} | awk '{print $1}'`
+   URL=`echo ${LINE} | awk '{print $2}'`
+   HUE_CREATE_URL="http://${HUE_SERVER}:${HUE_PORT}/${URL}"
+   echo ${ACTION}
+   echo ${HUE_CREATE_URL}
+
+   curl -X ${ACTION} --dump-header ${admin_user}_headers.txt -i -b ${admin_user}_cookie.txt "${HUE_CREATE_URL}" > /dev/null
+   
+   sleep 1
+
+done < <(cat hbase.txt)

+ 229 - 0
tools/ops/hue_beeswax_tester.sh

@@ -0,0 +1,229 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This script requires curl to be installed.  Also, it does not have to run on the Hue server, it can run anywhere that curl is installed.  As long as that host can reach the Hue server.
+
+# Please change USERS to contain a list of users that you would like to use to generate load
+
+#USERS="cconner test1 test2 test3 test4 test5 test6 test7 test8 test9 test10"
+
+#Please enter the Hue server name below
+HUE_SERVER="cdh412-1"
+
+#Please enter the Hue server port below
+HUE_PORT="8888"
+
+#Plase enter the query to be run in beeswax
+#QUERY="select+*+from+movies+where+name+like+'D%'%3B"
+#QUERY="select+t1.id%2C%0D%0A+t1.name%2C+t2.movie_date%2C+t3.movie_url%2C%0D%0A+CASE+WHEN+t1.id+%3C+500+THEN+%270%27%0D%0A+WHEN+t1.id+%3C+1000+THEN+%27500%27%0D%0A+WHEN+t1.id+%3C+1500+THEN+%271000%27%0D%0A+else+%271500%27+END+as+id_number%0D%0A+from+rc_movies+t1%0D%0A+join+movie2.movie_date+t2+on+%28t1.id+%3D+t2.id%29%0D%0A+join+movie3.movie_url+t3+on+%28t1.id+%3D+t3.id%29%0D%0A+where+t2.movie_date+like+%27%251993%25%27%0D%0A+or+t2.movie_date+like+%27%251994%25%27%3B"
+#QUERY="select+t1.id,+t1.name,+t2.movie_date,+t3.movie_url+from+rc_movies+t1+join+movie2.movie_date+t2+on+(t1.id+=+t2.id)+join+movie3.movie_url+t3+on+(t1.id+=+t3.id)+where+t2.movie_date+like+'%1993%'+or+t2.movie_date+like+'%1994%'%3B"
+
+
+#Please create an entry below for every user that you added to USERS above.  After the = replace "password" with the correct password for each user.  If you have more or less than the number of entries below, add or delete rows as necessary.
+#declare -A password_hash
+#password_hash[cconner]="password"
+#password_hash[test1]="password"
+#password_hash[test2]="password"
+#password_hash[test3]="password"
+#password_hash[test4]="password"
+#password_hash[test5]="password"
+#password_hash[test6]="password"
+#password_hash[test7]="password"
+#password_hash[test8]="password"
+#password_hash[test9]="password"
+#password_hash[test10]="password"
+
+declare -A query_hash
+query_hash[1]="blah"
+
+HUE_PASS_URL="${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_BEES_URL="${HUE_SERVER}:${HUE_PORT}/beeswax/execute"
+HUE_USERS=60
+
+rm -f /tmp/test*.txt
+
+#for user in $USERS
+for ((x=1;x<=${HUE_USERS};x++))
+do
+    user="test$x"
+    echo "Running Command:"
+#    echo "curl -i -c /tmp/${user}_cookie.txt -d \"username=${user}&password=${password_hash[${user}]}\" \"${HUE_PASS_URL}\""
+#    curl -i -c /tmp/${user}_cookie.txt -d "username=${user}&password=${password_hash[${user}]}" "${HUE_PASS_URL}" > /dev/null
+    echo "curl -i -c /tmp/${user}_cookie.txt -d \"username=${user}&password=password\" \"${HUE_PASS_URL}\""
+    curl -i -c /tmp/${user}_cookie.txt -d "username=${user}&password=password" "${HUE_PASS_URL}" > /dev/null
+done
+
+
+while [ 1 ]
+do
+
+#   for ((x=1;x<=${HUE_USERS};x++))
+#   do
+#      user="test$x"
+#      QUERY_CREATE_DB="create+database+if+not+exists+${user}%3B"
+#      echo "running Command:"
+#      echo "curl --data \"query-is_parameterized=on&query-query=${QUERY_CREATE_DB}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute\" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt \"${HUE_BEES_URL}\""
+#      curl --data "query-is_parameterized=on&query-query=${QUERY_CREATE_DB}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt "${HUE_BEES_URL}" 2>&1 > /dev/null
+#      rand_sleep=$((RANDOM%10+1))
+##      sleep ${rand_sleep}
+#      sleep 2
+#   done
+#   for ((x=1;x<=${HUE_USERS};x++))
+#   do
+#      user="test$x"
+#      echo "Gathering Location from /tmp/${user}_headers.txt"
+#      DESIGN=`grep "Location: " /tmp/${user}_headers.txt  | awk '{print $2}'`
+#      RESULTS=""
+#      while [ "$RESULTS" == "" ]
+#      do
+#         sleep 2 
+#         echo "Running Command:"
+#         echo "curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN}"
+#         curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN} 2>&1 > /dev/null
+#         echo "Gathering Location from /tmp/${user}_headers.txt"
+#         RESULTS=`grep "Location: " /tmp/${user}_headers.txt | awk '{print$2}'`
+#      done
+#      echo "Running Command:"
+#      echo "curl -i -b /tmp/${user}_cookie.txt ${RESULTS}"
+#      curl -i -b /tmp/${user}_cookie.txt ${RESULTS} > /tmp/${user}_results.txt
+#    done
+
+#   for ((x=1;x<=${HUE_USERS};x++))
+#   do
+#      user="test$x"
+#      QUERY_DROP="drop+table+if+exists+${user}.students%3B"
+#      echo "running Command:"
+#      echo "curl --data \"query-is_parameterized=on&query-query=${QUERY_DROP}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute\" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt \"${HUE_BEES_URL}\""
+#      curl --data "query-is_parameterized=on&query-query=${QUERY_DROP}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt "${HUE_BEES_URL}" 2>&1 > /dev/null
+#      rand_sleep=$((RANDOM%10+1))
+##      sleep ${rand_sleep}
+#      sleep 2
+#   done
+#   for ((x=1;x<=${HUE_USERS};x++))
+#   do
+#      user="test$x"
+#      echo "Gathering Location from /tmp/${user}_headers.txt"
+#      DESIGN=`grep "Location: " /tmp/${user}_headers.txt  | awk '{print $2}'`
+##      RESULTS=""
+#      while [ "$RESULTS" == "" ]
+#      do
+#         sleep 2 
+#         echo "Running Command:"
+#         echo "curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN}"
+#         curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN} 2>&1 > /dev/null
+#         echo "Gathering Location from /tmp/${user}_headers.txt"
+#         RESULTS=`grep "Location: " /tmp/${user}_headers.txt | awk '{print$2}'`
+#      done
+#      echo "Running Command:"
+#      echo "curl -i -b /tmp/${user}_cookie.txt ${RESULTS}"
+#      curl -i -b /tmp/${user}_cookie.txt ${RESULTS} > /tmp/${user}_results.txt
+#    done
+#
+#   for ((x=1;x<=${HUE_USERS};x++))
+#   do
+#      user="test$x"
+#      QUERY_CREATE_TABLE="create+table+${user}.students(name+String,+age+Int,+gpa+Int,+date1+Timestamp)+ROW+FORMAT+DELIMITED+FIELDS+TERMINATED+BY+','%3B"
+#      echo "running Command:"
+#      echo "curl --data \"query-is_parameterized=on&query-query=${QUERY_CREATE_TABLE}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute\" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt \"${HUE_BEES_URL}\""
+#      curl --data "query-is_parameterized=on&query-query=${QUERY_CREATE_TABLE}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt "${HUE_BEES_URL}" 2>&1 > /dev/null
+#      rand_sleep=$((RANDOM%10+1))
+##      sleep ${rand_sleep}
+#      sleep 2
+#   done
+#   for ((x=1;x<=${HUE_USERS};x++))
+#   do
+#      user="test$x"
+#      echo "Gathering Location from /tmp/${user}_headers.txt"
+#      DESIGN=`grep "Location: " /tmp/${user}_headers.txt  | awk '{print $2}'`
+#      RESULTS=""
+#      while [ "$RESULTS" == "" ]
+#      do
+#         sleep 2 
+#         echo "Running Command:"
+#         echo "curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN}"
+#         curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN} 2>&1 > /dev/null
+#         echo "Gathering Location from /tmp/${user}_headers.txt"
+#         RESULTS=`grep "Location: " /tmp/${user}_headers.txt | awk '{print$2}'`
+#      done
+#      echo "Running Command:"
+#      echo "curl -i -b /tmp/${user}_cookie.txt ${RESULTS}"
+#      curl -i -b /tmp/${user}_cookie.txt ${RESULTS} > /tmp/${user}_results.txt
+#    done
+#
+   for ((x=1;x<=${HUE_USERS};x++))
+   do
+      user="test$x"
+      QUERY_LOAD_DATA="LOAD+DATA+INPATH+'/user/cconner/data/random_names2/blah.txt'+INTO+TABLE+${user}.students%3B"
+      echo "running Command:"
+      echo "curl --data \"query-is_parameterized=on&query-query=${QUERY_LOAD_DATA}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute\" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt \"${HUE_BEES_URL}\""
+      curl --data "query-is_parameterized=on&query-query=${QUERY_LOAD_DATA}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt "${HUE_BEES_URL}" 2>&1 > /dev/null
+      rand_sleep=$((RANDOM%10+1))
+#      sleep ${rand_sleep}
+      sleep 2
+   done
+   for ((x=1;x<=${HUE_USERS};x++))
+   do
+      user="test$x"
+      echo "Gathering Location from /tmp/${user}_headers.txt"
+      DESIGN=`grep "Location: " /tmp/${user}_headers.txt  | awk '{print $2}'`
+      RESULTS=""
+      while [ "$RESULTS" == "" ]
+      do
+         sleep 2 
+         echo "Running Command:"
+         echo "curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN}"
+         curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN} 2>&1 > /dev/null
+         echo "Gathering Location from /tmp/${user}_headers.txt"
+         RESULTS=`grep "Location: " /tmp/${user}_headers.txt | awk '{print$2}'`
+      done
+      echo "Running Command:"
+      echo "curl -i -b /tmp/${user}_cookie.txt ${RESULTS}"
+      curl -i -b /tmp/${user}_cookie.txt ${RESULTS} > /tmp/${user}_results.txt
+    done
+
+   for ((x=1;x<=${HUE_USERS};x++))
+   do
+      user="test$x"
+      QUERY_SELECT="select+count(*)+from+${user}.students1+where+name+like+'D%'%3B"
+      echo "running Command:"
+      echo "curl --data \"query-is_parameterized=on&query-query=${QUERY_SELECT}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute\" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt \"${HUE_BEES_URL}\""
+      curl --data "query-is_parameterized=on&query-query=${QUERY_SELECT}&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&button-submit=Execute" --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt "${HUE_BEES_URL}" 2>&1 > /dev/null
+      rand_sleep=$((RANDOM%10+1))
+#      sleep ${rand_sleep}
+      sleep 2
+   done
+   for ((x=1;x<=${HUE_USERS};x++))
+   do
+      user="test$x"
+      echo "Gathering Location from /tmp/${user}_headers.txt"
+      DESIGN=`grep "Location: " /tmp/${user}_headers.txt  | awk '{print $2}'`
+      RESULTS=""
+      while [ "$RESULTS" == "" ]
+      do
+         sleep 2 
+         echo "Running Command:"
+         echo "curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN}"
+         curl --dump-header /tmp/${user}_headers.txt -i -b /tmp/${user}_cookie.txt ${DESIGN} 2>&1 > /dev/null
+         echo "Gathering Location from /tmp/${user}_headers.txt"
+         RESULTS=`grep "Location: " /tmp/${user}_headers.txt | awk '{print$2}'`
+      done
+      echo "Running Command:"
+      echo "curl -i -b /tmp/${user}_cookie.txt ${RESULTS}"
+      curl -i -b /tmp/${user}_cookie.txt ${RESULTS} > /tmp/${user}_results.txt
+    done
+
+done

+ 43 - 0
tools/ops/hue_beeswax_tester_setup.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#NOTE: This script requires curl to be installed.  Also, it does not have to run on the Hue server, it can run anywhere that curl is installed.  As long as that host can reach the Hue server.
+
+#Please enter the Hue server name below
+HUE_SERVER="cdh412-1"
+
+#Please enter the Hue server port below
+HUE_PORT="8888"
+
+HUE_PASS_URL="${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_USER_URL="${HUE_SERVER}:${HUE_PORT}/useradmin/users/new"
+
+
+echo "curl -i -c /tmp/admin_cookie.txt -d \"username=admin&password=admin\" \"${HUE_PASS_URL}\""
+curl -i -c /tmp/admin_cookie.txt -d "username=admin&password=admin" "${HUE_PASS_URL}" > /dev/null
+
+for x in {1..100}
+do
+
+   user="test$x"
+   echo "running Command:"
+   echo "curl --data \"username=${user}&is_active=on&first_name=${user}&last_name=${user}&email=&ensure_home_directory=on&groups=1&password1=password&password2=password\" --dump-header /tmp/${user}_headers.txt -i -b /tmp/admin_cookie.txt \"${HUE_USER_URL}\""
+   curl --data "username=${user}&is_active=on&first_name=${user}&last_name=${user}&email=&ensure_home_directory=on&groups=1&password1=password&password2=password" --dump-header /tmp/${user}_headers.txt -i -b /tmp/admin_cookie.txt "${HUE_USER_URL}"
+   sudo useradd ${user}
+
+done
+

+ 319 - 115
tools/ops/hue_collect_data.sh

@@ -15,119 +15,300 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#NOTE: This script requires curl, strace and lsof to be installed. It
-# must be run on the Hue server. Set HUE_USER to a user in Hue with
-# Superuser access to get thread dumps.  Set HUE_PASSWORD to HUE_USER's
+# NOTE: This script requires curl, strace and lsof to be installed. It
+# must be run on the Hue server. Set HUE_USER to a user in Hue with 
+# Superuser access to get thread dumps.  Set HUE_PASSWORD to HUE_USER's 
 # password.
 
-# Location for output files
-OUTPUT_DIR=/tmp/hue_collect_data
-
-# Location of Hue logs
-
-HUE_LOG_DIR=/var/log/hue
-
 # Please change USER to contain the user to login
 HUE_USER="admin"
 
 # Please change PASSWORD to contain the password for the above user
 HUE_PASSWORD="admin"
 
-# Set to true for more debug
-VERBOSE="false"
-
-#How many seconds of strace to capture on each run
-STRACE_WAIT=15
+#parse command line arguments
+parse_arguments()
+{
+  # Test that we're using compatible getopt version.
+  getopt -T > /dev/null
+  if [[ $? -ne 4 ]]; then
+    echo "Incompatible getopt version."
+    exit 1
+  fi
 
-#Number of data gathering runs
-RUNS=4
+  # Parse short and long option parameters.
+  STRACE_WAIT=15
+  RUNS=4
+  RUN_WAIT=30
+  OUTPUT_DIR_BASE=/tmp/hue_collect_data
+  COLLECT_STRACE=true
+  COLLECT_LSOF=true
+  COLLECT_NETSTAT=true
+  COLLECT_THREADS=
+  #This is necessary to handle AD auth, doesn't seem to hurt non-ad auth
+  #if they have multiple ldap servers or for some reason the drop down
+  #at login says something other than "LDAP", then this must match the drop
+  #down
+  HUE_AUTH_SERVER="LDAP"
+  COLLECT_INSTANCE=true
+  GETOPT=`getopt -n $0 -o s,l,n,w:,r:,t,u:,p:,i,o:,d:,y:,a:,h \
+      -l strace,,lsof,netstat,wait:,runs:,threads,hueuser:,huepass:,instance,outdir:,huelog:,swait:,authserver:,help \
+      -- "$@"`
+  eval set -- "$GETOPT"
+  while true;
+  do
+    case "$1" in
+    -s|--strace)
+      COLLECT_STRACE=
+      shift
+      ;;
+    -y|--swait)
+      STRACE_WAIT=$2
+      shift 2
+      ;;
+    -l|--lsof)
+      COLLECT_LSOF=
+      shift
+      ;;
+    -n|--netstat)
+      COLLECT_NETSTAT=
+      shift
+      ;;
+    -t|--threads)
+      COLLECT_THREADS=$1
+      shift
+      ;;
+    -u|--hueuser)
+      HUE_USER=$2
+      shift 2
+      ;;
+    -p|--huepass)
+      HUE_PASSWORD=$2
+      shift 2
+      ;;
+    -a|--authserver)
+      HUE_AUTH_SERVER=$2
+      shift 2
+      ;;
+    -i|--instance)
+      COLLECT_INSTANCE=
+      shift
+      ;;
+    -w|--wait)
+      RUN_WAIT=$2
+      shift 2
+      ;;
+    -r|--runs)
+      RUNS=$2
+      shift 2
+      ;;
+    -o|--outdir)
+      OUTPUT_DIR_BASE=$2
+      shift 2
+      ;;
+    -d|--huelog)
+      HUE_LOG_DIR=$2
+      shift 2
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+    esac
+  done
+  #
+}
 
-#Time to wait between each run
-RUN_WAIT=30
+usage()
+{
+cat << EOF
+usage: $0 [options]
+
+Collects troubleshooting data from Hue server:
+
+OPTIONS
+   -s|--strace             Setting this will disable collecting strace, default 
+                           strace will be collected
+   -y|--swait <secs>       Interval to wait before stopping strace - default 15.
+   -l|--lsof               Setting this will disable collecting lsof, default
+                           lsof will be collected
+   -n|--netstat            Setting this will disable collecting netstat, default
+                           netstat will be collected
+   -t|--threads            This will enable thread collection, default threads
+                           will NOT be collected
+   -u|--hueuser		   Hue username for collecting threads - must be admin user - default admin.
+   -p|--huepass		   Hue password for collecting threads - default admin.
+   -a|--authserver         This is the Ldap auth server name in the hue.ini if using
+                           multiple ldap servers for auth.  Must be set to the auth 
+                           server that "hueuser" belongs to.
+   -i|--instance	   Setting this will disable collecting CM Hue configs, default
+                           CM Hue configs will be collected
+   -w|--wait <secs>        Seconds to wait between runs of collecting data - default 30.
+   -r|--runs <numruns>     Number of times to collect data - default 4.
+   -o|--outdir <outdir>    Location to dump collected data - default /tmp/hue_collect_data.
+   -h|--help               Show this message.
+EOF
+}
 
 main()
 {
 
-if [[ ! ${USER} =~ .*root* ]]
-then
-   echo "Script must be run as root: exiting"
-   exit 1
-fi
-
-get_cm_process_dir
-
-#This is necessary to handle AD auth, doesn't seem to hurt non-ad auth
-#if they have multiple ldap servers or for some reason the drop down
-#at login says something other than "LDAP", then this must match the drop
-#down
-HUE_AUTH_SERVER="LDAP"
-DATE=$(date '+%Y%m%d-%H%M')
-OUTPUT_DIR=${OUTPUT_DIR}/${DATE}
-HOSTNAME=$(hostname)
-HUE_SERVER=${HOSTNAME}
-HUE_PORT=`grep http_port ${HUE_INI} | awk -F= '{print $2}'`
-if [[ -z ${HUE_PORT} ]]
-then
-   HUE_PORT=8888
-fi
-SSL_CERT=`grep ssl_certificate ${HUE_INI} | awk -F= '{print $2}'`
-if [[ ! -z ${SSL_CERT} ]]
-then
-   HUE_HTTP="https"
-else
-   HUE_HTTP="http"
-fi
-HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
-HUE_THREADS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/desktop/debug/threads"
-HUE_USAGE_FILE=${OUTPUT_DIR}/cpu_mem_usage
-HUE_THREADS_FILE=${OUTPUT_DIR}/threads
-HUE_STRACE_FILE=${OUTPUT_DIR}/strace
-HUE_LSOF_FILE=${OUTPUT_DIR}/lsof
-COOKIE_JAR=${OUTPUT_DIR}/${USER}_cookie.jar
-
-echo "Making ${OUTPUT_DIR} if it does not exist"
-mkdir -p ${OUTPUT_DIR}
-
-do_ps
-
-hue_login
-echo "Gathering info:"
-for (( x=1; x<=${RUNS}; x++ ))
-do
-   DATE=$(date '+%Y%m%d-%H%M%S')
-   echo "DATE: ${DATE}"
-
-   echo "Getting CPU and Memory usage"
-   do_ps
-   echo "PID CPU MEM MEM_MB" >> ${HUE_USAGE_FILE}_${DATE}
-   echo "${PID} ${CPU} ${MEM} ${MEM_MB}" >> ${HUE_USAGE_FILE}_${DATE}
-   echo "free -m results" >> ${HUE_USAGE_FILE}_${DATE}
-   free -m >> ${HUE_USAGE_FILE}_${DATE}
-
-   echo "Getting strace"
-   do_strace \
-        ${PID} \
-        ${STRACE_WAIT} \
-        -o ${HUE_STRACE_FILE}_${DATE} -T -t
-
-   echo "Getting open connections"
-   do_lsof \
-        ${PID} \
-        ${HUE_LSOF_FILE}_${DATE}
-
-   echo "Getting a thread dump:"
-   do_curl \
-	GET \
-	"${HUE_THREADS_URL}" \
-	-L -o ${HUE_THREADS_FILE}_${DATE}
+   parse_arguments "$@"
 
-   sleep ${RUN_WAIT}
-done
+   if [[ ! ${USER} =~ .*root* ]]
+   then
+      echo "Script must be run as root: exiting"
+      exit 1
+   fi
 
-mkdir ${OUTPUT_DIR}/logs
-cp -pr ${HUE_LOG_DIR}/* ${OUTPUT_DIR}/logs
+   AGENT_PROCESS_DIR="/var/run/cloudera-scm-agent/process"
+   START_DATE=$(date '+%Y%m%d-%H%M')
+   MKDIR="mkdir -p"
+
+   for PID in `ps aux | grep [r]uncherrypyserver | awk '{print $2}'`
+   do
+      if [[ ! ${PID} == ?(-)+([0-9]) ]]
+      then
+        echo "Unable to get PID from Process, either Hue is not running on this host or Hue is not using CherryPy server"
+        exit 1
+      fi
+
+      OUTPUT_DIR_DATE=${OUTPUT_DIR_BASE}/${START_DATE}
+      OUTPUT_DIR=${OUTPUT_DIR_DATE}/${PID}
+      HUE_USAGE_FILE=${OUTPUT_DIR}/cpu_mem_usage/cpu_mem_usage
+      HUE_THREADS_FILE=${OUTPUT_DIR}/threads/threads
+      HUE_STRACE_FILE=${OUTPUT_DIR}/strace/strace
+      HUE_LSOF_FILE=${OUTPUT_DIR}/lsof/lsof
+      HUE_ENVIRON_FILE=${OUTPUT_DIR}/environ/environ
+      HUE_CMDLINE_FILE=${OUTPUT_DIR}/cmdline/cmdline
+      HUE_LIMITS_FILE=${OUTPUT_DIR}/limits/limits
+      HUE_NETSTAT_FILE=${OUTPUT_DIR}/netstat/netstat
+      HUE_SUDO=${OUTPUT_DIR}/sudo
+      HUE_CONFS=${OUTPUT_DIR}/confs
+      COOKIE_JAR=${OUTPUT_DIR}/${USER}_cookie.jar
+
+      HOSTNAME=$(hostname)
+      HUE_SERVER=${HOSTNAME}
+
+      echo "Making ${OUTPUT_DIR} if it does not exist"
+      ${MKDIR} ${OUTPUT_DIR}
+
+      get_cm_process_dir ${PID}
+
+      if [[ -f ${CM_PROPS} ]]
+      then
+         HUE_LOG_DIR=`grep log_dir ${CM_PROPS} | awk '{print $3}'`
+      else
+         HUE_LOG_DIR=/var/log/hue
+      fi
+      HUE_PORT=`grep http_port ${HUE_INI} | awk -F= '{print $2}'`
+      if [[ -z ${HUE_PORT} ]]
+      then
+         HUE_PORT=8888
+      fi
+      SSL_CERT=`grep ssl_certificate ${HUE_INI} | awk -F= '{print $2}'`
+      if [[ ! -z ${SSL_CERT} ]]
+      then
+         HUE_HTTP="https"
+      else
+         HUE_HTTP="http"
+      fi
+      HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+      HUE_THREADS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/desktop/debug/threads"
+   
+      if [[ ! -z ${COLLECT_THREADS} ]]
+      then
+         hue_login
+      fi
+   
+      echo "Gathering info:"
+      for (( x=1; x<=${RUNS}; x++ ))
+      do
+         DATE=$(date '+%Y%m%d-%H%M%S')
+         echo "DATE: ${DATE}"
+      
+         echo "Getting CPU and Memory usage"
+         do_ps ${PID}
+         ${MKDIR} `dirname ${HUE_USAGE_FILE}`
+         echo "PID CPU MEM MEM_MB" >> ${HUE_USAGE_FILE}_${DATE}
+         echo "${PID} ${CPU} ${MEM} ${MEM_MB}" >> ${HUE_USAGE_FILE}_${DATE}
+         echo "free -m results" >> ${HUE_USAGE_FILE}_${DATE}
+         free -m >> ${HUE_USAGE_FILE}_${DATE}
+   
+         if [[ ${COLLECT_NETSTAT} ]]
+         then
+            echo "Gathering netstat info"
+            ${MKDIR} `dirname ${HUE_NETSTAT_FILE}`
+            netstat -anp | grep ${PID} >> ${HUE_NETSTAT_FILE}_${DATE}
+         fi
+
+         if [[ ${COLLECT_STRACE} ]]
+         then
+            echo "Getting strace"
+            ${MKDIR} `dirname ${HUE_STRACE_FILE}`
+            do_strace \
+                 ${PID} \
+                 ${STRACE_WAIT} \
+                 -o ${HUE_STRACE_FILE}_${DATE} -T -t
+         fi
+
+         if [[ ${COLLECT_LSOF} ]]
+         then
+            echo "Getting open connections"
+            ${MKDIR} `dirname ${HUE_LSOF_FILE}`
+            do_lsof \
+                 ${PID} \
+                 ${HUE_LSOF_FILE}_${DATE}
+         fi
+
+         if [[ ! -z ${COLLECT_THREADS} ]]
+         then
+            echo "Getting a thread dump:"
+            ${MKDIR} `dirname ${HUE_THREADS_FILE}`
+            do_curl \
+                 GET \
+                 "${HUE_THREADS_URL}" \
+                 -L -o ${HUE_THREADS_FILE}_${DATE}
+         fi
+
+         sleep ${RUN_WAIT}
+      done
+   
+      if [[ ! -z ${COLLECT_INSTANCE} ]]
+      then
+         echo "Gathering CM config instances"
+         do_instances ${HUE_PORT}
+      fi
+
+      echo "Gathering process info"
+      ${MKDIR} `dirname ${HUE_ENVIRON_FILE}`
+      strings /proc/${PID}/environ >> ${HUE_ENVIRON_FILE}_${DATE}
+      ${MKDIR} `dirname ${HUE_CMDLINE_FILE}`
+      strings /proc/${PID}/cmdline >> ${HUE_CMDLINE_FILE}_${DATE}
+      ${MKDIR} `dirname ${HUE_LIMITS_FILE}`
+      strings /proc/${PID}/limits >> ${HUE_LIMITS_FILE}_${DATE}
+
+      ${MKDIR} ${OUTPUT_DIR}/logs
+      cp -pr ${HUE_LOG_DIR}/* ${OUTPUT_DIR}/logs 
+
+   done
+
+   echo "Gathering environment info"
+   do_sudo \
+        ${HUE_SUDO} \
+        ${DATE}
+
+  if [[ ! -z ${STRACE_PID} ]]
+   then
+      echo "strace still running, waiting for it to complete: ${STRACE_PID}"
+      wait ${STRACE_PID}
+   fi
 
-echo "Collecting done, please zip ${OUTPUT_DIR} and upload to the ticket"
+   echo "Collecting done, please zip ${OUTPUT_DIR_DATE} and upload to the ticket"
 }
 
 function do_curl() {
@@ -193,7 +374,13 @@ function do_strace()
    then
       echo "strace not found, unable to collect strace info"
    else
-      timeout ${WAIT}s ${STRACE} -p ${SPID} ${ARGS} &
+      if [[ ! -z ${STRACE_PID} ]]
+      then
+         echo "strace still running, waiting for it to complete: ${STRACE_PID}"
+         wait ${STRACE_PID}
+      fi
+      timeout ${WAIT}s ${STRACE} -f -v -p ${SPID} ${ARGS} &
+      STRACE_PID=$!
    fi
 }
 
@@ -221,40 +408,57 @@ function do_lsof()
 
 function do_ps()
 {
-   PS_COMMAND=$(ps aux | grep [r]uncherrypyserver | awk '{print $6" "$2" "$3" "$12}')
+   PID=$1
+   shift
+   PS_COMMAND=$(ps aux | grep ${PID} | grep [r]uncherrypyserver | tail -1 | awk '{print $6" "$2" "$3" "$12}')
    MEM=$(echo ${PS_COMMAND} | awk '{print $1}')
-   PID=$(echo ${PS_COMMAND} | awk '{print $2}')
    CPU=$(echo ${PS_COMMAND} | awk '{print $3}')
    PROC=$(echo ${PS_COMMAND} | awk '{print $4}')
-
-   if [[ ! ${PID} == ?(-)+([0-9]) ]]
-   then
-     echo "Unable to get PID from Process, either Hue is not running on this host or Hue is not using CherryPy server"
-     exit 1
-   fi
-
    MEM_MB=$(expr ${MEM} / 1024)
 }
 
+function do_sudo()
+{
+   HUE_SUDO=$1
+   shift
+   DATE=$1
+   shift
+   ARGS=$@
+   sudo -u hue /bin/bash -c "/usr/bin/env" >> ${HUE_SUDO}_env_${DATE}
+   sudo -u hue /bin/bash -c "/usr/bin/env which python2.6" >> ${HUE_SUDO}_python_${DATE}
+}
+
 function get_cm_process_dir()
 {
-   if [[ -d /var/run/cloudera-scm-agent/process ]]
+   PID=$1
+   shift
+   if [[ -d ${AGENT_PROCESS_DIR} ]]
    then
-      HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
+      HUE_CONF_DIR=`strings /proc/${PID}/environ | grep HUE_CONF_DIR | awk -F\= '{print $2}'`
    else
       HUE_CONF_DIR=/etc/hue/conf
    fi
    HUE_INI=${HUE_CONF_DIR}/hue.ini
+   CM_PROPS=${HUE_CONF_DIR}/cloudera-monitor.properties
 }
 
-debug()
+function do_instances()
 {
-
-  if [[ ! -z $VERBOSE ]]
-  then
-    echo "$1" >> ${LOG_FILE}
-  fi
-
+   HUE_PORT=$1
+   ${MKDIR} ${HUE_CONFS}
+   if [[ -d ${AGENT_PROCESS_DIR} ]]
+   then
+      for x in `find ${AGENT_PROCESS_DIR}/*hue-HUE_SERVER -name "hue.ini" -exec grep -H ${HUE_PORT} {} \; | awk -F\/ '{print $6}' | sort -n | tail -3`
+      do
+         cp -pr ${AGENT_PROCESS_DIR}/$x ${HUE_CONFS}
+      done
+   else
+      cp -pr /etc/hue/conf ${HUE_CONFS}/0-hue-HUE_SERVER
+   fi
+   for x in `find ${HUE_CONFS} -name "*" -type f`
+   do
+      sed -i "/password/Id" $x
+   done
 }
 
 main "$@"

+ 76 - 0
tools/ops/hue_file_tester.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This script requires curl to be installed.  Also, it does not have to run on the Hue server, it can run anywhere that curl is installed.  As long as that host can reach the Hue server.
+
+# Please change USERS to contain a list of users that you would like to use to generate load
+USERS="cconner test1 test2 test3 test4 test5 test6 test7 test8 test9 test10"
+
+#Please enter the Hue server name below
+HUE_SERVER="cdh412-1"
+
+#Please enter the Hue server port below
+HUE_PORT="8888"
+
+#Please enter the base name of each file.  They will be called ${HUE_FILE_BASE}${count}.txt.  So hue_file_test_ = hue_file_test_1.txt ....  NOTE: this must match HUE_FILE_BASE in hue_file_tester_setup.sh
+HUE_FILE_BASE="hue_file_test_"
+
+#Plase enter the directory in HDFS where the above files were stored.  NOTE: this must match HDFS_LOCATION in hue_file_tester_setup.sh
+HDFS_LOCATION="/tmp/hue_file_test"
+
+#Please enter the number of files that were created in $HDFS_LOCATION.  NOTE: this must match or be less than the value of FILE_COUNT in hue_file_tester_setup.sh
+FILE_COUNT=10
+
+#Please create an entry below for every user that you added to USERS above.  After the = replace "password" with the correct password for each user.  If you have more or less than the number of entries below, add or delete rows as necessary.
+declare -A password_hash
+password_hash[cconner]="password"
+password_hash[test1]="password"
+password_hash[test2]="password"
+password_hash[test3]="password"
+password_hash[test4]="password"
+password_hash[test5]="password"
+password_hash[test6]="password"
+password_hash[test7]="password"
+password_hash[test8]="password"
+password_hash[test9]="password"
+password_hash[test10]="password"
+
+
+HUE_PASS_URL="${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_FILE_URL="${HUE_SERVER}:${HUE_PORT}/filebrowser/view${HDFS_LOCATION}"
+
+for user in $USERS
+do
+    echo "Running Command:"
+    echo "curl -i -c /tmp/${user}_cookie.txt -d \"username=${user}&password=${password_hash[${user}]}\" \"${HUE_PASS_URL}\""
+    curl -i -c /tmp/${user}_cookie.txt -d "username=${user}&password=${password_hash[${user}]}" "${HUE_PASS_URL}"
+done
+
+while [ 1 ]
+do
+   for user in $USERS
+   do
+      rand_data=$((RANDOM%${FILE_COUNT}+1))
+      rand_sleep=$((RANDOM%10+1))
+      rand_offset=$((RANDOM%32000+1))
+      hue_file="${HUE_FILE_BASE}${rand_data}.txt"
+      echo "Running Command:"
+      echo "curl -i -b /tmp/${user}_cookie.txt \"$HUE_FILE_URL/${hue_file}?offset=${rand_offset}&length=4096&compression=none&mode=text\" > /dev/null"
+      curl -i -b /tmp/${user}_cookie.txt "$HUE_FILE_URL/${hue_file}?offset=${rand_offset}&length=4096&compression=none&mode=text" > /dev/null
+      sleep ${rand_sleep}
+    done
+done

+ 54 - 0
tools/ops/hue_file_tester_setup.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Please enter the base name of each file.  They will be called ${HUE_FILE_BASE}${count}.txt.  So hue_file_test_ = hue_file_test_1.txt ....  NOTE: this must match HUE_FILE_BASE in hue_file_tester.sh
+HUE_FILE_BASE="hue_file_test_"
+
+#Plase enter the directory in HDFS where the above files will be stored.  NOTE: this must match HDFS_LOCATION in hue_file_tester.sh.  I recommend a location in /tmp so that all of the test users can reach the file.  This script will set permissions to 777 on these files.
+HDFS_LOCATION="/tmp/hue_file_test"
+
+#Please enter the number of files that will be created in $HDFS_LOCATION.  NOTE: this must match or be less than the value of FILE_COUNT in hue_file_tester.sh.  This should be a large number to generate realistic load.  I used 300.
+FILE_COUNT=10
+
+#Please enter the temporary local file system location to store the files created above before they are put in HDFS.
+OUTPUT_LOCATION="/tmp/hue_file_test"
+
+#Please specify the size of the files to be created in MB below.  Just enter an integer.  8 will create 8MB files, 50 will create 50MB files.  Larger is better since your issues come from large files.
+FILE_SIZE_MB=8
+
+if [ ! -d ${OUTPUT_LOCATION} ]
+then
+   mkdir -p ${OUTPUT_LOCATION}
+fi
+
+for (( count=1; count<=${FILE_COUNT}; count++ )) 
+do
+   FILE_NAME="${HUE_FILE_BASE}${count}.txt"
+   echo "Running Command:"
+   echo "dd if=/dev/zero of=${OUTPUT_LOCATION}/${FILE_NAME} bs=1024 count=0 seek=$[1024*${FILE_SIZE_MB}]"
+   dd if=/dev/zero of=${OUTPUT_LOCATION}/${FILE_NAME} bs=1024 count=0 seek=$[1024*${FILE_SIZE_MB}]
+done
+
+echo "Running Command:"
+echo "hadoop fs -mkdir ${HDFS_LOCATION}"
+hadoop fs -mkdir ${HDFS_LOCATION}
+echo "Running Command:"
+echo "hadoop fs -put ${OUTPUT_LOCATION}/* ${HDFS_LOCATION}"
+hadoop fs -put ${OUTPUT_LOCATION}/* ${HDFS_LOCATION}
+echo "Running Command:"
+echo "hadoop fs -chmod -R 777 ${HDFS_LOCATION}"
+hadoop fs -chmod -R 777 ${HDFS_LOCATION}

+ 33 - 9
tools/ops/hue_history_cron.sh

@@ -14,11 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
 #Cleans up old oozie workflow and beeswax savedqueries to
 #prevent the DB from getting too large.
 PARCEL_DIR=/opt/cloudera/parcels/CDH
-LOG_FILE=/var/log/hue_history_cron.log
+LOG_FILE=/var/log/hue/`basename "$0" | awk -F\. '{print $1}'`.log
 LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
 LOG_ROTATE_COUNT=2 #number of log files, so 20MB max
 DATE=`date '+%Y%m%d-%H%M'`
@@ -45,18 +45,23 @@ else
    COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
 fi
 
-export CDH_HOME HUE_CONF_DIR COMMAND
+ORACLE_HOME=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2/
+LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
+export CDH_HOME HUE_CONF_DIR ORACLE_HOME LD_LIBRARY_PATH COMMAND
 
-${COMMAND} <<EOF
+${COMMAND} >> /dev/null 2>&1 <<EOF
 from beeswax.models import SavedQuery
 from datetime import date, timedelta
 from oozie.models import Workflow
+from django.db.utils import DatabaseError
 import logging
 import logging.handlers
 import sys
 
 LOGFILE="${LOG_FILE}"
 keepDays = ${KEEP_DAYS}
+deleteRecords = 900
+errorCount = 0
 log = logging.getLogger('')
 log.setLevel(logging.INFO)
 format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -69,15 +74,34 @@ log.info('HUE_CONF_DIR: ${HUE_CONF_DIR}')
 log.info("Cleaning up anything in the Hue tables oozie*, desktop* and beeswax* older than ${KEEP_DAYS} old")
 
 savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))
-log.info("SavedQuery count is: %s" % savedQuerys.count())
-savedQuerys.delete()
-log.info("SavedQuery new count is: %s" % savedQuerys.count())
+totalQuerys = savedQuerys.count()
+loopCount = totalQuerys
+deleteCount = deleteRecords
+log.info("SavedQuerys left: %s" % totalQuerys)
+log.info("Looping through querys")
+while loopCount > 0:
+   if loopCount < deleteCount:
+      deleteCount = loopCount
+   excludeCount = loopCount - deleteCount
+   savedQuerys = SavedQuery.objects.filter(is_auto=True, mtime__lte=date.today() - timedelta(days=keepDays))[:excludeCount].values_list("id", flat=True)
+   try:
+      SavedQuery.objects.exclude(pk__in=list(savedQuerys)).delete()
+      loopCount -= deleteCount
+      errorCount = 0
+      deleteCount = deleteRecords
+   except DatabaseError, e:
+      log.info("Non Fatal Exception: %s: %s" % (e.__class__.__name__, e))
+      errorCount += 1
+      deleteCount = 1
+      if errorCount > 9:
+         raise
+   log.info("querys left: %s" % loopCount)
 
 workflows = Workflow.objects.filter(is_trashed=True, last_modified__lte=date.today() - timedelta(days=keepDays))
 totalWorkflows = workflows.count()
 loopCount = 1
 maxCount = 1000
-log.info("workflows left: %s" % totalWorkflows)
+log.info("Workflows left: %s" % totalWorkflows)
 log.info("Looping through workflows")
 for w in workflows:
    w.delete(skip_trash=True)
@@ -85,6 +109,6 @@ for w in workflows:
    if loopCount == maxCount:
       totalWorkflows = totalWorkflows - maxCount
       loopCount = 1
-      log.info("workflows left: %s" % totalWorkflows)
+      log.info("Workflows left: %s" % totalWorkflows)
 
 EOF

+ 200 - 0
tools/ops/hue_impala_post.sh

@@ -0,0 +1,200 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#parse command line arguments
+parse_arguments()
+{
+  # Test that we're using compatible getopt version.
+  getopt -T > /dev/null
+  if [[ $? -ne 4 ]]; then
+    echo "Incompatible getopt version."
+    exit 1
+  fi
+
+  HUE_AUTH_SERVER="LDAP"
+  GETOPT=`getopt -n $0 -o u:,w:,s:,p:,e,v,h \
+      -l user:,password:,server:,port:,enablessl,verbose,help \
+      -- "$@"`
+  eval set -- "$GETOPT"
+  while true;
+  do
+    case "$1" in
+    -u|--user)
+      HUE_USER=$2
+      shift 2
+      ;;
+    -w|--password)
+      HUE_PASSWORD=$2
+      shift 2
+      ;;
+    -s|--server)
+      HUE_SERVER=$2
+      shift 2
+      ;;
+    -p|--port)
+      HUE_PORT=$2
+      shift 2
+      ;;
+    -e|--enablessl)
+      ENABLESSL=1
+      shift
+      ;;
+    -v|--verbose)
+      VERBOSE=1
+      shift
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+    esac
+  done
+  #
+  if [[ -z ${HUE_USER} ]]
+  then
+    HUE_USER="admin"
+  fi
+  if [[ -z ${HUE_PASSWORD} ]]
+  then
+    HUE_PASSWORD="admin"
+  fi
+  if [[ -z ${HUE_SERVER} ]]
+  then
+    HUE_SERVER="localhost"
+  fi
+  if [[ -z ${HUE_PORT} ]]
+  then
+    HUE_PORT="8888"
+  fi
+
+}
+
+usage()
+{
+cat << EOF
+usage: $0 [options]
+
+Collects troubleshooting data from Hue server:
+
+OPTIONS
+   -u|--user		   Hue username - default admin.
+   -w|--password	   Hue password - default admin.
+   -s|--server	           Hue server host - localhost.
+   -p|--port               Hue server port - 8888.
+   -h|--help               Show this message.
+EOF
+}
+
+main()
+{
+
+   parse_arguments "$@"
+   
+   if [[ -z ${ENABLESSL} ]]
+   then
+      HUE_HTTP="http"
+   else
+      HUE_HTTP="https"
+   fi
+   HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+   HUE_IMPALA_PARAMS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/impala/api/query/parameters"
+   POST_STRING="query-query=drop+table+test1%3B&query-database=default&settings-next_form_id=0&file_resources-next_form_id=0&functions-next_form_id=0&query-email_notify=false&query-is_parameterized=true"
+   
+   hue_login
+   echo "Posting Impala Params:"
+   do_curl \
+        POST \
+        "${HUE_IMPALA_PARAMS_URL}" \
+        -H "content-type:application/x-www-form-urlencoded" -d "${POST_STRING}"
+
+}
+
+function do_curl() {
+
+   METHOD=$1
+   shift
+   URL=$1
+   shift
+   ARGS=$@
+
+   CURL=$(which curl)
+   if [ -z ${COOKIE_JAR} ]
+   then
+      COOKIE_JAR=/tmp/cookie.jar
+   fi
+   if [ -f ${COOKIE_JAR} ]
+   then
+      CSRF_TOKEN=`grep ${HUE_SERVER} ${COOKIE_JAR} | grep csrftoken | cut -f 7`
+   fi
+   if [ ! -f ${CURL} ]
+   then
+      echo "curl not found, unable to run any curl commands"
+   else
+      debug "Connecting to ${URL}"
+      debug "${CURL} \
+         ${CURL_OPTS} \
+         -k \
+         -e \"${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/\" \
+         -b @${COOKIE_JAR} \
+         -c ${COOKIE_JAR} \
+         -H \"X-CSRFToken: ${CSRF_TOKEN}\" \
+         -X ${METHOD} \
+         -f \
+         ${URL} \
+         ${ARGS}"
+
+      ${CURL} \
+         ${CURL_OPTS} \
+         -k \
+         -e "${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/" \
+         -b @${COOKIE_JAR} \
+         -c ${COOKIE_JAR} \
+         -H "X-CSRFToken: ${CSRF_TOKEN}" \
+         -X ${METHOD} \
+         -f \
+         ${URL} \
+         ${ARGS}
+   fi
+
+}
+
+function hue_login() {
+   echo "Login to Hue to get Cookie:"
+   do_curl \
+	GET \
+	"${HUE_PASS_URL}" \
+	-L 2>&1 > /dev/null
+
+   do_curl \
+        POST \
+        "${HUE_PASS_URL}" \
+        -F username=${HUE_USER} -F password="${HUE_PASSWORD}" -F server="${HUE_AUTH_SERVER}" 2>&1 > /dev/null
+}
+
+debug()
+{
+   if [[ ! -z $VERBOSE ]]
+   then
+      echo "$1"
+   fi
+}
+
+main "$@"

+ 375 - 0
tools/ops/hue_ldap_test.sh

@@ -0,0 +1,375 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#This script will grab current Hue configuration for ldap
+#and run ldapsearch commands to validate the config
+#NOTE: This script requires ldapsearch to be installed
+
+#parse command line arguments
+parse_arguments()
+{
+  # Test that we're using compatible getopt version.
+  getopt -T > /dev/null
+  if [[ $? -ne 4 ]]; then
+    echo "Incompatible getopt version."
+    exit 1
+  fi
+
+  # Parse short and long option parameters.
+  OUTPUT_DIR=/tmp/hue_ldap_test
+  TEST_USER=
+  TEST_GROUP=
+  HUE_CONF_DIR=
+  GETOPT=`getopt -n $0 -o o:,u:,g:,c:,v,h \
+      -l outdir:,user:,group:,conf:,verbose,help \
+      -- "$@"`
+  eval set -- "$GETOPT"
+  while true;
+  do
+    case "$1" in
+    -o|--outdir)
+      OUTPUT_DIR=$2
+      shift 2
+      ;;
+    -u|--user)
+      TEST_USER=$2
+      shift 2
+      ;;
+    -g|--group)
+      TEST_GROUP=$2
+      shift 2
+      ;;
+    -c|--conf)
+      HUE_CONF_DIR=$2
+      shift 2
+      ;;
+    -v|--verbose)
+      VERBOSE=1
+      shift
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+    esac
+  done
+  #
+}
+
+usage()
+{
+cat << EOF
+usage: $0 [options]
+
+Tests Hue Server Ldap Config by runnning ldapsearch:
+
+OPTIONS
+   -o|--outdir <outdir>    Location to dump ldap test report - default /tmp/hue_ldap_test.
+   -u|--user <user>        Required: User that exists in ldap to search for
+   -g|--group <group>      Group that exists in ldap to search for - default, does not search for group.
+   -c|--conf <dir>         Custom Hue conf directory with hue.ini for quicker testing - default, hue process dir.
+   -h|--help               Show this message.
+EOF
+}
+
+debug()
+{
+   if [[ ! -z $VERBOSE ]]
+   then
+      echo "$1"
+   fi
+}
+
+report()
+{
+   echo "$1" | tee -a ${REPORT_FILE}
+}
+
+message()
+{
+   case "$1" in
+   "SEARCH_METHOD")
+     MESSAGE="WARN: ldap_username_pattern, nt_domain and search_bind_authentication are exclusive, only one should be set at a time."
+     ;;
+   *)
+     MESSAGE="Unknown message type"
+     ;;
+   esac
+   if [[ -z $2 ]]
+   then
+     echo "${MESSAGE}" | tee -a ${REPORT_FILE}
+   fi
+}
+
+main()
+{
+   parse_arguments "$@"
+
+   LDAPSEARCH=$(which ldapsearch)
+
+   if [[ -z ${TEST_USER} ]]
+   then
+      echo "-u <ldapuser> required"
+      usage
+      exit 1
+   fi
+
+   if [[ ! ${USER} =~ .*root* ]]
+   then
+      echo "Script must be run as root: exiting"
+      exit 1
+   fi
+
+   if [[ ! -f ${LDAPSEARCH} ]]
+   then
+      echo "ldapsearch not found, please install ldapsearch"
+      exit 1
+   else
+      LDAPSEARCH_COMMAND="${LDAPSEARCH} -x -LLL"
+   fi
+
+   AGENT_PROCESS_DIR="/var/run/cloudera-scm-agent/process"
+   PARCEL_DIR=/opt/cloudera/parcels/CDH
+   ORACLE_HOME=/opt/cloudera/parcels/ORACLE_INSTANT_CLIENT/instantclient_11_2/
+   LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ORACLE_HOME}
+
+   if [ ! -d "/usr/lib/hadoop" ]
+   then
+      CDH_HOME=$PARCEL_DIR
+   else
+      CDH_HOME=/usr
+   fi
+
+   if [[ -z ${HUE_CONF_DIR} ]]
+   then
+      if [ -d "${AGENT_PROCESS_DIR}" ]
+      then
+         HUE_CONF_DIR="${AGENT_PROCESS_DIR}/`ls -1 ${AGENT_PROCESS_DIR} | grep HUE | sort -n | tail -1 `"
+      else
+         HUE_CONF_DIR="/etc/hue/conf"
+      fi
+   fi
+   TMP_ENV_FILE=${HUE_CONF_DIR}/hue_tmp_env.sh
+   TMP_PASS_FILE=${HUE_CONF_DIR}/hue_tmp_ldap_pass.txt
+   TMP_FILE=${HUE_CONF_DIR}/hue_tmp.txt
+   touch ${TMP_ENV_FILE}
+   chmod 600 ${TMP_ENV_FILE}
+   touch ${TMP_PASS_FILE}
+   chmod 600 ${TMP_PASS_FILE}
+   mkdir -p ${OUTPUT_DIR}
+   REPORT_FILE=${OUTPUT_DIR}/hue_ldap_report.txt
+   rm -f ${REPORT_FILE}
+
+   if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+   then
+      COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue shell"
+   else
+      COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue shell"
+   fi
+
+   export CDH_HOME HUE_CONF_DIR ORACLE_HOME LD_LIBRARY_PATH COMMAND
+   report "CDH_HOME: ${CDH_HOME}"
+   report "HUE_CONF_DIR: ${HUE_CONF_DIR}"
+   report ""
+
+   ${COMMAND} >> /dev/null 2>&1 <<EOF
+import desktop.conf
+
+def write_property( hue_ldap_conf_file, ldap_config, property_name):
+  if property_name != "bind_password":
+    try:
+      func = getattr(ldap_config, "%s" % (property_name.upper()))
+    except AttributeError:
+      print 'function not found "%s" ()' % (property_name.upper().get())
+    else:
+      property_value=func.get()
+  else:
+    try:
+      property_value = desktop.conf.get_ldap_bind_password(ldap_config)
+    except AttributeError:
+      property_value = ldap_config.BIND_PASSWORD.get()
+  hue_ldap_conf_file.write("%s=\"%s\"\n" % (property_name,property_value))
+  return
+
+hue_ldap_conf_file = open('${TMP_ENV_FILE}', 'w')
+hue_ldap_conf_file.write("#!/bin/bash\n")
+server = None
+ldap_config = desktop.conf.LDAP.LDAP_SERVERS.get()[server] if server else desktop.conf.LDAP
+
+write_property( hue_ldap_conf_file, ldap_config, "ldap_url")
+write_property( hue_ldap_conf_file, ldap_config, "bind_dn")
+write_property( hue_ldap_conf_file, ldap_config, "bind_password")
+write_property( hue_ldap_conf_file, ldap_config, "ldap_cert")
+write_property( hue_ldap_conf_file, ldap_config, "search_bind_authentication")
+write_property( hue_ldap_conf_file, ldap_config, "base_dn")
+write_property( hue_ldap_conf_file, ldap_config, "nt_domain")
+write_property( hue_ldap_conf_file, ldap_config, "use_start_tls")
+write_property( hue_ldap_conf_file, ldap_config, "ldap_username_pattern")
+write_property( hue_ldap_conf_file, ldap_config, "follow_referrals")
+write_property( hue_ldap_conf_file, ldap_config.USERS, "user_filter")
+write_property( hue_ldap_conf_file, ldap_config.USERS, "user_name_attr")
+write_property( hue_ldap_conf_file, ldap_config.GROUPS, "group_filter")
+write_property( hue_ldap_conf_file, ldap_config.GROUPS, "group_name_attr")
+write_property( hue_ldap_conf_file, ldap_config.GROUPS, "group_member_attr")
+EOF
+
+source ${TMP_ENV_FILE}
+
+if [[ -z ${ldap_url} ]]
+then
+   report "Required attribute ldap_url is not set"
+   exit 1
+else
+   LDAPSEARCH_COMMAND="${LDAPSEARCH_COMMAND} -H ${ldap_url}"
+fi
+
+if [[ -z ${ldap_cert} || ${ldap_cert} =~ .*None.* ]]
+then
+   LDAPSEARCH_COMMAND="LDAPTLS_REQCERT=never ${LDAPSEARCH_COMMAND}"
+else
+   LDAPSEARCH_COMMAND="LDAPTLS_REQCERT=ALLOW LDAPTLS_CACERT=${ldap_cert} ${LDAPSEARCH_COMMAND}"
+fi
+
+LDAPSEARCH_COMMAND_NOBASE=${LDAPSEARCH_COMMAND}
+if [[ -z ${base_dn} || ${base_dn} == "None" ]]
+then
+   if [[ -z ${ldap_username_pattern} || ${ldap_username_pattern} == "None" ]]
+   then
+      report "WARN: base_dn is not set and may be required"
+   fi
+else
+   LDAPSEARCH_COMMAND="${LDAPSEARCH_COMMAND} -b \"${base_dn}\""
+fi
+
+LDAPSEARCH_COMMAND_NOAUTH=${LDAPSEARCH_COMMAND}
+if [[ ! -z ${bind_dn} && ${bind_dn} != "None" ]]
+then
+   if [[ -z ${bind_password} || ${bind_password} == "None" ]]
+   then
+      report "WARN: if bind_dn is set, then bind_password is required"
+   fi
+   if [[ ! -z ${nt_domain} && ${nt_domain} != "None" ]]
+   then
+      bind_dn=${bind_dn}@${nt_domain}
+   fi
+   echo -n "${bind_password}" > ${TMP_PASS_FILE}
+   LDAPSEARCH_COMMAND="${LDAPSEARCH_COMMAND} -D ${bind_dn// /\\ } -y ${TMP_PASS_FILE}"
+   LDAPSEARCH_COMMAND_NOBASE="${LDAPSEARCH_COMMAND} -D ${bind_dn// /\\ } -y ${TMP_PASS_FILE}"
+fi
+
+SEARCH_METHOD_FLAG=
+if [[ ${search_bind_authentication} == "True" ]]
+then
+   if [[ ! -z ${nt_domain} && ${nt_domain} != "None" ]]
+   then
+      message "SEARCH_METHOD" ${SEARCH_METHOD_FLAG}
+      SEARCH_METHOD_FLAG=true
+   fi
+   if [[ ! -z ${ldap_username_pattern} && ${ldap_username_pattern} != "None" ]]
+   then
+      message "SEARCH_METHOD" ${SEARCH_METHOD_FLAG}
+      SEARCH_METHOD_FLAG=true
+   fi
+fi
+
+if [[ ! -z ${nt_domain} && ${nt_domain} != "None"  ]]
+then
+   if [[ ${search_bind_authentication} == "True" ]]
+   then
+      message "SEARCH_METHOD" ${SEARCH_METHOD_FLAG}
+      SEARCH_METHOD_FLAG=true
+   fi
+   if [[ ! -z ${ldap_username_pattern} && ${ldap_username_pattern} != "None" ]]
+   then
+      message "SEARCH_METHOD" ${SEARCH_METHOD_FLAG}
+      SEARCH_METHOD_FLAG=true
+   fi
+fi
+
+if [[ ! -z ${ldap_username_pattern} && ${ldap_username_pattern} != "None"  ]]
+then
+   if [[ ${search_bind_authentication} == "True" ]]
+   then
+      message "SEARCH_METHOD" ${SEARCH_METHOD_FLAG}
+      SEARCH_METHOD_FLAG=true
+   fi
+   if [[ ! -z ${nt_domain} && ${nt_domain} != "None" ]]
+   then
+      message "SEARCH_METHOD" ${SEARCH_METHOD_FLAG}
+      SEARCH_METHOD_FLAG=true
+   fi
+fi
+
+USER_FILTER="(&(${user_filter})(${user_name_attr}=${TEST_USER}))"
+GROUP_FILTER="(&(${group_filter})(${group_name_attr}=${TEST_GROUP}))"
+cat ${TMP_ENV_FILE} | grep -v bind_password | grep -v bash >> ${REPORT_FILE}
+report ""
+if [[ ! -z ${ldap_username_pattern} && ${ldap_username_pattern} != "None"  ]]
+then
+   LDAPSEARCH_USER_COMMAND="${LDAPSEARCH_COMMAND_NOBASE} -b \"${ldap_username_pattern//\<username\>/${TEST_USER}}\""
+else
+   LDAPSEARCH_USER_COMMAND="${LDAPSEARCH_COMMAND} '${USER_FILTER}'"
+fi
+report "Running ldapsearch command on user ${TEST_USER}:"
+report "${LDAPSEARCH_USER_COMMAND}"
+eval ${LDAPSEARCH_USER_COMMAND} 2>&1 | grep -vi password | tee ${TMP_FILE}
+USER_BIND_DN=`grep -i "dn:" ${TMP_FILE} | awk -F\: '{print $2}'`
+cat ${TMP_FILE} >> ${REPORT_FILE}
+
+report ""
+LDAPSEARCH_USER_COMMAND="${LDAPSEARCH_COMMAND_NOBASE} '(${user_name_attr}=${TEST_USER})'"
+report "Running ldapsearch command on user ${TEST_USER} without base or filter:"
+report "${LDAPSEARCH_USER_COMMAND}"
+eval ${LDAPSEARCH_USER_COMMAND} 2>&1 | grep -vi password | tee -a ${REPORT_FILE}
+
+report ""
+if [[ ! -z ${TEST_GROUP} ]]
+then
+   LDAPSEARCH_GROUP_COMMAND="${LDAPSEARCH_COMMAND} '${GROUP_FILTER}'"
+   report "Running ldapsearch command on group ${TEST_GROUP}:"
+   report "${LDAPSEARCH_GROUP_COMMAND}"
+   eval ${LDAPSEARCH_GROUP_COMMAND} 2>&1 | tee -a ${REPORT_FILE}
+   report ""
+   LDAPSEARCH_GROUP_COMMAND="${LDAPSEARCH_COMMAND_NOBASE} '(${group_name_attr}=${TEST_GROU})'"
+   report "Running ldapsearch command on group ${TEST_GROUP} without base or filter:"
+   report "${LDAPSEARCH_GROUP_COMMAND}"
+   eval ${LDAPSEARCH_GROUP_COMMAND} 2>&1 | tee -a ${REPORT_FILE}
+fi
+report ""
+
+LDAPSEARCH_ROOT_COMMAND="${LDAPSEARCH_COMMAND_NOBASE} -s base -b ''"
+report "Running ldapsearch command on root dse:"
+report "${LDAPSEARCH_ROOT_COMMAND}"
+eval ${LDAPSEARCH_ROOT_COMMAND} 2>&1 | tee -a ${REPORT_FILE}
+
+LDAPSEARCH_USER_BIND_COMMAND="${LDAPSEARCH_COMMAND_NOAUTH} -D ${USER_BIND_DN// /\\ } -W '${USER_FILTER}' dn ${user_name_attr}"
+report "Running ldapsearch command binding as ${USER_BIND_DN}(${TEST_USER}):"
+report "When prompted please enter ${USER_BIND_DN}'s password:"
+report "${LDAPSEARCH_USER_BIND_COMMAND}"
+eval ${LDAPSEARCH_USER_BIND_COMMAND} 2>&1 | tee -a ${REPORT_FILE}
+
+echo "View ${REPORT_FILE} for more details"
+
+rm -f ${TMP_ENV_FILE} ${TMP_PASS_FILE} ${TMP_FILE}
+
+}
+
+main "$@"

+ 28 - 10
tools/ops/hue_mem_cron.sh

@@ -24,27 +24,44 @@ KILL_ME=5000  #This is the number of MB at which it will kill.
              #Starting with 5000(5gb)
 VERBOSE=true #true then this writes out the proc info each time it runs, leave blank
              #to only write out when we kill the process
-LOG_FILE=/var/log/hue_mem_cron.log
+LOG_FILE=/var/log/hue/`basename "$0" | awk -F\. '{print $1}'`.log
 ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1, we only keep
                #2 log files, so 20MB max
 
 main()
 {
-#  Command to get memory usage.  This will handle multiple Hue instances per server, so
-#  this command grabs the highest memory Hue process at the time of running and
+#  Command to get memory usage.  This will handle multiple Hue instances per server, so 
+#  this command grabs the highest memory Hue process at the time of running and 
 #  kills it.  Then the next run it'll get the next process.
-PS_COMMAND=`ps aux | grep [r]uncherrypyserver | awk '{print $6" "$2" "$3" "$12}'`
-MEM=`echo ${PS_COMMAND} | awk '{print $1}'`
+if [[ ! ${USER} =~ .*root* ]]
+then
+   echo "Script must be run as root: exiting"
+   exit 1
+fi
+
+PS_COMMAND="ps aux | grep [r]uncherrypyserver | awk '{print \$6\" \"\$2\" \"\$3\" \"\$12}'"
+PS_RESULTS=`eval ${PS_COMMAND}`
+if [[ -z ${PS_RESULTS} ]]
+then
+   PS_COMMAND="ps aux | grep [r]unspawningserver | awk '{print \$6\" \"\$2\" \"\$3\" \"\$12}'"
+   PS_RESULTS=`eval ${PS_COMMAND}`
+fi
+if [[ -z ${PS_RESULTS} ]]
+then
+   echo "No Hue processes found: runcherrypyserver or runspawningserver process must be running"
+   exit 1
+fi
+MEM=`echo ${PS_RESULTS} | awk '{print $1}'`
 MEM_MB=`expr ${MEM} / 1024`
-PID=`echo ${PS_COMMAND} | awk '{print $2}'`
-CPU=`echo ${PS_COMMAND} | awk '{print $3}'`
-PROC=`echo ${PS_COMMAND} | awk '{print $4}'`
+PID=`echo ${PS_RESULTS} | awk '{print $2}'`
+CPU=`echo ${PS_RESULTS} | awk '{print $3}'`
+PROC=`echo ${PS_RESULTS} | awk '{print $4}'`
 DATE=`date '+%Y%m%d-%H%M'`
 
 if [[ -f ${LOG_FILE} ]]
 then
    LOG_SIZE=`du -sm ${LOG_FILE} | awk '{print $1}'`
-   if [ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]
+   if [[ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]]
    then
       mv ${LOG_FILE} ${LOG_FILE}.1
    fi
@@ -57,7 +74,8 @@ then
    echo "${DATE} - Killing Hue Process: Too much memory: ${MEM_MB} : PID: ${PID}" >> ${LOG_FILE}
    kill ${PID}
    sleep 30
-   PID2=`ps aux | grep [r]uncherrypyserver | awk '{print $2}'`
+   PS_RESULTS=`eval ${PS_COMMAND}`
+   PID2=`echo ${PS_RESULTS} | awk '{print $2}'`
    if [[ ${PID} == ${PID2} ]]
    then
       kill -9 ${PID}

+ 3 - 3
tools/ops/hue_mem_cron_cm.sh

@@ -26,7 +26,7 @@ KILL_ME=5000  #This is the number of MB at which it will kill.
              #Starting with 5000(5gb)
 VERBOSE=true #true then this writes out the proc info each time it runs, leave blank
              #to only write out when we kill the process
-LOG_FILE=/var/log/hue_mem_cron.log
+LOG_FILE=/var/log/hue/`basename "$0" | awk -F\. '{print $1}'`.log
 ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1, we only keep
                #2 log files, so 20MB max
 TMP_LOCATION=/tmp/hue_mem_cron
@@ -52,7 +52,7 @@ mkdir -p ${TMP_LOCATION}
 if [[ -f ${LOG_FILE} ]]
 then
    LOG_SIZE=`du -sm ${LOG_FILE} | awk '{print $1}'`
-   if [ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]
+   if [[ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]]
    then
       mv ${LOG_FILE} ${LOG_FILE}.1
    fi
@@ -81,7 +81,7 @@ do
       MEM=`echo ${LINE} | awk '{print $3}' | awk -F, '{print $1}'`
       MEM=`printf "%.f" $MEM` # convert from scientific to decimal
       MEM_MB=`expr ${MEM} / ${MB_BYTES}`
-      debug "${DATE} - ROLENAME: ${ROLENAME} - MEM: ${MEM} - MEM_MB: ${MEM_MB}"
+      debug "${DATE} - ROLENAME: ${ROLENAME} - MEM: ${MEM} - MEM_MB: ${MEM_MB}" 
       if [ ${MEM_MB} -gt ${KILL_ME} ]
       then
          echo "${DATE} - Restart the Hue Process: Too much memory: ${MEM_MB} : ROLENAME: ${ROLENAME}" >> ${LOG_FILE}

+ 66 - 0
tools/ops/hue_sync_ldap_groups_cron.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Cron script to sync ldap groups via cron
+PARCEL_DIR=/opt/cloudera/parcels/CDH
+DATE=`date '+%Y%m%d-%H%M'`
+LOG_FILE=/var/log/hue/`basename "$0" | awk -F\. '{print $1}'`.log
+LOG_ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1
+LOG_ROTATE_COUNT=2 #number of log files, so 20MB max
+
+if [[ ! ${USER} =~ .*root* ]]
+then
+   echo "Script must be run as root: exiting"
+   exit 1
+fi
+
+if [[ -f ${LOG_FILE} ]]
+then
+   LOG_SIZE=`du -sm ${LOG_FILE} | awk '{print $1}'`
+   if [[ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]]
+   then
+      mv ${LOG_FILE} ${LOG_FILE}.1
+   fi
+fi
+
+if [ ! -d "/usr/lib/hadoop" ]
+then
+   CDH_HOME=$PARCEL_DIR
+else
+   CDH_HOME=/usr
+fi
+
+if [ -d "/var/run/cloudera-scm-agent/process" ]
+then
+   HUE_CONF_DIR="/var/run/cloudera-scm-agent/process/`ls -1 /var/run/cloudera-scm-agent/process | grep HUE | sort -n | tail -1 `"
+else
+   HUE_CONF_DIR="/etc/hue/conf"
+fi
+
+if [ -d "${CDH_HOME}/lib/hue/build/env/bin" ]
+then
+   COMMAND="${CDH_HOME}/lib/hue/build/env/bin/hue sync_ldap_users_and_groups"
+else
+   COMMAND="${CDH_HOME}/share/hue/build/env/bin/hue sync_ldap_users_and_groups"
+fi
+
+export CDH_HOME HUE_CONF_DIR COMMAND
+
+echo "${DATE} - Syncing Ldap Groups" >> ${LOG_FILE}
+echo "${DATE} - HUE_CONF_DIR: ${HUE_CONF_DIR}" >> ${LOG_FILE}
+cd /tmp
+${COMMAND} >> ${LOG_FILE}

+ 52 - 0
tools/ops/sync_ldap_group.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This script requires curl to be installed.  Also, it does not have to run on the Hue server, it can run anywhere that curl is installed.  As long as that host can reach the Hue server.
+
+# Please change USER to contain the user to login
+USER="cconner"
+
+# Please change PASSWORD to contain the password for the above user
+PASSWORD="password"
+
+#Please enter the Hue server name below
+HUE_SERVER="cdh46-1"
+
+#Please enter the Hue server port below
+HUE_PORT="8888"
+
+#Please enter the group name you want to sync
+GROUP_NAME="admins"
+
+#Please enter off or on based on if you want to import the members of the group
+IMPORT_MEMBERS="on"
+
+#Please enter off or on based on if you want to create home directories for new users
+CREATE_DIRS="on"
+
+POST_STRING="groupname_pattern=${GROUP_NAME}&import_members=${IMPORT_MEMBERS}&ensure_home_directories=${CREATE_DIRS}"
+
+HUE_PASS_URL="${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_LDAP_GROUP_SYNC_URL="${HUE_SERVER}:${HUE_PORT}/useradmin/users/add_ldap_groups"
+
+echo "Running Command:"
+echo "curl -i -c /tmp/${USER}_cookie.txt -d \"username=${USER}&password=${PASSWORD}\" \"${HUE_PASS_URL}\""
+curl -i -c /tmp/${USER}_cookie.txt -d "username=${USER}&password=${PASSWORD}" "${HUE_PASS_URL}"
+
+echo "Running Command:"
+echo "curl -i -b /tmp/${USER}_cookie.txt -d \"${POST_STRING}\" \"${HUE_LDAP_GROUP_SYNC_URL}\" > /dev/null"
+curl -i -b /tmp/${USER}_cookie.txt -d "${POST_STRING}" "${HUE_LDAP_GROUP_SYNC_URL}" > /dev/null

+ 49 - 0
tools/ops/sync_ldap_user.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This script requires curl to be installed.  Also, it does not have to run on the Hue server, it can run anywhere that curl is installed.  As long as that host can reach the Hue server.
+
+# Please change USER to contain the user to login
+USER="cconner"
+
+# Please change PASSWORD to contain the password for the above user
+PASSWORD="password"
+
+#Please enter the Hue server name below
+HUE_SERVER="cdh46-1"
+
+#Please enter the Hue server port below
+HUE_PORT="8888"
+
+#Please enter the user name you want to sync
+USER_NAME="tuser2"
+
+#Please enter off or on based on if you want to create home directories for new users
+CREATE_DIR="on"
+
+POST_STRING="username_pattern=${USER_NAME}&ensure_home_directory=${CREATE_DIR}"
+
+HUE_PASS_URL="${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_LDAP_USER_SYNC_URL="${HUE_SERVER}:${HUE_PORT}/useradmin/users/add_ldap_users"
+
+echo "Running Command:"
+echo "curl -i -c /tmp/${USER}_cookie.txt -d \"username=${USER}&password=${PASSWORD}\" \"${HUE_PASS_URL}\""
+curl -i -c /tmp/${USER}_cookie.txt -d "username=${USER}&password=${PASSWORD}" "${HUE_PASS_URL}"
+
+echo "Running Command:"
+echo "curl -i -b /tmp/${USER}_cookie.txt -d \"${POST_STRING}\" \"${HUE_LDAP_USER_SYNC_URL}\" > /dev/null"
+curl -i -b /tmp/${USER}_cookie.txt -d "${POST_STRING}" "${HUE_LDAP_USER_SYNC_URL}" > /dev/null

+ 50 - 0
tools/ops/sync_ldap_users_groups.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: This script requires curl to be installed.  Also, it does not have to run on the Hue server, it can run anywhere that curl is installed.  As long as that host can reach the Hue server.
+
+# Please change USER to contain the user to login
+USER="cconner"
+
+# Please change PASSWORD to contain the password for the above user
+PASSWORD="password"
+
+#Please enter the Hue server name below
+HUE_SERVER="cdh46-1"
+
+#Please enter the Hue server port below
+HUE_PORT="8888"
+
+#Please enter the group name you want to sync
+GROUP_NAME="admins"
+
+#Please enter off or on based on if you want to import the members of the group
+IMPORT_MEMBERS="on"
+
+#Please enter off or on based on if you want to create home directories for new users
+CREATE_DIRS="on"
+
+HUE_PASS_URL="${HUE_SERVER}:${HUE_PORT}/accounts/login/"
+HUE_LDAP_USER_GROUP_SYNC_URL="${HUE_SERVER}:${HUE_PORT}/useradmin/users/sync_ldap_users_groups"
+
+echo "Running Command:"
+echo "curl -i -c /tmp/${USER}_cookie.txt -d \"username=${USER}&password=${PASSWORD}\" \"${HUE_PASS_URL}\""
+curl -i -c /tmp/${USER}_cookie.txt -d "username=${USER}&password=${PASSWORD}" "${HUE_PASS_URL}"
+
+echo "Running Command:"
+echo "curl -i -b /tmp/${USER}_cookie.txt \"${HUE_LDAP_USER_GROUP_SYNC_URL}\" > /dev/null"
+curl -i -b /tmp/${USER}_cookie.txt "${HUE_LDAP_USER_GROUP_SYNC_URL}" > /dev/null