hue_file_tester_setup.sh 2.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354
  1. #!/bin/bash
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #Please enter the base name of each file. They will be called ${HUE_FILE_BASE}${count}.txt. So hue_file_test_ = hue_file_test_1.txt .... NOTE: this must match HUE_FILE_BASE in hue_file_tester.sh
  18. HUE_FILE_BASE="hue_file_test_"
  19. #Plase enter the directory in HDFS where the above files will be stored. NOTE: this must match HDFS_LOCATION in hue_file_tester.sh. I recommend a location in /tmp so that all of the test users can reach the file. This script will set permissions to 777 on these files.
  20. HDFS_LOCATION="/tmp/hue_file_test"
  21. #Please enter the number of files that will be created in $HDFS_LOCATION. NOTE: this must match or be less than the value of FILE_COUNT in hue_file_tester.sh. This should be a large number to generate realistic load. I used 300.
  22. FILE_COUNT=10
  23. #Please enter the temporary local file system location to store the files created above before they are put in HDFS.
  24. OUTPUT_LOCATION="/tmp/hue_file_test"
  25. #Please specify the size of the files to be created in MB below. Just enter an integer. 8 will create 8MB files, 50 will create 50MB files. Larger is better since your issues come from large files.
  26. FILE_SIZE_MB=8
  27. if [ ! -d ${OUTPUT_LOCATION} ]
  28. then
  29. mkdir -p ${OUTPUT_LOCATION}
  30. fi
  31. for (( count=1; count<=${FILE_COUNT}; count++ ))
  32. do
  33. FILE_NAME="${HUE_FILE_BASE}${count}.txt"
  34. echo "Running Command:"
  35. echo "dd if=/dev/zero of=${OUTPUT_LOCATION}/${FILE_NAME} bs=1024 count=0 seek=$[1024*${FILE_SIZE_MB}]"
  36. dd if=/dev/zero of=${OUTPUT_LOCATION}/${FILE_NAME} bs=1024 count=0 seek=$[1024*${FILE_SIZE_MB}]
  37. done
  38. echo "Running Command:"
  39. echo "hadoop fs -mkdir ${HDFS_LOCATION}"
  40. hadoop fs -mkdir ${HDFS_LOCATION}
  41. echo "Running Command:"
  42. echo "hadoop fs -put ${OUTPUT_LOCATION}/* ${HDFS_LOCATION}"
  43. hadoop fs -put ${OUTPUT_LOCATION}/* ${HDFS_LOCATION}
  44. echo "Running Command:"
  45. echo "hadoop fs -chmod -R 777 ${HDFS_LOCATION}"
  46. hadoop fs -chmod -R 777 ${HDFS_LOCATION}