extract_archive_in_hdfs.sh 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156
  1. #!/usr/bin/env bash
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. set -x
  18. # Linked 6 to stdout
  19. exec 6>&1
  20. ## Close STDOUT file descriptor
  21. #exec 1<&-
  22. ## Close STDERR FD
  23. #exec 2<&-
  24. export LOG_FILE=/tmp/$0.log
  25. # Open STDOUT as $LOG_FILE file for read and write.
  26. exec 1<>$LOG_FILE
  27. # Redirect STDERR to STDOUT
  28. exec 2>&1
  29. env
  30. UPLOAD_PATH=
  31. FILE_NAME=
  32. OUTPUT_PATH=
  33. function usage()
  34. {
  35. echo "Arguments '-u' and '-f' are mandatory."
  36. echo "Usage:"
  37. echo "\t-h --help"
  38. echo "\t[-u | --upload-path]=<PATH_IN_HDFS>"
  39. echo "\t[-f | --file-name]=<FILE_NAME>"
  40. echo ""
  41. }
  42. function cat_exit()
  43. {
  44. STATUS=$1
  45. if [[ -z ${STATUS} ]]
  46. then
  47. STATUS=0
  48. fi
  49. # Restore stdout and close 6
  50. exec 1>&6 6>&-
  51. cat ${LOG_FILE}
  52. exit ${STATUS}
  53. }
  54. while [ "$1" != "" ]; do
  55. PARAM=`echo $1 | awk -F= '{print $1}'`
  56. VALUE=`echo $1 | awk -F= '{print $2}'`
  57. case $PARAM in
  58. -h | --help)
  59. usage
  60. cat_exit
  61. ;;
  62. -u | --upload-path)
  63. UPLOAD_PATH=$VALUE
  64. ;;
  65. -f | --file-name)
  66. FILE_NAME=$VALUE
  67. ;;
  68. -o | --output-path)
  69. OUTPUT_PATH=$VALUE
  70. ;;
  71. *)
  72. echo "ERROR: unknown parameter \"$PARAM\""
  73. usage
  74. cat_exit 1
  75. ;;
  76. esac
  77. shift
  78. done
  79. if [ -z "$UPLOAD_PATH" ] || [ -z "$FILE_NAME" ] || [ -z "$OUTPUT_PATH" ]
  80. then
  81. echo "ERROR: Missing Arguments"
  82. usage
  83. cat_exit 1
  84. fi
  85. exit_status=0
  86. temp_output_dir=`mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir'`
  87. echo 'Created temporary output directory: '$temp_output_dir
  88. if [[ "$FILE_NAME" =~ \.zip$ ]]
  89. then
  90. echo "Unzipping $FILE_NAME"
  91. which unzip
  92. unzip "$FILE_NAME" -d $temp_output_dir
  93. exit_status=$(echo $?)
  94. elif [[ "$FILE_NAME" =~ \.tar\.gz$ ]] || [[ "$FILE_NAME" =~ \.tgz$ ]]
  95. then
  96. echo "Untarring $FILE_NAME"
  97. which tar
  98. tar -xvzf "$FILE_NAME" -C $temp_output_dir
  99. exit_status=$(echo $?)
  100. elif [[ "$FILE_NAME" =~ \.bz2$ ]] || [[ "$FILE_NAME" =~ \.bzip2$ ]]
  101. then
  102. echo "Bunzipping $FILE_NAME"
  103. filename_without_extension=$(echo "$FILE_NAME" | cut -f 1 -d '.')
  104. which bzip2
  105. bzip2 -dc "$FILE_NAME" > $temp_output_dir/"$filename_without_extension"
  106. exit_status=$(echo $?)
  107. else
  108. echo 'ERROR: Could not interpret archive type.'
  109. exit_status=1
  110. fi
  111. echo "Exit status: $exit_status"
  112. echo "Getting extracted file count"
  113. extracted_file_count=$(($(find $temp_output_dir/* -type d -maxdepth 0 | wc -l) + $(find $temp_output_dir/* -type f -maxdepth 0 | wc -l)))
  114. echo "Extracted file count: $extracted_file_count"
  115. if [ $extracted_file_count != 0 ] && [ $exit_status == 0 ]
  116. then
  117. echo "File count > 0 and exit status ==0"
  118. if ! $(hadoop fs -test -d $OUTPUT_PATH)
  119. then
  120. echo "Creating output directory '$OUTPUT_PATH' in HDFS"
  121. hadoop fs -mkdir $OUTPUT_PATH
  122. fi
  123. echo "Copying extracted files to '$OUTPUT_PATH' in HDFS"
  124. hadoop fs -put $temp_output_dir/* "$OUTPUT_PATH"
  125. exit_status=$(echo $?)
  126. if [ $exit_status != 0 ]
  127. then
  128. echo "Failed to copy files to HDFS directory '$OUTPUT_PATH'."
  129. else
  130. echo "Copy to HDFS directory '$OUTPUT_PATH' complete."
  131. fi
  132. else
  133. exit_status=1
  134. fi
  135. echo "Deleting temporary output directory exit status: $exit_status"
  136. rm -rf $temp_output_dir
  137. echo 'Deleted temporary output directory: '$temp_output_dir
  138. cat_exit $exit_status