Browse Source

HUE-5506 [fb] Add script to compress hdfs files

krish 9 years ago
parent
commit
030608176f

+ 1 - 1
apps/filebrowser/src/filebrowser/views.py

@@ -1347,7 +1347,7 @@ def extract_archive_using_batch_job(request):
       try:
         response = extract_archive_in_hdfs(request, upload_path, archive_name)
       except Exception, e:
-        response['message'] = _('Exception occurred while extracting the archive: %s' % e)
+        response['message'] = _('Exception occurred while compressing: %s' % e)
   else:
     response['message'] = _('ERROR: Configuration parameter enable_extract_uploaded_archive ' +
                             'has to be enabled before calling this method.')

+ 78 - 1
desktop/core/src/desktop/lib/tasks/compress_files/compress_in_hdfs.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,3 +14,80 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+UPLOAD_PATH=
+FILE_NAMES=
+
+function usage()
+{
+    echo "Arguments '-u' and '-f' are mandatory."
+    echo "Usage:"
+    echo "\t-h --help"
+    echo "\t[-u | --upload-path]=<PATH_IN_HDFS>"
+    echo "\t[-f | --file-names]=<FILE_NAMES>"
+    echo ""
+}
+
+while [ "$1" != "" ]; do
+    PARAM=`echo $1 | awk -F= '{print $1}'`
+    VALUE=`echo $1 | awk -F= '{print $2}'`
+    case $PARAM in
+        -h | --help)
+            usage
+            exit
+            ;;
+        -u | --upload-path)
+            UPLOAD_PATH=$VALUE
+            ;;
+        -f | --file-names)
+            FILE_NAMES=$VALUE
+            ;;
+        *)
+            echo "ERROR: unknown parameter \"$PARAM\""
+            usage
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+if [ -z $UPLOAD_PATH ] || [ -z $FILE_NAMES ]
+then
+	echo "ERROR: Missing Arguments"
+	usage
+	exit 1
+fi
+
+FILE_NAMES=(${FILE_NAMES//,/ })
+exit_status=0
+
+# output directory inside HDFS upload dir
+filename_without_extension=$(echo $FILE_NAME | cut -f 1 -d '.')
+
+temp_output_dir=`mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir'`
+echo 'Created temporary output directory: '$temp_output_dir
+
+set -x
+zip -r $temp_output_dir/hue_compressed.zip ${FILE_NAMES[@]}
+exit_status=$(echo $?)
+
+set +x
+if [ $exit_status == 0 ]
+then
+	echo "Copying hue_compressed.zip to '$UPLOAD_PATH' in HDFS"
+	hadoop fs -put -f $temp_output_dir/hue_compressed.zip $UPLOAD_PATH
+	exit_status=$(echo $?)
+	if [ $exit_status == 0 ]
+	then
+	    echo "Copy to HDFS directory '$UPLOAD_PATH' complete!!!"
+	else
+	    echo "ERROR: Copy to HDFS directory '$UPLOAD_PATH' FAILED!!!"
+	fi
+else
+	exit_status=1
+fi
+
+rm -rf $temp_output_dir
+echo 'Deleted temporary output directory: '$temp_output_dir
+
+exit $exit_status