Browse Source

HUE-8554 [core] Support dist Spark installed when running envelope via shell

Romain Rigaux 7 years ago
parent
commit
da58f9d
1 changed files with 1 additions and 0 deletions
  1. 1 0
      desktop/libs/indexer/src/indexer/indexers/envelope.py

+ 1 - 0
desktop/libs/indexer/src/indexer/indexers/envelope.py

@@ -69,6 +69,7 @@ class EnvelopeIndexer(object):
       shell_command_name = "pipeline.sh"
       shell_command = """#!/bin/bash
 
+export SPARK_DIST_CLASSPATH=`hadoop classpath`
 SPARK_KAFKA_VERSION=0.10 spark2-submit envelope.jar envelope.conf"""
       hdfs_shell_cmd_path = os.path.join(workspace_path, shell_command_name)
       self.fs.do_as_user(self.username, self.fs.create, hdfs_shell_cmd_path, data=shell_command)