Browse Source

HUE-8540 [sqoop] Add ability to set default jdbc driver path for any sqoop job

Upload jdbc driver jars to

[indexer]
  # Filesystem directory containing JDBC libs.
  config_jdbc_libs_path=/tmp/jdbc_drivers

which then will automatically add them to any job:

    In editor
    Workflow
    Importer
Romain Rigaux 7 years ago
parent
commit
521e993

+ 5 - 2
desktop/conf.dist/hue.ini

@@ -1377,12 +1377,15 @@
 
 [indexer]
 
-  # Flag to turn on the Morphline Solr indexer.
+  # Flag to turn on the Solr Morphline indexer.
   ## enable_scalable_indexer=true
 
-  # Oozie workspace template for indexing.
+  # Filesystem directory containing Solr Morphline indexing libs.
   ## config_indexer_libs_path=/tmp/smart_indexer_lib
 
+  # Filesystem directory containing JDBC libs.
+  ## config_jdbc_libs_path=/user/oozie/libext/jdbc_drivers
+
   # Flag to turn on Sqoop ingest.
   ## enable_sqoop=true
 

+ 5 - 2
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1379,12 +1379,15 @@
 
 [indexer]
 
-  # Flag to turn on the Morphline Solr indexer.
+  # Flag to turn on the Solr Morphline indexer.
   ## enable_scalable_indexer=true
 
-  # Oozie workspace template for indexing.
+  # Filesystem directory containing Solr Morphline indexing libs.
   ## config_indexer_libs_path=/tmp/smart_indexer_lib
 
+  # Filesystem directory containing JDBC libs.
+  ## config_jdbc_libs_path=/user/oozie/libext/jdbc_drivers
+
   # Flag to turn on Sqoop ingest.
   ## enable_sqoop=true
 

+ 8 - 1
desktop/libs/indexer/src/indexer/conf.py

@@ -84,11 +84,18 @@ ENABLE_SCALABLE_INDEXER = Config(
 
 CONFIG_INDEXER_LIBS_PATH = Config(
   key="config_indexer_libs_path",
-  help=_t("oozie workspace template for indexing:"),
+  help=_t("Filesystem directory containing Solr Morphline indexing libs."),
   type=str,
   default='/tmp/smart_indexer_lib'
 )
 
+CONFIG_JDBC_LIBS_PATH = Config(
+  key="config_jdbc_libs_path",
+  help=_t("Filesystem directory containing JDBC libs."),
+  type=str,
+  default='/user/oozie/libext/jdbc_drivers'
+)
+
 ENABLE_SQOOP = Config(
   key="enable_sqoop",
   help=_t("Flag to turn on Sqoop imports."),

+ 1 - 1
desktop/libs/indexer/src/indexer/indexers/rdbms.py

@@ -173,7 +173,7 @@ def run_sqoop(request, source, destination, start_time):
 
     lib_files = []
     if destination['sqoopJobLibPaths']:
-      lib_files = [{'path': f['path'], 'type': 'jar'} for f in destination['sqoopJobLibPaths']]
+      lib_files = [{'path': f['path'], 'type': 'jar'} for f in destination['sqoopJobLibPaths'] if f['path']]
 
     statement = '--connect jdbc:%(rdbmsType)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password-file %(passwordFilePath)s' % {
       'rdbmsType': get_connector_name(rdbms_name),

+ 9 - 1
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -31,15 +31,16 @@ from desktop.lib.i18n import smart_str
 from desktop.lib.parameterization import find_variables
 from desktop.lib.paths import get_desktop_root
 from desktop.models import Document2
+from indexer.conf import CONFIG_JDBC_LIBS_PATH
 from metadata.conf import ALTUS
 from oozie.utils import convert_to_server_timezone
 
 from hadoop import cluster
 from hadoop.fs.hadoopfs import Hdfs
 
-from liboozie.oozie_api import get_oozie
 from liboozie.conf import REMOTE_DEPLOYMENT_DIR, USE_LIBPATH_FOR_JARS
 from liboozie.credentials import Credentials
+from liboozie.oozie_api import get_oozie
 
 
 LOG = logging.getLogger(__name__)
@@ -333,6 +334,13 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
             hive_site_content = get_hive_site_content()
             if not self.fs.do_as_user(self.user, self.fs.exists, hive_site_lib) and hive_site_content:
               self.fs.do_as_user(self.user, self.fs.create, hive_site_lib, overwrite=True, permission=0700, data=smart_str(hive_site_content))
+          if action.data['type'] in ('sqoop', 'sqoop-document'):
+            if CONFIG_JDBC_LIBS_PATH.get() and CONFIG_JDBC_LIBS_PATH.get() not in self.properties.get('oozie.libpath', ''):
+              LOG.debug("Adding to oozie.libpath %s" % CONFIG_JDBC_LIBS_PATH.get())
+              paths = [CONFIG_JDBC_LIBS_PATH.get()]
+              if self.properties.get('oozie.libpath'):
+                paths.append(self.properties['oozie.libpath'])
+              self.properties['oozie.libpath'] = ','.join(paths)
 
     oozie_xml = self.job.to_xml(self.properties)
     self._do_as(self.user.username, self._copy_files, deployment_dir, oozie_xml, self.properties)