Răsfoiți Sursa

[sqoop] Support kerberos authentication

Romain Rigaux 10 ani în urmă
părinte
comite
00fcc5d

+ 13 - 12
apps/sqoop/src/sqoop/client/base.py

@@ -15,23 +15,20 @@
 # limitations under the License.
 
 import logging
-import posixpath
-import threading
 import time
 import json
 
-from django.utils.translation import ugettext as _
 
-from desktop.conf import TIME_ZONE
 from desktop.lib.rest.http_client import HttpClient
 
-from link import Link
-from job import Job
-from connector import Connector
-from driver import Driver
-from exception import SqoopException
-from submission import Submission, SqoopSubmissionException
-from resource import SqoopResource
+from sqoop.client.link import Link
+from sqoop.client.job import Job
+from sqoop.client.connector import Connector
+from sqoop.client.driver import Driver
+from sqoop.client.exception import SqoopException
+from sqoop.client.submission import Submission, SqoopSubmissionException
+from sqoop.client.resource import SqoopResource
+from sqoop.sqoop_properties import has_sqoop_has_security
 
 
 LOG = logging.getLogger(__name__)
@@ -53,8 +50,12 @@ class SqoopClient(object):
     self._language = language
     self._username = username
 
+    if has_sqoop_has_security():
+      self._client.set_kerberos_auth()
+    self._security_enabled = has_sqoop_has_security()
+
   def __str__(self):
-    return "SqoopClient at %s" % self._url
+    return "SqoopClient at %s with security %s" % (self._url, self._security_enabled)
 
   @property
   def url(self):

+ 21 - 1
apps/sqoop/src/sqoop/conf.py

@@ -15,12 +15,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from django.utils.translation import ugettext as _, ugettext_lazy as _t
+import os
+
+from django.utils.translation import ugettext_lazy as _t
 
 from desktop.lib.conf import Config
+from hadoop import cluster
+from sqoop.settings import NICE_NAME
 
 
 SERVER_URL = Config(
   key="server_url",
   default='http://localhost:12000/sqoop',
   help=_t("The sqoop server URL."))
+
+SQOOP_CONF_DIR = Config(
+  key="sqoop_conf_dir",
+  default='/etc/sqoop2/conf',
+  help=_t("Path to Sqoop2 configuration directory."))
+
+
+def config_validator(user):
+  res = []
+
+  yarn_cluster = cluster.get_cluster_conf_for_job_submission()
+
+  if yarn_cluster.SECURITY_ENABLED.get() and not os.path.exists(SQOOP_CONF_DIR.get()):
+    res.append((NICE_NAME, _t("The app won't work without a valid %s property.") % SQOOP_CONF_DIR.grab_key))
+
+  return res

+ 65 - 0
apps/sqoop/src/sqoop/sqoop_properties.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import logging
+import os
+
+from sqoop.conf import SQOOP_CONF_DIR
+
+
+LOG = logging.getLogger(__name__)
+
+
+_PROPERTIES_DICT = None
+_CONF_SQOOP_AUTHENTICATION_TYPE = 'org.apache.sqoop.security.authentication.type'
+
+
+def reset():
+  global _PROPERTIES_DICT
+  _PROPERTIES_DICT = None
+
+
+def get_props():
+  if _PROPERTIES_DICT is None:
+    _parse_properties()
+  return _PROPERTIES_DICT
+
+
+def has_sqoop_has_security():
+  return get_props().get(_CONF_SQOOP_AUTHENTICATION_TYPE, 'SIMPLE').upper() == 'KERBEROS'
+
+
+
+def _parse_properties():
+  global _PROPERTIES_DICT
+
+  properties_file = os.path.join(SQOOP_CONF_DIR.get(), 'sqoop.properties')
+  _PROPERTIES_DICT = _parse_site(properties_file)
+
+
+def _parse_site(site_path):
+  try:
+    with open(site_path, 'r') as f:
+      data = f.read()
+  except IOError, err:
+    if err.errno != errno.ENOENT:
+      LOG.error('Cannot read from "%s": %s' % (site_path, err))
+      return
+    data = ""
+
+  return dict([line.split('=', 1) for line in data.split('\n') if '=' in line and not line.startswith('#')])

+ 89 - 0
apps/sqoop/src/sqoop/test_client.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+
+from nose.tools import assert_true, assert_equal, assert_false
+
+from sqoop.conf import SQOOP_CONF_DIR
+from sqoop.client.base import SqoopClient
+from sqoop.sqoop_properties import reset
+
+
+def test_security_plain():
+  tmpdir = tempfile.mkdtemp()
+  finish = SQOOP_CONF_DIR.set_for_testing(tmpdir)
+
+  try:
+    xml = sqoop_properties(authentication='SIMPLE')
+    with file(os.path.join(tmpdir, 'sqoop.properties'), 'w') as f:
+      f.write(xml)
+    reset()
+
+    client = SqoopClient('test.com', 'test')
+    assert_false(client._security_enabled)
+  finally:
+    reset()
+    finish()
+    shutil.rmtree(tmpdir)
+
+
+def test_security_kerberos():
+  tmpdir = tempfile.mkdtemp()
+  finish = SQOOP_CONF_DIR.set_for_testing(tmpdir)
+
+  try:
+    xml = sqoop_properties(authentication='KERBEROS')
+    with file(os.path.join(tmpdir, 'sqoop.properties'), 'w') as f:
+      f.write(xml)
+    reset()
+
+    client = SqoopClient('test.com', 'test')
+    assert_true(client._security_enabled)
+  finally:
+    reset()
+    finish()
+    shutil.rmtree(tmpdir)
+
+
+def sqoop_properties(authentication='SIMPLE'):
+
+  return """
+org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider
+org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED
+org.apache.sqoop.repository.jdbc.maximum.connections=10
+org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler
+org.apache.sqoop.repository.jdbc.url=jdbc:derby:/var/lib/sqoop2/repository/db;create=true
+org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
+org.apache.sqoop.repository.jdbc.create.schema=true
+org.apache.sqoop.repository.jdbc.user=sa
+org.apache.sqoop.repository.jdbc.password=
+org.apache.sqoop.repository.sysprop.derby.stream.error.file=/var/log/sqoop2/derbyrepo.log
+org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine
+org.apache.sqoop.submission.engine.mapreduce.configuration.directory={{CMF_CONF_DIR}}/yarn-conf
+org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine
+org.apache.sqoop.security.authentication.type=%(authentication)s
+org.apache.sqoop.security.authentication.handler=org.apache.sqoop.security.KerberosAuthenticationHandler
+org.apache.sqoop.security.authentication.kerberos.principal=sqoop2/_HOST@VPC.CLOUDERA.COM
+org.apache.sqoop.security.authentication.kerberos.http.principal=HTTP/_HOST@VPC.CLOUDERA.COM
+org.apache.sqoop.security.authentication.kerberos.keytab=sqoop.keytab
+org.apache.sqoop.security.authentication.kerberos.http.keytab=sqoop.keytab
+  """ % {
+    'authentication': authentication,
+  }

+ 4 - 1
desktop/conf.dist/hue.ini

@@ -913,7 +913,7 @@
 
 
 ###########################################################################
-# Settings to configure Sqoop
+# Settings to configure Sqoop2
 ###########################################################################
 
 [sqoop]
@@ -922,6 +922,9 @@
   # Sqoop server URL
   ## server_url=http://localhost:12000/sqoop
 
+  # Path to configuration directory
+  ## sqoop_conf_dir=/etc/sqoop2/conf
+
 
 ###########################################################################
 # Settings to configure Proxy

+ 4 - 1
desktop/conf/pseudo-distributed.ini.tmpl

@@ -920,7 +920,7 @@
 
 
 ###########################################################################
-# Settings to configure Sqoop
+# Settings to configure Sqoop2
 ###########################################################################
 
 [sqoop]
@@ -929,6 +929,9 @@
   # Sqoop server URL
   ## server_url=http://localhost:12000/sqoop
 
+  # Path to configuration directory
+  ## sqoop_conf_dir=/etc/sqoop2/conf
+
 
 ###########################################################################
 # Settings to configure Proxy