conf.py 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. from __future__ import absolute_import
  17. import logging
  18. import os
  19. import re
  20. import boto.utils
  21. from boto.s3.connection import Location
  22. from django.utils.translation import ugettext_lazy as _, ugettext as _t
  23. import aws
  24. from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_bool, coerce_password_from_script
  25. from hadoop.core_site import get_s3a_access_key, get_s3a_secret_key
  26. LOG = logging.getLogger(__name__)
  27. DEFAULT_CALLING_FORMAT = 'boto.s3.connection.OrdinaryCallingFormat'
  28. SUBDOMAIN_ENDPOINT_RE = 's3.(?P<region>[a-z0-9-]+).amazonaws.com'
  29. HYPHEN_ENDPOINT_RE = 's3-(?P<region>[a-z0-9-]+).amazonaws.com'
  30. DUALSTACK_ENDPOINT_RE = 's3.dualstack.(?P<region>[a-z0-9-]+).amazonaws.com'
  31. AWS_ACCOUNT_REGION_DEFAULT = 'us-east-1' # Location.USEast
  32. def get_locations():
  33. return ('EU', # Ireland
  34. 'eu-central-1', # Frankfurt
  35. 'eu-west-1',
  36. 'eu-west-2',
  37. 'eu-west-3',
  38. 'ca-central-1',
  39. 'us-east-1',
  40. 'us-east-2',
  41. 'us-west-1',
  42. 'us-west-2',
  43. 'sa-east-1',
  44. 'ap-northeast-1',
  45. 'ap-northeast-2',
  46. 'ap-northeast-3',
  47. 'ap-southeast-1',
  48. 'ap-southeast-2',
  49. 'ap-south-1',
  50. 'cn-north-1',
  51. 'cn-northwest-1')
  52. def get_default_access_key_id():
  53. """
  54. Attempt to set AWS access key ID from script, else core-site, else None
  55. """
  56. access_key_id_script = AWS_ACCOUNTS['default'].ACCESS_KEY_ID_SCRIPT.get()
  57. return access_key_id_script or get_s3a_access_key()
  58. def get_default_secret_key():
  59. """
  60. Attempt to set AWS secret key from script, else core-site, else None
  61. """
  62. secret_access_key_script = AWS_ACCOUNTS['default'].SECRET_ACCESS_KEY_SCRIPT.get()
  63. return secret_access_key_script or get_s3a_secret_key()
  64. def get_default_region():
  65. region = ''
  66. if 'default' in AWS_ACCOUNTS:
  67. # First check the host/endpoint configuration
  68. if AWS_ACCOUNTS['default'].HOST.get():
  69. endpoint = AWS_ACCOUNTS['default'].HOST.get()
  70. if re.search(SUBDOMAIN_ENDPOINT_RE, endpoint, re.IGNORECASE):
  71. region = re.search(SUBDOMAIN_ENDPOINT_RE, endpoint, re.IGNORECASE).group('region')
  72. elif re.search(HYPHEN_ENDPOINT_RE, endpoint, re.IGNORECASE):
  73. region = re.search(HYPHEN_ENDPOINT_RE, endpoint, re.IGNORECASE).group('region')
  74. elif re.search(DUALSTACK_ENDPOINT_RE, endpoint, re.IGNORECASE):
  75. region = re.search(DUALSTACK_ENDPOINT_RE, endpoint, re.IGNORECASE).group('region')
  76. elif AWS_ACCOUNTS['default'].REGION.get():
  77. region = AWS_ACCOUNTS['default'].REGION.get()
  78. # If the parsed out region is not in the list of supported regions, fallback to the default
  79. if region not in get_locations():
  80. LOG.warn("Region, %s, not found in the list of supported regions: %s" % (region, ', '.join(get_locations())))
  81. region = ''
  82. return region
  83. AWS_ACCOUNTS = UnspecifiedConfigSection(
  84. 'aws_accounts',
  85. help=_('One entry for each AWS account'),
  86. each=ConfigSection(
  87. help=_('Information about single AWS account'),
  88. members=dict(
  89. ACCESS_KEY_ID=Config(
  90. key='access_key_id',
  91. type=str,
  92. dynamic_default=get_default_access_key_id
  93. ),
  94. ACCESS_KEY_ID_SCRIPT=Config(
  95. key='access_key_id_script',
  96. default=None,
  97. private=True,
  98. type=coerce_password_from_script,
  99. help=_("Execute this script to produce the AWS access key ID.")),
  100. SECRET_ACCESS_KEY=Config(
  101. key='secret_access_key',
  102. type=str,
  103. private=True,
  104. dynamic_default=get_default_secret_key
  105. ),
  106. SECRET_ACCESS_KEY_SCRIPT=Config(
  107. key='secret_access_key_script',
  108. default=None,
  109. private=True,
  110. type=coerce_password_from_script,
  111. help=_("Execute this script to produce the AWS secret access key.")
  112. ),
  113. SECURITY_TOKEN=Config(
  114. key='security_token',
  115. type=str,
  116. private=True,
  117. ),
  118. ALLOW_ENVIRONMENT_CREDENTIALS=Config(
  119. help=_('Allow to use environment sources of credentials (environment variables, EC2 profile).'),
  120. key='allow_environment_credentials',
  121. default=True,
  122. type=coerce_bool
  123. ),
  124. REGION=Config(
  125. key='region',
  126. default=AWS_ACCOUNT_REGION_DEFAULT,
  127. type=str
  128. ),
  129. HOST=Config(
  130. help=_('Alternate address for the S3 endpoint.'),
  131. key='host',
  132. default=None,
  133. type=str
  134. ),
  135. PROXY_ADDRESS=Config(
  136. help=_('Proxy address to use for the S3 connection.'),
  137. key='proxy_address',
  138. default=None,
  139. type=str
  140. ),
  141. PROXY_PORT=Config(
  142. help=_('Proxy port to use for the S3 connection.'),
  143. key='proxy_port',
  144. default=8080,
  145. type=int
  146. ),
  147. PROXY_USER=Config(
  148. help=_('Proxy user to use for the S3 connection.'),
  149. key='proxy_user',
  150. default=None,
  151. type=str
  152. ),
  153. PROXY_PASS=Config(
  154. help=_('Proxy password to use for the S3 connection.'),
  155. key='proxy_pass',
  156. default=None,
  157. type=str
  158. ),
  159. CALLING_FORMAT=Config(
  160. key='calling_format',
  161. default=DEFAULT_CALLING_FORMAT,
  162. type=str
  163. ),
  164. IS_SECURE=Config(
  165. key='is_secure',
  166. default=True,
  167. type=coerce_bool
  168. )
  169. )
  170. )
  171. )
  172. def is_enabled():
  173. return ('default' in AWS_ACCOUNTS.keys() and AWS_ACCOUNTS['default'].get_raw() and AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) or has_iam_metadata()
  174. def has_iam_metadata():
  175. try:
  176. # To avoid unnecessary network call, check if Hue is running on EC2 instance
  177. # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
  178. if os.path.exists('/sys/hypervisor/uuid') and open('/sys/hypervisor/uuid', 'read').read()[:3] == 'ec2':
  179. metadata = boto.utils.get_instance_metadata(timeout=1, num_retries=1)
  180. return 'iam' in metadata
  181. except Exception, e:
  182. LOG.exception("Encountered error when checking IAM metadata: %s" % e)
  183. return False
  184. def has_s3_access(user):
  185. from desktop.auth.backend import is_admin
  186. return user.is_authenticated() and user.is_active and (is_admin(user) or user.has_hue_permission(action="s3_access", app="filebrowser"))
  187. def config_validator(user):
  188. res = []
  189. if is_enabled():
  190. try:
  191. conn = aws.get_client('default').get_s3_connection()
  192. conn.get_canonical_user_id()
  193. except Exception, e:
  194. LOG.exception('AWS failed configuration check.')
  195. res.append(('aws', _t('Failed to connect to S3, check your AWS credentials.')))
  196. return res