utils.py 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091
  1. # Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
  2. # Copyright (c) 2010, Eucalyptus Systems, Inc.
  3. # Copyright (c) 2012 Amazon.com, Inc. or its affiliates.
  4. # All rights reserved.
  5. #
  6. # Permission is hereby granted, free of charge, to any person obtaining a
  7. # copy of this software and associated documentation files (the
  8. # "Software"), to deal in the Software without restriction, including
  9. # without limitation the rights to use, copy, modify, merge, publish, dis-
  10. # tribute, sublicense, and/or sell copies of the Software, and to permit
  11. # persons to whom the Software is furnished to do so, subject to the fol-
  12. # lowing conditions:
  13. #
  14. # The above copyright notice and this permission notice shall be included
  15. # in all copies or substantial portions of the Software.
  16. #
  17. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  18. # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
  19. # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
  20. # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  21. # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23. # IN THE SOFTWARE.
  24. #
  25. # Parts of this code were copied or derived from sample code supplied by AWS.
  26. # The following notice applies to that code.
  27. #
  28. # This software code is made available "AS IS" without warranties of any
  29. # kind. You may copy, display, modify and redistribute the software
  30. # code either by itself or as incorporated into your code; provided that
  31. # you do not remove any proprietary notices. Your use of this software
  32. # code is at your own risk and you waive any claim against Amazon
  33. # Digital Services, Inc. or its affiliates with respect to your use of
  34. # this software code. (c) 2006 Amazon Digital Services, Inc. or its
  35. # affiliates.
  36. """
  37. Some handy utility functions used by several classes.
  38. """
  39. import subprocess
  40. import time
  41. import logging.handlers
  42. import boto
  43. import boto.provider
  44. import tempfile
  45. import random
  46. import smtplib
  47. import datetime
  48. import re
  49. import email.mime.multipart
  50. import email.mime.base
  51. import email.mime.text
  52. import email.utils
  53. import email.encoders
  54. import gzip
  55. import threading
  56. import locale
  57. from boto.compat import six, StringIO, urllib, encodebytes
  58. from contextlib import contextmanager
  59. from hashlib import md5, sha512
  60. _hashfn = sha512
  61. from boto.compat import json
  62. try:
  63. from boto.compat.json import JSONDecodeError
  64. except ImportError:
  65. JSONDecodeError = ValueError
  66. # List of Query String Arguments of Interest
  67. qsa_of_interest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
  68. 'partNumber', 'policy', 'requestPayment', 'torrent',
  69. 'versioning', 'versionId', 'versions', 'website',
  70. 'uploads', 'uploadId', 'response-content-type',
  71. 'response-content-language', 'response-expires',
  72. 'response-cache-control', 'response-content-disposition',
  73. 'response-content-encoding', 'delete', 'lifecycle',
  74. 'tagging', 'restore',
  75. # storageClass is a QSA for buckets in Google Cloud Storage.
  76. # (StorageClass is associated to individual keys in S3, but
  77. # having it listed here should cause no problems because
  78. # GET bucket?storageClass is not part of the S3 API.)
  79. 'storageClass',
  80. # websiteConfig is a QSA for buckets in Google Cloud
  81. # Storage.
  82. 'websiteConfig',
  83. # compose is a QSA for objects in Google Cloud Storage.
  84. 'compose']
  85. _first_cap_regex = re.compile('(.)([A-Z][a-z]+)')
  86. _number_cap_regex = re.compile('([a-z])([0-9]+)')
  87. _end_cap_regex = re.compile('([a-z0-9])([A-Z])')
  88. def unquote_v(nv):
  89. if len(nv) == 1:
  90. return nv
  91. else:
  92. return (nv[0], urllib.parse.unquote(nv[1]))
  93. def canonical_string(method, path, headers, expires=None,
  94. provider=None):
  95. """
  96. Generates the aws canonical string for the given parameters
  97. """
  98. if not provider:
  99. provider = boto.provider.get_default()
  100. interesting_headers = {}
  101. for key in headers:
  102. lk = key.lower()
  103. if headers[key] is not None and \
  104. (lk in ['content-md5', 'content-type', 'date'] or
  105. lk.startswith(provider.header_prefix)):
  106. interesting_headers[lk] = str(headers[key]).strip()
  107. # these keys get empty strings if they don't exist
  108. if 'content-type' not in interesting_headers:
  109. interesting_headers['content-type'] = ''
  110. if 'content-md5' not in interesting_headers:
  111. interesting_headers['content-md5'] = ''
  112. # just in case someone used this. it's not necessary in this lib.
  113. if provider.date_header in interesting_headers:
  114. interesting_headers['date'] = ''
  115. # if you're using expires for query string auth, then it trumps date
  116. # (and provider.date_header)
  117. if expires:
  118. interesting_headers['date'] = str(expires)
  119. sorted_header_keys = sorted(interesting_headers.keys())
  120. buf = "%s\n" % method
  121. for key in sorted_header_keys:
  122. val = interesting_headers[key]
  123. if key.startswith(provider.header_prefix):
  124. buf += "%s:%s\n" % (key, val)
  125. else:
  126. buf += "%s\n" % val
  127. # don't include anything after the first ? in the resource...
  128. # unless it is one of the QSA of interest, defined above
  129. t = path.split('?')
  130. buf += t[0]
  131. if len(t) > 1:
  132. qsa = t[1].split('&')
  133. qsa = [a.split('=', 1) for a in qsa]
  134. qsa = [unquote_v(a) for a in qsa if a[0] in qsa_of_interest]
  135. if len(qsa) > 0:
  136. qsa.sort(key=lambda x: x[0])
  137. qsa = ['='.join(a) for a in qsa]
  138. buf += '?'
  139. buf += '&'.join(qsa)
  140. return buf
  141. def merge_meta(headers, metadata, provider=None):
  142. if not provider:
  143. provider = boto.provider.get_default()
  144. metadata_prefix = provider.metadata_prefix
  145. final_headers = headers.copy()
  146. for k in metadata.keys():
  147. if k.lower() in boto.s3.key.Key.base_user_settable_fields:
  148. final_headers[k] = metadata[k]
  149. else:
  150. final_headers[metadata_prefix + k] = metadata[k]
  151. return final_headers
  152. def get_aws_metadata(headers, provider=None):
  153. if not provider:
  154. provider = boto.provider.get_default()
  155. metadata_prefix = provider.metadata_prefix
  156. metadata = {}
  157. for hkey in headers.keys():
  158. if hkey.lower().startswith(metadata_prefix):
  159. val = urllib.parse.unquote(headers[hkey])
  160. if isinstance(val, bytes):
  161. try:
  162. val = val.decode('utf-8')
  163. except UnicodeDecodeError:
  164. # Just leave the value as-is
  165. pass
  166. metadata[hkey[len(metadata_prefix):]] = val
  167. del headers[hkey]
  168. return metadata
  169. def retry_url(url, retry_on_404=True, num_retries=10, timeout=None):
  170. """
  171. Retry a url. This is specifically used for accessing the metadata
  172. service on an instance. Since this address should never be proxied
  173. (for security reasons), we create a ProxyHandler with a NULL
  174. dictionary to override any proxy settings in the environment.
  175. """
  176. for i in range(0, num_retries):
  177. try:
  178. proxy_handler = urllib.request.ProxyHandler({})
  179. opener = urllib.request.build_opener(proxy_handler)
  180. req = urllib.request.Request(url)
  181. r = opener.open(req, timeout=timeout)
  182. result = r.read()
  183. if(not isinstance(result, six.string_types) and
  184. hasattr(result, 'decode')):
  185. result = result.decode('utf-8')
  186. return result
  187. except urllib.error.HTTPError as e:
  188. code = e.getcode()
  189. if code == 404 and not retry_on_404:
  190. return ''
  191. except Exception as e:
  192. boto.log.exception('Caught exception reading instance data')
  193. # If not on the last iteration of the loop then sleep.
  194. if i + 1 != num_retries:
  195. boto.log.debug('Sleeping before retrying')
  196. time.sleep(min(2 ** i,
  197. boto.config.get('Boto', 'max_retry_delay', 60)))
  198. boto.log.error('Unable to read instance data, giving up')
  199. return ''
  200. def _get_instance_metadata(url, num_retries, timeout=None):
  201. return LazyLoadMetadata(url, num_retries, timeout)
  202. class LazyLoadMetadata(dict):
  203. def __init__(self, url, num_retries, timeout=None):
  204. self._url = url
  205. self._num_retries = num_retries
  206. self._leaves = {}
  207. self._dicts = []
  208. self._timeout = timeout
  209. data = boto.utils.retry_url(self._url, num_retries=self._num_retries, timeout=self._timeout)
  210. if data:
  211. fields = data.split('\n')
  212. for field in fields:
  213. if field.endswith('/'):
  214. key = field[0:-1]
  215. self._dicts.append(key)
  216. else:
  217. p = field.find('=')
  218. if p > 0:
  219. key = field[p + 1:]
  220. resource = field[0:p] + '/openssh-key'
  221. else:
  222. key = resource = field
  223. self._leaves[key] = resource
  224. self[key] = None
  225. def _materialize(self):
  226. for key in self:
  227. self[key]
  228. def __getitem__(self, key):
  229. if key not in self:
  230. # allow dict to throw the KeyError
  231. return super(LazyLoadMetadata, self).__getitem__(key)
  232. # already loaded
  233. val = super(LazyLoadMetadata, self).__getitem__(key)
  234. if val is not None:
  235. return val
  236. if key in self._leaves:
  237. resource = self._leaves[key]
  238. last_exception = None
  239. for i in range(0, self._num_retries):
  240. try:
  241. val = boto.utils.retry_url(
  242. self._url + urllib.parse.quote(resource,
  243. safe="/:"),
  244. num_retries=self._num_retries,
  245. timeout=self._timeout)
  246. if val and val[0] == '{':
  247. val = json.loads(val)
  248. break
  249. else:
  250. p = val.find('\n')
  251. if p > 0:
  252. val = val.split('\n')
  253. break
  254. except JSONDecodeError as e:
  255. boto.log.debug(
  256. "encountered '%s' exception: %s" % (
  257. e.__class__.__name__, e))
  258. boto.log.debug(
  259. 'corrupted JSON data found: %s' % val)
  260. last_exception = e
  261. except Exception as e:
  262. boto.log.debug("encountered unretryable" +
  263. " '%s' exception, re-raising" % (
  264. e.__class__.__name__))
  265. last_exception = e
  266. raise
  267. boto.log.error("Caught exception reading meta data" +
  268. " for the '%s' try" % (i + 1))
  269. if i + 1 != self._num_retries:
  270. next_sleep = min(
  271. random.random() * 2 ** i,
  272. boto.config.get('Boto', 'max_retry_delay', 60))
  273. time.sleep(next_sleep)
  274. else:
  275. boto.log.error('Unable to read meta data, giving up')
  276. boto.log.error(
  277. "encountered '%s' exception: %s" % (
  278. last_exception.__class__.__name__, last_exception))
  279. raise last_exception
  280. self[key] = val
  281. elif key in self._dicts:
  282. self[key] = LazyLoadMetadata(self._url + key + '/',
  283. self._num_retries)
  284. return super(LazyLoadMetadata, self).__getitem__(key)
  285. def get(self, key, default=None):
  286. try:
  287. return self[key]
  288. except KeyError:
  289. return default
  290. def values(self):
  291. self._materialize()
  292. return super(LazyLoadMetadata, self).values()
  293. def items(self):
  294. self._materialize()
  295. return super(LazyLoadMetadata, self).items()
  296. def __str__(self):
  297. self._materialize()
  298. return super(LazyLoadMetadata, self).__str__()
  299. def __repr__(self):
  300. self._materialize()
  301. return super(LazyLoadMetadata, self).__repr__()
  302. def _build_instance_metadata_url(url, version, path):
  303. """
  304. Builds an EC2 metadata URL for fetching information about an instance.
  305. Example:
  306. >>> _build_instance_metadata_url('http://169.254.169.254', 'latest', 'meta-data/')
  307. http://169.254.169.254/latest/meta-data/
  308. :type url: string
  309. :param url: URL to metadata service, e.g. 'http://169.254.169.254'
  310. :type version: string
  311. :param version: Version of the metadata to get, e.g. 'latest'
  312. :type path: string
  313. :param path: Path of the metadata to get, e.g. 'meta-data/'. If a trailing
  314. slash is required it must be passed in with the path.
  315. :return: The full metadata URL
  316. """
  317. return '%s/%s/%s' % (url, version, path)
  318. def get_instance_metadata(version='latest', url='http://169.254.169.254',
  319. data='meta-data/', timeout=None, num_retries=5):
  320. """
  321. Returns the instance metadata as a nested Python dictionary.
  322. Simple values (e.g. local_hostname, hostname, etc.) will be
  323. stored as string values. Values such as ancestor-ami-ids will
  324. be stored in the dict as a list of string values. More complex
  325. fields such as public-keys and will be stored as nested dicts.
  326. If the timeout is specified, the connection to the specified url
  327. will time out after the specified number of seconds.
  328. """
  329. try:
  330. metadata_url = _build_instance_metadata_url(url, version, data)
  331. return _get_instance_metadata(metadata_url, num_retries=num_retries, timeout=timeout)
  332. except urllib.error.URLError:
  333. boto.log.exception("Exception caught when trying to retrieve "
  334. "instance metadata for: %s", data)
  335. return None
  336. def get_instance_identity(version='latest', url='http://169.254.169.254',
  337. timeout=None, num_retries=5):
  338. """
  339. Returns the instance identity as a nested Python dictionary.
  340. """
  341. iid = {}
  342. base_url = _build_instance_metadata_url(url, version,
  343. 'dynamic/instance-identity/')
  344. try:
  345. data = retry_url(base_url, num_retries=num_retries, timeout=timeout)
  346. fields = data.split('\n')
  347. for field in fields:
  348. val = retry_url(base_url + '/' + field + '/', num_retries=num_retries, timeout=timeout)
  349. if val[0] == '{':
  350. val = json.loads(val)
  351. if field:
  352. iid[field] = val
  353. return iid
  354. except urllib.error.URLError:
  355. return None
  356. def get_instance_userdata(version='latest', sep=None,
  357. url='http://169.254.169.254', timeout=None, num_retries=5):
  358. ud_url = _build_instance_metadata_url(url, version, 'user-data')
  359. user_data = retry_url(ud_url, retry_on_404=False, num_retries=num_retries, timeout=timeout)
  360. if user_data:
  361. if sep:
  362. l = user_data.split(sep)
  363. user_data = {}
  364. for nvpair in l:
  365. t = nvpair.split('=')
  366. user_data[t[0].strip()] = t[1].strip()
  367. return user_data
  368. ISO8601 = '%Y-%m-%dT%H:%M:%SZ'
  369. ISO8601_MS = '%Y-%m-%dT%H:%M:%S.%fZ'
  370. RFC1123 = '%a, %d %b %Y %H:%M:%S %Z'
  371. LOCALE_LOCK = threading.Lock()
  372. @contextmanager
  373. def setlocale(name):
  374. """
  375. A context manager to set the locale in a threadsafe manner.
  376. """
  377. with LOCALE_LOCK:
  378. saved = locale.setlocale(locale.LC_ALL)
  379. try:
  380. yield locale.setlocale(locale.LC_ALL, name)
  381. finally:
  382. locale.setlocale(locale.LC_ALL, saved)
  383. def get_ts(ts=None):
  384. if not ts:
  385. ts = time.gmtime()
  386. return time.strftime(ISO8601, ts)
  387. def parse_ts(ts):
  388. with setlocale('C'):
  389. ts = ts.strip()
  390. try:
  391. dt = datetime.datetime.strptime(ts, ISO8601)
  392. return dt
  393. except ValueError:
  394. try:
  395. dt = datetime.datetime.strptime(ts, ISO8601_MS)
  396. return dt
  397. except ValueError:
  398. dt = datetime.datetime.strptime(ts, RFC1123)
  399. return dt
  400. def find_class(module_name, class_name=None):
  401. if class_name:
  402. module_name = "%s.%s" % (module_name, class_name)
  403. modules = module_name.split('.')
  404. c = None
  405. try:
  406. for m in modules[1:]:
  407. if c:
  408. c = getattr(c, m)
  409. else:
  410. c = getattr(__import__(".".join(modules[0:-1])), m)
  411. return c
  412. except:
  413. return None
  414. def update_dme(username, password, dme_id, ip_address):
  415. """
  416. Update your Dynamic DNS record with DNSMadeEasy.com
  417. """
  418. dme_url = 'https://www.dnsmadeeasy.com/servlet/updateip'
  419. dme_url += '?username=%s&password=%s&id=%s&ip=%s'
  420. s = urllib.request.urlopen(dme_url % (username, password, dme_id, ip_address))
  421. return s.read()
  422. def fetch_file(uri, file=None, username=None, password=None):
  423. """
  424. Fetch a file based on the URI provided.
  425. If you do not pass in a file pointer a tempfile.NamedTemporaryFile,
  426. or None if the file could not be retrieved is returned.
  427. The URI can be either an HTTP url, or "s3://bucket_name/key_name"
  428. """
  429. boto.log.info('Fetching %s' % uri)
  430. if file is None:
  431. file = tempfile.NamedTemporaryFile()
  432. try:
  433. if uri.startswith('s3://'):
  434. bucket_name, key_name = uri[len('s3://'):].split('/', 1)
  435. c = boto.connect_s3(aws_access_key_id=username,
  436. aws_secret_access_key=password)
  437. bucket = c.get_bucket(bucket_name)
  438. key = bucket.get_key(key_name)
  439. key.get_contents_to_file(file)
  440. else:
  441. if username and password:
  442. passman = urllib.request.HTTPPasswordMgrWithDefaultRealm()
  443. passman.add_password(None, uri, username, password)
  444. authhandler = urllib.request.HTTPBasicAuthHandler(passman)
  445. opener = urllib.request.build_opener(authhandler)
  446. urllib.request.install_opener(opener)
  447. s = urllib.request.urlopen(uri)
  448. file.write(s.read())
  449. file.seek(0)
  450. except:
  451. raise
  452. boto.log.exception('Problem Retrieving file: %s' % uri)
  453. file = None
  454. return file
  455. class ShellCommand(object):
  456. def __init__(self, command, wait=True, fail_fast=False, cwd=None):
  457. self.exit_code = 0
  458. self.command = command
  459. self.log_fp = StringIO()
  460. self.wait = wait
  461. self.fail_fast = fail_fast
  462. self.run(cwd=cwd)
  463. def run(self, cwd=None):
  464. boto.log.info('running:%s' % self.command)
  465. self.process = subprocess.Popen(self.command, shell=True,
  466. stdin=subprocess.PIPE,
  467. stdout=subprocess.PIPE,
  468. stderr=subprocess.PIPE,
  469. cwd=cwd)
  470. if(self.wait):
  471. while self.process.poll() is None:
  472. time.sleep(1)
  473. t = self.process.communicate()
  474. self.log_fp.write(t[0])
  475. self.log_fp.write(t[1])
  476. boto.log.info(self.log_fp.getvalue())
  477. self.exit_code = self.process.returncode
  478. if self.fail_fast and self.exit_code != 0:
  479. raise Exception("Command " + self.command +
  480. " failed with status " + self.exit_code)
  481. return self.exit_code
  482. def setReadOnly(self, value):
  483. raise AttributeError
  484. def getStatus(self):
  485. return self.exit_code
  486. status = property(getStatus, setReadOnly, None,
  487. 'The exit code for the command')
  488. def getOutput(self):
  489. return self.log_fp.getvalue()
  490. output = property(getOutput, setReadOnly, None,
  491. 'The STDIN and STDERR output of the command')
  492. class AuthSMTPHandler(logging.handlers.SMTPHandler):
  493. """
  494. This class extends the SMTPHandler in the standard Python logging module
  495. to accept a username and password on the constructor and to then use those
  496. credentials to authenticate with the SMTP server. To use this, you could
  497. add something like this in your boto config file:
  498. [handler_hand07]
  499. class=boto.utils.AuthSMTPHandler
  500. level=WARN
  501. formatter=form07
  502. args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject')
  503. """
  504. def __init__(self, mailhost, username, password,
  505. fromaddr, toaddrs, subject):
  506. """
  507. Initialize the handler.
  508. We have extended the constructor to accept a username/password
  509. for SMTP authentication.
  510. """
  511. super(AuthSMTPHandler, self).__init__(mailhost, fromaddr,
  512. toaddrs, subject)
  513. self.username = username
  514. self.password = password
  515. def emit(self, record):
  516. """
  517. Emit a record.
  518. Format the record and send it to the specified addressees.
  519. It would be really nice if I could add authorization to this class
  520. without having to resort to cut and paste inheritance but, no.
  521. """
  522. try:
  523. port = self.mailport
  524. if not port:
  525. port = smtplib.SMTP_PORT
  526. smtp = smtplib.SMTP(self.mailhost, port)
  527. smtp.login(self.username, self.password)
  528. msg = self.format(record)
  529. msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % (
  530. self.fromaddr,
  531. ','.join(self.toaddrs),
  532. self.getSubject(record),
  533. email.utils.formatdate(), msg)
  534. smtp.sendmail(self.fromaddr, self.toaddrs, msg)
  535. smtp.quit()
  536. except (KeyboardInterrupt, SystemExit):
  537. raise
  538. except:
  539. self.handleError(record)
  540. class LRUCache(dict):
  541. """A dictionary-like object that stores only a certain number of items, and
  542. discards its least recently used item when full.
  543. >>> cache = LRUCache(3)
  544. >>> cache['A'] = 0
  545. >>> cache['B'] = 1
  546. >>> cache['C'] = 2
  547. >>> len(cache)
  548. 3
  549. >>> cache['A']
  550. 0
  551. Adding new items to the cache does not increase its size. Instead, the least
  552. recently used item is dropped:
  553. >>> cache['D'] = 3
  554. >>> len(cache)
  555. 3
  556. >>> 'B' in cache
  557. False
  558. Iterating over the cache returns the keys, starting with the most recently
  559. used:
  560. >>> for key in cache:
  561. ... print key
  562. D
  563. A
  564. C
  565. This code is based on the LRUCache class from Genshi which is based on
  566. `Myghty <http://www.myghty.org>`_'s LRUCache from ``myghtyutils.util``,
  567. written by Mike Bayer and released under the MIT license (Genshi uses the
  568. BSD License).
  569. """
  570. class _Item(object):
  571. def __init__(self, key, value):
  572. self.previous = self.next = None
  573. self.key = key
  574. self.value = value
  575. def __repr__(self):
  576. return repr(self.value)
  577. def __init__(self, capacity):
  578. self._dict = dict()
  579. self.capacity = capacity
  580. self.head = None
  581. self.tail = None
  582. def __contains__(self, key):
  583. return key in self._dict
  584. def __iter__(self):
  585. cur = self.head
  586. while cur:
  587. yield cur.key
  588. cur = cur.next
  589. def __len__(self):
  590. return len(self._dict)
  591. def __getitem__(self, key):
  592. item = self._dict[key]
  593. self._update_item(item)
  594. return item.value
  595. def __setitem__(self, key, value):
  596. item = self._dict.get(key)
  597. if item is None:
  598. item = self._Item(key, value)
  599. self._dict[key] = item
  600. self._insert_item(item)
  601. else:
  602. item.value = value
  603. self._update_item(item)
  604. self._manage_size()
  605. def __repr__(self):
  606. return repr(self._dict)
  607. def _insert_item(self, item):
  608. item.previous = None
  609. item.next = self.head
  610. if self.head is not None:
  611. self.head.previous = item
  612. else:
  613. self.tail = item
  614. self.head = item
  615. self._manage_size()
  616. def _manage_size(self):
  617. while len(self._dict) > self.capacity:
  618. del self._dict[self.tail.key]
  619. if self.tail != self.head:
  620. self.tail = self.tail.previous
  621. self.tail.next = None
  622. else:
  623. self.head = self.tail = None
  624. def _update_item(self, item):
  625. if self.head == item:
  626. return
  627. previous = item.previous
  628. previous.next = item.next
  629. if item.next is not None:
  630. item.next.previous = previous
  631. else:
  632. self.tail = previous
  633. item.previous = None
  634. item.next = self.head
  635. self.head.previous = self.head = item
  636. class Password(object):
  637. """
  638. Password object that stores itself as hashed.
  639. Hash defaults to SHA512 if available, MD5 otherwise.
  640. """
  641. hashfunc = _hashfn
  642. def __init__(self, str=None, hashfunc=None):
  643. """
  644. Load the string from an initial value, this should be the
  645. raw hashed password.
  646. """
  647. self.str = str
  648. if hashfunc:
  649. self.hashfunc = hashfunc
  650. def set(self, value):
  651. if not isinstance(value, bytes):
  652. value = value.encode('utf-8')
  653. self.str = self.hashfunc(value).hexdigest()
  654. def __str__(self):
  655. return str(self.str)
  656. def __eq__(self, other):
  657. if other is None:
  658. return False
  659. if not isinstance(other, bytes):
  660. other = other.encode('utf-8')
  661. return str(self.hashfunc(other).hexdigest()) == str(self.str)
  662. def __len__(self):
  663. if self.str:
  664. return len(self.str)
  665. else:
  666. return 0
  667. def notify(subject, body=None, html_body=None, to_string=None,
  668. attachments=None, append_instance_id=True):
  669. attachments = attachments or []
  670. if append_instance_id:
  671. subject = "[%s] %s" % (
  672. boto.config.get_value("Instance", "instance-id"), subject)
  673. if not to_string:
  674. to_string = boto.config.get_value('Notification', 'smtp_to', None)
  675. if to_string:
  676. try:
  677. from_string = boto.config.get_value('Notification',
  678. 'smtp_from', 'boto')
  679. msg = email.mime.multipart.MIMEMultipart()
  680. msg['From'] = from_string
  681. msg['Reply-To'] = from_string
  682. msg['To'] = to_string
  683. msg['Date'] = email.utils.formatdate(localtime=True)
  684. msg['Subject'] = subject
  685. if body:
  686. msg.attach(email.mime.text.MIMEText(body))
  687. if html_body:
  688. part = email.mime.base.MIMEBase('text', 'html')
  689. part.set_payload(html_body)
  690. email.encoders.encode_base64(part)
  691. msg.attach(part)
  692. for part in attachments:
  693. msg.attach(part)
  694. smtp_host = boto.config.get_value('Notification',
  695. 'smtp_host', 'localhost')
  696. # Alternate port support
  697. if boto.config.get_value("Notification", "smtp_port"):
  698. server = smtplib.SMTP(smtp_host, int(
  699. boto.config.get_value("Notification", "smtp_port")))
  700. else:
  701. server = smtplib.SMTP(smtp_host)
  702. # TLS support
  703. if boto.config.getbool("Notification", "smtp_tls"):
  704. server.ehlo()
  705. server.starttls()
  706. server.ehlo()
  707. smtp_user = boto.config.get_value('Notification', 'smtp_user', '')
  708. smtp_pass = boto.config.get_value('Notification', 'smtp_pass', '')
  709. if smtp_user:
  710. server.login(smtp_user, smtp_pass)
  711. server.sendmail(from_string, to_string, msg.as_string())
  712. server.quit()
  713. except:
  714. boto.log.exception('notify failed')
  715. def get_utf8_value(value):
  716. if not six.PY2 and isinstance(value, bytes):
  717. return value
  718. if not isinstance(value, six.string_types):
  719. value = six.text_type(value)
  720. if isinstance(value, six.text_type):
  721. value = value.encode('utf-8')
  722. return value
  723. def mklist(value):
  724. if not isinstance(value, list):
  725. if isinstance(value, tuple):
  726. value = list(value)
  727. else:
  728. value = [value]
  729. return value
  730. def pythonize_name(name):
  731. """Convert camel case to a "pythonic" name.
  732. Examples::
  733. pythonize_name('CamelCase') -> 'camel_case'
  734. pythonize_name('already_pythonized') -> 'already_pythonized'
  735. pythonize_name('HTTPRequest') -> 'http_request'
  736. pythonize_name('HTTPStatus200Ok') -> 'http_status_200_ok'
  737. pythonize_name('UPPER') -> 'upper'
  738. pythonize_name('') -> ''
  739. """
  740. s1 = _first_cap_regex.sub(r'\1_\2', name)
  741. s2 = _number_cap_regex.sub(r'\1_\2', s1)
  742. return _end_cap_regex.sub(r'\1_\2', s2).lower()
  743. def write_mime_multipart(content, compress=False, deftype='text/plain', delimiter=':'):
  744. """Description:
  745. :param content: A list of tuples of name-content pairs. This is used
  746. instead of a dict to ensure that scripts run in order
  747. :type list of tuples:
  748. :param compress: Use gzip to compress the scripts, defaults to no compression
  749. :type bool:
  750. :param deftype: The type that should be assumed if nothing else can be figured out
  751. :type str:
  752. :param delimiter: mime delimiter
  753. :type str:
  754. :return: Final mime multipart
  755. :rtype: str:
  756. """
  757. wrapper = email.mime.multipart.MIMEMultipart()
  758. for name, con in content:
  759. definite_type = guess_mime_type(con, deftype)
  760. maintype, subtype = definite_type.split('/', 1)
  761. if maintype == 'text':
  762. mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
  763. else:
  764. mime_con = email.mime.base.MIMEBase(maintype, subtype)
  765. mime_con.set_payload(con)
  766. # Encode the payload using Base64
  767. email.encoders.encode_base64(mime_con)
  768. mime_con.add_header('Content-Disposition', 'attachment', filename=name)
  769. wrapper.attach(mime_con)
  770. rcontent = wrapper.as_string()
  771. if compress:
  772. buf = StringIO()
  773. gz = gzip.GzipFile(mode='wb', fileobj=buf)
  774. try:
  775. gz.write(rcontent)
  776. finally:
  777. gz.close()
  778. rcontent = buf.getvalue()
  779. return rcontent
  780. def guess_mime_type(content, deftype):
  781. """Description: Guess the mime type of a block of text
  782. :param content: content we're finding the type of
  783. :type str:
  784. :param deftype: Default mime type
  785. :type str:
  786. :rtype: <type>:
  787. :return: <description>
  788. """
  789. # Mappings recognized by cloudinit
  790. starts_with_mappings = {
  791. '#include': 'text/x-include-url',
  792. '#!': 'text/x-shellscript',
  793. '#cloud-config': 'text/cloud-config',
  794. '#upstart-job': 'text/upstart-job',
  795. '#part-handler': 'text/part-handler',
  796. '#cloud-boothook': 'text/cloud-boothook'
  797. }
  798. rtype = deftype
  799. for possible_type, mimetype in starts_with_mappings.items():
  800. if content.startswith(possible_type):
  801. rtype = mimetype
  802. break
  803. return(rtype)
  804. def compute_md5(fp, buf_size=8192, size=None):
  805. """
  806. Compute MD5 hash on passed file and return results in a tuple of values.
  807. :type fp: file
  808. :param fp: File pointer to the file to MD5 hash. The file pointer
  809. will be reset to its current location before the
  810. method returns.
  811. :type buf_size: integer
  812. :param buf_size: Number of bytes per read request.
  813. :type size: int
  814. :param size: (optional) The Maximum number of bytes to read from
  815. the file pointer (fp). This is useful when uploading
  816. a file in multiple parts where the file is being
  817. split inplace into different parts. Less bytes may
  818. be available.
  819. :rtype: tuple
  820. :return: A tuple containing the hex digest version of the MD5 hash
  821. as the first element, the base64 encoded version of the
  822. plain digest as the second element and the data size as
  823. the third element.
  824. """
  825. return compute_hash(fp, buf_size, size, hash_algorithm=md5)
  826. def compute_hash(fp, buf_size=8192, size=None, hash_algorithm=md5):
  827. hash_obj = hash_algorithm()
  828. spos = fp.tell()
  829. if size and size < buf_size:
  830. s = fp.read(size)
  831. else:
  832. s = fp.read(buf_size)
  833. while s:
  834. if not isinstance(s, bytes):
  835. s = s.encode('utf-8')
  836. hash_obj.update(s)
  837. if size:
  838. size -= len(s)
  839. if size <= 0:
  840. break
  841. if size and size < buf_size:
  842. s = fp.read(size)
  843. else:
  844. s = fp.read(buf_size)
  845. hex_digest = hash_obj.hexdigest()
  846. base64_digest = encodebytes(hash_obj.digest()).decode('utf-8')
  847. if base64_digest[-1] == '\n':
  848. base64_digest = base64_digest[0:-1]
  849. # data_size based on bytes read.
  850. data_size = fp.tell() - spos
  851. fp.seek(spos)
  852. return (hex_digest, base64_digest, data_size)
  853. def find_matching_headers(name, headers):
  854. """
  855. Takes a specific header name and a dict of headers {"name": "value"}.
  856. Returns a list of matching header names, case-insensitive.
  857. """
  858. return [h for h in headers if h.lower() == name.lower()]
  859. def merge_headers_by_name(name, headers):
  860. """
  861. Takes a specific header name and a dict of headers {"name": "value"}.
  862. Returns a string of all header values, comma-separated, that match the
  863. input header name, case-insensitive.
  864. """
  865. matching_headers = find_matching_headers(name, headers)
  866. return ','.join(str(headers[h]) for h in matching_headers
  867. if headers[h] is not None)
  868. class RequestHook(object):
  869. """
  870. This can be extended and supplied to the connection object
  871. to gain access to request and response object after the request completes.
  872. One use for this would be to implement some specific request logging.
  873. """
  874. def handle_request_data(self, request, response, error=False):
  875. pass
  876. def host_is_ipv6(hostname):
  877. """
  878. Detect (naively) if the hostname is an IPV6 host.
  879. Return a boolean.
  880. """
  881. # empty strings or anything that is not a string is automatically not an
  882. # IPV6 address
  883. if not hostname or not isinstance(hostname, str):
  884. return False
  885. if hostname.startswith('['):
  886. return True
  887. if len(hostname.split(':')) > 2:
  888. return True
  889. # Anything else that doesn't start with brackets or doesn't have more than
  890. # one ':' should not be an IPV6 address. This is very naive but the rest of
  891. # the connection chain should error accordingly for typos or ill formed
  892. # addresses
  893. return False
  894. def parse_host(hostname):
  895. """
  896. Given a hostname that may have a port name, ensure that the port is trimmed
  897. returning only the host, including hostnames that are IPV6 and may include
  898. brackets.
  899. """
  900. # ensure that hostname does not have any whitespaces
  901. hostname = hostname.strip()
  902. if host_is_ipv6(hostname):
  903. return hostname.split(']:', 1)[0].strip('[]')
  904. else:
  905. return hostname.split(':', 1)[0]