webcrawler.py 824 B

1234567891011121314151617181920212223242526
  1. #! /usr/bin/env python
  2. """
  3. This is a simple web "crawler" that fetches a bunch of urls using a pool to
  4. control the number of outbound connections. It has as many simultaneously open
  5. connections as coroutines in the pool.
  6. The prints in the body of the fetch function are there to demonstrate that the
  7. requests are truly made in parallel.
  8. """
  9. urls = ["http://www.google.com/intl/en_ALL/images/logo.gif",
  10. "https://wiki.secondlife.com/w/images/secondlife.jpg",
  11. "http://us.i1.yimg.com/us.yimg.com/i/ww/beta/y3.gif"]
  12. import eventlet
  13. from eventlet.green import urllib2
  14. def fetch(url):
  15. print "opening", url
  16. body = urllib2.urlopen(url).read()
  17. print "done with", url
  18. return url, body
  19. pool = eventlet.GreenPool(200)
  20. for url, body in pool.imap(fetch, urls):
  21. print "got body from", url, "of length", len(body)