wsgi.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612
  1. import errno
  2. import os
  3. import sys
  4. import time
  5. import traceback
  6. import warnings
  7. from eventlet.green import urllib
  8. from eventlet.green import socket
  9. from eventlet.green import BaseHTTPServer
  10. from eventlet import greenpool
  11. from eventlet import greenio
  12. from eventlet.support import get_errno
  13. DEFAULT_MAX_SIMULTANEOUS_REQUESTS = 1024
  14. DEFAULT_MAX_HTTP_VERSION = 'HTTP/1.1'
  15. MAX_REQUEST_LINE = 8192
  16. MINIMUM_CHUNK_SIZE = 4096
  17. DEFAULT_LOG_FORMAT= ('%(client_ip)s - - [%(date_time)s] "%(request_line)s"'
  18. ' %(status_code)s %(body_length)s %(wall_seconds).6f')
  19. __all__ = ['server', 'format_date_time']
  20. # Weekday and month names for HTTP date/time formatting; always English!
  21. _weekdayname = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
  22. _monthname = [None, # Dummy so we can use 1-based month numbers
  23. "Jan", "Feb", "Mar", "Apr", "May", "Jun",
  24. "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
  25. def format_date_time(timestamp):
  26. """Formats a unix timestamp into an HTTP standard string."""
  27. year, month, day, hh, mm, ss, wd, _y, _z = time.gmtime(timestamp)
  28. return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
  29. _weekdayname[wd], day, _monthname[month], year, hh, mm, ss
  30. )
  31. # Collections of error codes to compare against. Not all attributes are set
  32. # on errno module on all platforms, so some are literals :(
  33. BAD_SOCK = set((errno.EBADF, 10053))
  34. BROKEN_SOCK = set((errno.EPIPE, errno.ECONNRESET))
  35. # special flag return value for apps
  36. class _AlreadyHandled(object):
  37. def __iter__(self):
  38. return self
  39. def next(self):
  40. raise StopIteration
  41. ALREADY_HANDLED = _AlreadyHandled()
  42. class Input(object):
  43. def __init__(self,
  44. rfile,
  45. content_length,
  46. wfile=None,
  47. wfile_line=None,
  48. chunked_input=False):
  49. self.rfile = rfile
  50. if content_length is not None:
  51. content_length = int(content_length)
  52. self.content_length = content_length
  53. self.wfile = wfile
  54. self.wfile_line = wfile_line
  55. self.position = 0
  56. self.chunked_input = chunked_input
  57. self.chunk_length = -1
  58. def _do_read(self, reader, length=None):
  59. if self.wfile is not None:
  60. ## 100 Continue
  61. self.wfile.write(self.wfile_line)
  62. self.wfile = None
  63. self.wfile_line = None
  64. if length is None and self.content_length is not None:
  65. length = self.content_length - self.position
  66. if length and length > self.content_length - self.position:
  67. length = self.content_length - self.position
  68. if not length:
  69. return ''
  70. try:
  71. read = reader(length)
  72. except greenio.SSL.ZeroReturnError:
  73. read = ''
  74. self.position += len(read)
  75. return read
  76. def _chunked_read(self, rfile, length=None, use_readline=False):
  77. if self.wfile is not None:
  78. ## 100 Continue
  79. self.wfile.write(self.wfile_line)
  80. self.wfile = None
  81. self.wfile_line = None
  82. try:
  83. if length == 0:
  84. return ""
  85. if length < 0:
  86. length = None
  87. if use_readline:
  88. reader = self.rfile.readline
  89. else:
  90. reader = self.rfile.read
  91. response = []
  92. while self.chunk_length != 0:
  93. maxreadlen = self.chunk_length - self.position
  94. if length is not None and length < maxreadlen:
  95. maxreadlen = length
  96. if maxreadlen > 0:
  97. data = reader(maxreadlen)
  98. if not data:
  99. self.chunk_length = 0
  100. raise IOError("unexpected end of file while parsing chunked data")
  101. datalen = len(data)
  102. response.append(data)
  103. self.position += datalen
  104. if self.chunk_length == self.position:
  105. rfile.readline()
  106. if length is not None:
  107. length -= datalen
  108. if length == 0:
  109. break
  110. if use_readline and data[-1] == "\n":
  111. break
  112. else:
  113. self.chunk_length = int(rfile.readline().split(";", 1)[0], 16)
  114. self.position = 0
  115. if self.chunk_length == 0:
  116. rfile.readline()
  117. except greenio.SSL.ZeroReturnError:
  118. pass
  119. return ''.join(response)
  120. def read(self, length=None):
  121. if self.chunked_input:
  122. return self._chunked_read(self.rfile, length)
  123. return self._do_read(self.rfile.read, length)
  124. def readline(self, size=None):
  125. if self.chunked_input:
  126. return self._chunked_read(self.rfile, size, True)
  127. else:
  128. return self._do_read(self.rfile.readline, size)
  129. def readlines(self, hint=None):
  130. return self._do_read(self.rfile.readlines, hint)
  131. def __iter__(self):
  132. return iter(self.read())
  133. def get_socket(self):
  134. return self.rfile._sock
  135. class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
  136. protocol_version = 'HTTP/1.1'
  137. minimum_chunk_size = MINIMUM_CHUNK_SIZE
  138. def setup(self):
  139. # overriding SocketServer.setup to correctly handle SSL.Connection objects
  140. conn = self.connection = self.request
  141. try:
  142. self.rfile = conn.makefile('rb', self.rbufsize)
  143. self.wfile = conn.makefile('wb', self.wbufsize)
  144. except (AttributeError, NotImplementedError):
  145. if hasattr(conn, 'send') and hasattr(conn, 'recv'):
  146. # it's an SSL.Connection
  147. self.rfile = socket._fileobject(conn, "rb", self.rbufsize)
  148. self.wfile = socket._fileobject(conn, "wb", self.wbufsize)
  149. else:
  150. # it's a SSLObject, or a martian
  151. raise NotImplementedError("wsgi.py doesn't support sockets "\
  152. "of type %s" % type(conn))
  153. def handle_one_request(self):
  154. if self.server.max_http_version:
  155. self.protocol_version = self.server.max_http_version
  156. if self.rfile.closed:
  157. self.close_connection = 1
  158. return
  159. try:
  160. self.raw_requestline = self.rfile.readline(MAX_REQUEST_LINE)
  161. if len(self.raw_requestline) == MAX_REQUEST_LINE:
  162. self.wfile.write(
  163. "HTTP/1.0 414 Request URI Too Long\r\n"
  164. "Connection: close\r\nContent-length: 0\r\n\r\n")
  165. self.close_connection = 1
  166. return
  167. except greenio.SSL.ZeroReturnError:
  168. self.raw_requestline = ''
  169. except socket.error, e:
  170. if get_errno(e) not in BAD_SOCK:
  171. raise
  172. self.raw_requestline = ''
  173. if not self.raw_requestline:
  174. self.close_connection = 1
  175. return
  176. if not self.parse_request():
  177. return
  178. content_length = self.headers.getheader('content-length')
  179. if content_length:
  180. try:
  181. int(content_length)
  182. except ValueError:
  183. self.wfile.write(
  184. "HTTP/1.0 400 Bad Request\r\n"
  185. "Connection: close\r\nContent-length: 0\r\n\r\n")
  186. self.close_connection = 1
  187. return
  188. self.environ = self.get_environ()
  189. self.application = self.server.app
  190. try:
  191. self.server.outstanding_requests += 1
  192. try:
  193. self.handle_one_response()
  194. except socket.error, e:
  195. # Broken pipe, connection reset by peer
  196. if get_errno(e) not in BROKEN_SOCK:
  197. raise
  198. finally:
  199. self.server.outstanding_requests -= 1
  200. def handle_one_response(self):
  201. start = time.time()
  202. headers_set = []
  203. headers_sent = []
  204. wfile = self.wfile
  205. result = None
  206. use_chunked = [False]
  207. length = [0]
  208. status_code = [200]
  209. def write(data, _writelines=wfile.writelines):
  210. towrite = []
  211. if not headers_set:
  212. raise AssertionError("write() before start_response()")
  213. elif not headers_sent:
  214. status, response_headers = headers_set
  215. headers_sent.append(1)
  216. header_list = [header[0].lower() for header in response_headers]
  217. towrite.append('%s %s\r\n' % (self.protocol_version, status))
  218. for header in response_headers:
  219. towrite.append('%s: %s\r\n' % header)
  220. # send Date header?
  221. if 'date' not in header_list:
  222. towrite.append('Date: %s\r\n' % (format_date_time(time.time()),))
  223. client_conn = self.headers.get('Connection', '').lower()
  224. send_keep_alive = False
  225. if self.close_connection == 0 and \
  226. self.server.keepalive and (client_conn == 'keep-alive' or \
  227. (self.request_version == 'HTTP/1.1' and
  228. not client_conn == 'close')):
  229. # only send keep-alives back to clients that sent them,
  230. # it's redundant for 1.1 connections
  231. send_keep_alive = (client_conn == 'keep-alive')
  232. self.close_connection = 0
  233. else:
  234. self.close_connection = 1
  235. if 'content-length' not in header_list:
  236. if self.request_version == 'HTTP/1.1':
  237. use_chunked[0] = True
  238. towrite.append('Transfer-Encoding: chunked\r\n')
  239. elif 'content-length' not in header_list:
  240. # client is 1.0 and therefore must read to EOF
  241. self.close_connection = 1
  242. if self.close_connection:
  243. towrite.append('Connection: close\r\n')
  244. elif send_keep_alive:
  245. towrite.append('Connection: keep-alive\r\n')
  246. towrite.append('\r\n')
  247. # end of header writing
  248. if use_chunked[0]:
  249. ## Write the chunked encoding
  250. towrite.append("%x\r\n%s\r\n" % (len(data), data))
  251. else:
  252. towrite.append(data)
  253. try:
  254. _writelines(towrite)
  255. length[0] = length[0] + sum(map(len, towrite))
  256. except UnicodeEncodeError:
  257. self.server.log_message("Encountered non-ascii unicode while attempting to write wsgi response: %r" % [x for x in towrite if isinstance(x, unicode)])
  258. self.server.log_message(traceback.format_exc())
  259. _writelines(
  260. ["HTTP/1.1 500 Internal Server Error\r\n",
  261. "Connection: close\r\n",
  262. "Content-type: text/plain\r\n",
  263. "Content-length: 98\r\n",
  264. "Date: %s\r\n" % format_date_time(time.time()),
  265. "\r\n",
  266. ("Internal Server Error: wsgi application passed "
  267. "a unicode object to the server instead of a string.")])
  268. def start_response(status, response_headers, exc_info=None):
  269. status_code[0] = status.split()[0]
  270. if exc_info:
  271. try:
  272. if headers_sent:
  273. # Re-raise original exception if headers sent
  274. raise exc_info[0], exc_info[1], exc_info[2]
  275. finally:
  276. # Avoid dangling circular ref
  277. exc_info = None
  278. capitalized_headers = [('-'.join([x.capitalize()
  279. for x in key.split('-')]), value)
  280. for key, value in response_headers]
  281. headers_set[:] = [status, capitalized_headers]
  282. return write
  283. try:
  284. try:
  285. result = self.application(self.environ, start_response)
  286. if (isinstance(result, _AlreadyHandled)
  287. or isinstance(getattr(result, '_obj', None), _AlreadyHandled)):
  288. self.close_connection = 1
  289. return
  290. if not headers_sent and hasattr(result, '__len__') and \
  291. 'Content-Length' not in [h for h, _v in headers_set[1]]:
  292. headers_set[1].append(('Content-Length', str(sum(map(len, result)))))
  293. towrite = []
  294. towrite_size = 0
  295. just_written_size = 0
  296. for data in result:
  297. towrite.append(data)
  298. towrite_size += len(data)
  299. if towrite_size >= self.minimum_chunk_size:
  300. write(''.join(towrite))
  301. towrite = []
  302. just_written_size = towrite_size
  303. towrite_size = 0
  304. if towrite:
  305. just_written_size = towrite_size
  306. write(''.join(towrite))
  307. if not headers_sent or (use_chunked[0] and just_written_size):
  308. write('')
  309. except Exception:
  310. self.close_connection = 1
  311. exc = traceback.format_exc()
  312. self.server.log_message(exc)
  313. if not headers_set:
  314. start_response("500 Internal Server Error",
  315. [('Content-type', 'text/plain'),
  316. ('Content-length', len(exc))])
  317. write(exc)
  318. finally:
  319. if hasattr(result, 'close'):
  320. result.close()
  321. if (self.environ['eventlet.input'].chunked_input or
  322. self.environ['eventlet.input'].position \
  323. < self.environ['eventlet.input'].content_length):
  324. ## Read and discard body if there was no pending 100-continue
  325. if not self.environ['eventlet.input'].wfile:
  326. while self.environ['eventlet.input'].read(MINIMUM_CHUNK_SIZE):
  327. pass
  328. finish = time.time()
  329. for hook, args, kwargs in self.environ['eventlet.posthooks']:
  330. hook(self.environ, *args, **kwargs)
  331. self.server.log_message(self.server.log_format % dict(
  332. client_ip=self.get_client_ip(),
  333. date_time=self.log_date_time_string(),
  334. request_line=self.requestline,
  335. status_code=status_code[0],
  336. body_length=length[0],
  337. wall_seconds=finish - start))
  338. def get_client_ip(self):
  339. client_ip = self.client_address[0]
  340. if self.server.log_x_forwarded_for:
  341. forward = self.headers.get('X-Forwarded-For', '').replace(' ', '')
  342. if forward:
  343. client_ip = "%s,%s" % (forward, client_ip)
  344. return client_ip
  345. def get_environ(self):
  346. env = self.server.get_environ()
  347. env['REQUEST_METHOD'] = self.command
  348. env['SCRIPT_NAME'] = ''
  349. pq = self.path.split('?', 1)
  350. env['PATH_INFO'] = urllib.unquote(pq[0])
  351. if len(pq) > 1:
  352. env['QUERY_STRING'] = pq[1]
  353. if self.headers.typeheader is None:
  354. env['CONTENT_TYPE'] = self.headers.type
  355. else:
  356. env['CONTENT_TYPE'] = self.headers.typeheader
  357. length = self.headers.getheader('content-length')
  358. if length:
  359. env['CONTENT_LENGTH'] = length
  360. env['SERVER_PROTOCOL'] = 'HTTP/1.0'
  361. host, port = self.request.getsockname()[:2]
  362. env['SERVER_NAME'] = host
  363. env['SERVER_PORT'] = str(port)
  364. env['REMOTE_ADDR'] = self.client_address[0]
  365. env['GATEWAY_INTERFACE'] = 'CGI/1.1'
  366. for h in self.headers.headers:
  367. k, v = h.split(':', 1)
  368. k = k.replace('-', '_').upper()
  369. v = v.strip()
  370. if k in env:
  371. continue
  372. envk = 'HTTP_' + k
  373. if envk in env:
  374. env[envk] += ',' + v
  375. else:
  376. env[envk] = v
  377. if env.get('HTTP_EXPECT') == '100-continue':
  378. wfile = self.wfile
  379. wfile_line = 'HTTP/1.1 100 Continue\r\n\r\n'
  380. else:
  381. wfile = None
  382. wfile_line = None
  383. chunked = env.get('HTTP_TRANSFER_ENCODING', '').lower() == 'chunked'
  384. env['wsgi.input'] = env['eventlet.input'] = Input(
  385. self.rfile, length, wfile=wfile, wfile_line=wfile_line,
  386. chunked_input=chunked)
  387. env['eventlet.posthooks'] = []
  388. return env
  389. def finish(self):
  390. BaseHTTPServer.BaseHTTPRequestHandler.finish(self)
  391. greenio.shutdown_safe(self.connection)
  392. self.connection.close()
  393. class Server(BaseHTTPServer.HTTPServer):
  394. def __init__(self,
  395. socket,
  396. address,
  397. app,
  398. log=None,
  399. environ=None,
  400. max_http_version=None,
  401. protocol=HttpProtocol,
  402. minimum_chunk_size=None,
  403. log_x_forwarded_for=True,
  404. keepalive=True,
  405. log_format=DEFAULT_LOG_FORMAT):
  406. self.outstanding_requests = 0
  407. self.socket = socket
  408. self.address = address
  409. if log:
  410. self.log = log
  411. else:
  412. self.log = sys.stderr
  413. self.app = app
  414. self.keepalive = keepalive
  415. self.environ = environ
  416. self.max_http_version = max_http_version
  417. self.protocol = protocol
  418. self.pid = os.getpid()
  419. if minimum_chunk_size is not None:
  420. protocol.minimum_chunk_size = minimum_chunk_size
  421. self.log_x_forwarded_for = log_x_forwarded_for
  422. self.log_format = log_format
  423. def get_environ(self):
  424. d = {
  425. 'wsgi.errors': sys.stderr,
  426. 'wsgi.version': (1, 0),
  427. 'wsgi.multithread': True,
  428. 'wsgi.multiprocess': False,
  429. 'wsgi.run_once': False,
  430. 'wsgi.url_scheme': 'http',
  431. }
  432. # detect secure socket
  433. if hasattr(self.socket, 'do_handshake'):
  434. d['wsgi.url_scheme'] = 'https'
  435. d['HTTPS'] = 'on'
  436. if self.environ is not None:
  437. d.update(self.environ)
  438. return d
  439. def process_request(self, (socket, address)):
  440. proto = self.protocol(socket, address, self)
  441. proto.handle()
  442. def log_message(self, message):
  443. self.log.write(message + '\n')
  444. try:
  445. import ssl
  446. ACCEPT_EXCEPTIONS = (socket.error, ssl.SSLError)
  447. ACCEPT_ERRNO = set((errno.EPIPE, errno.EBADF, errno.ECONNRESET,
  448. ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_SSL))
  449. except ImportError:
  450. ACCEPT_EXCEPTIONS = (socket.error,)
  451. ACCEPT_ERRNO = set((errno.EPIPE, errno.EBADF, errno.ECONNRESET))
  452. def server(sock, site,
  453. log=None,
  454. environ=None,
  455. max_size=None,
  456. max_http_version=DEFAULT_MAX_HTTP_VERSION,
  457. protocol=HttpProtocol,
  458. server_event=None,
  459. minimum_chunk_size=None,
  460. log_x_forwarded_for=True,
  461. custom_pool=None,
  462. keepalive=True,
  463. log_format=DEFAULT_LOG_FORMAT):
  464. """ Start up a wsgi server handling requests from the supplied server
  465. socket. This function loops forever. The *sock* object will be closed after server exits,
  466. but the underlying file descriptor will remain open, so if you have a dup() of *sock*,
  467. it will remain usable.
  468. :param sock: Server socket, must be already bound to a port and listening.
  469. :param site: WSGI application function.
  470. :param log: File-like object that logs should be written to. If not specified, sys.stderr is used.
  471. :param environ: Additional parameters that go into the environ dictionary of every request.
  472. :param max_size: Maximum number of client connections opened at any time by this server.
  473. :param max_http_version: Set to "HTTP/1.0" to make the server pretend it only supports HTTP 1.0. This can help with applications or clients that don't behave properly using HTTP 1.1.
  474. :param protocol: Protocol class. Deprecated.
  475. :param server_event: Used to collect the Server object. Deprecated.
  476. :param minimum_chunk_size: Minimum size in bytes for http chunks. This can be used to improve performance of applications which yield many small strings, though using it technically violates the WSGI spec.
  477. :param log_x_forwarded_for: If True (the default), logs the contents of the x-forwarded-for header in addition to the actual client ip address in the 'client_ip' field of the log line.
  478. :param custom_pool: A custom GreenPool instance which is used to spawn client green threads. If this is supplied, max_size is ignored.
  479. :param keepalive: If set to False, disables keepalives on the server; all connections will be closed after serving one request.
  480. :param log_format: A python format string that is used as the template to generate log lines. The following values can be formatted into it: client_ip, date_time, request_line, status_code, body_length, wall_seconds. The default is a good example of how to use it.
  481. """
  482. serv = Server(sock, sock.getsockname(),
  483. site, log,
  484. environ=environ,
  485. max_http_version=max_http_version,
  486. protocol=protocol,
  487. minimum_chunk_size=minimum_chunk_size,
  488. log_x_forwarded_for=log_x_forwarded_for,
  489. keepalive=keepalive,
  490. log_format=log_format)
  491. if server_event is not None:
  492. server_event.send(serv)
  493. if max_size is None:
  494. max_size = DEFAULT_MAX_SIMULTANEOUS_REQUESTS
  495. if custom_pool is not None:
  496. pool = custom_pool
  497. else:
  498. pool = greenpool.GreenPool(max_size)
  499. try:
  500. host, port = sock.getsockname()[:2]
  501. port = ':%s' % (port, )
  502. if hasattr(sock, 'do_handshake'):
  503. scheme = 'https'
  504. if port == ':443':
  505. port = ''
  506. else:
  507. scheme = 'http'
  508. if port == ':80':
  509. port = ''
  510. serv.log.write("(%s) wsgi starting up on %s://%s%s/\n" % (
  511. os.getpid(), scheme, host, port))
  512. while True:
  513. try:
  514. client_socket = sock.accept()
  515. try:
  516. pool.spawn_n(serv.process_request, client_socket)
  517. except AttributeError:
  518. warnings.warn("wsgi's pool should be an instance of " \
  519. "eventlet.greenpool.GreenPool, is %s. Please convert your"\
  520. " call site to use GreenPool instead" % type(pool),
  521. DeprecationWarning, stacklevel=2)
  522. pool.execute_async(serv.process_request, client_socket)
  523. except ACCEPT_EXCEPTIONS, e:
  524. if get_errno(e) not in ACCEPT_ERRNO:
  525. raise
  526. except (KeyboardInterrupt, SystemExit):
  527. serv.log.write("wsgi exiting\n")
  528. break
  529. finally:
  530. try:
  531. # NOTE: It's not clear whether we want this to leave the
  532. # socket open or close it. Use cases like Spawning want
  533. # the underlying fd to remain open, but if we're going
  534. # that far we might as well not bother closing sock at
  535. # all.
  536. sock.close()
  537. except socket.error, e:
  538. if get_errno(e) not in BROKEN_SOCK:
  539. traceback.print_exc()