progress.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222
  1. # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
  2. # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
  3. # (c) 2005 Clark C. Evans
  4. # This module is part of the Python Paste Project and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. # This code was written with funding by http://prometheusresearch.com
  7. """
  8. Upload Progress Monitor
  9. This is a WSGI middleware component which monitors the status of files
  10. being uploaded. It includes a small query application which will return
  11. a list of all files being uploaded by particular session/user.
  12. >>> from paste.httpserver import serve
  13. >>> from paste.urlmap import URLMap
  14. >>> from paste.auth.basic import AuthBasicHandler
  15. >>> from paste.debug.debugapp import SlowConsumer, SimpleApplication
  16. >>> # from paste.progress import *
  17. >>> realm = 'Test Realm'
  18. >>> def authfunc(username, password):
  19. ... return username == password
  20. >>> map = URLMap({})
  21. >>> ups = UploadProgressMonitor(map, threshold=1024)
  22. >>> map['/upload'] = SlowConsumer()
  23. >>> map['/simple'] = SimpleApplication()
  24. >>> map['/report'] = UploadProgressReporter(ups)
  25. >>> serve(AuthBasicHandler(ups, realm, authfunc))
  26. serving on...
  27. .. note::
  28. This is experimental, and will change in the future.
  29. """
  30. import time
  31. from paste.wsgilib import catch_errors
  32. DEFAULT_THRESHOLD = 1024 * 1024 # one megabyte
  33. DEFAULT_TIMEOUT = 60*5 # five minutes
  34. ENVIRON_RECEIVED = 'paste.bytes_received'
  35. REQUEST_STARTED = 'paste.request_started'
  36. REQUEST_FINISHED = 'paste.request_finished'
  37. class _ProgressFile(object):
  38. """
  39. This is the input-file wrapper used to record the number of
  40. ``paste.bytes_received`` for the given request.
  41. """
  42. def __init__(self, environ, rfile):
  43. self._ProgressFile_environ = environ
  44. self._ProgressFile_rfile = rfile
  45. self.flush = rfile.flush
  46. self.write = rfile.write
  47. self.writelines = rfile.writelines
  48. def __iter__(self):
  49. environ = self._ProgressFile_environ
  50. riter = iter(self._ProgressFile_rfile)
  51. def iterwrap():
  52. for chunk in riter:
  53. environ[ENVIRON_RECEIVED] += len(chunk)
  54. yield chunk
  55. return iter(iterwrap)
  56. def read(self, size=-1):
  57. chunk = self._ProgressFile_rfile.read(size)
  58. self._ProgressFile_environ[ENVIRON_RECEIVED] += len(chunk)
  59. return chunk
  60. def readline(self):
  61. chunk = self._ProgressFile_rfile.readline()
  62. self._ProgressFile_environ[ENVIRON_RECEIVED] += len(chunk)
  63. return chunk
  64. def readlines(self, hint=None):
  65. chunk = self._ProgressFile_rfile.readlines(hint)
  66. self._ProgressFile_environ[ENVIRON_RECEIVED] += len(chunk)
  67. return chunk
  68. class UploadProgressMonitor(object):
  69. """
  70. monitors and reports on the status of uploads in progress
  71. Parameters:
  72. ``application``
  73. This is the next application in the WSGI stack.
  74. ``threshold``
  75. This is the size in bytes that is needed for the
  76. upload to be included in the monitor.
  77. ``timeout``
  78. This is the amount of time (in seconds) that a upload
  79. remains in the monitor after it has finished.
  80. Methods:
  81. ``uploads()``
  82. This returns a list of ``environ`` dict objects for each
  83. upload being currently monitored, or finished but whose time
  84. has not yet expired.
  85. For each request ``environ`` that is monitored, there are several
  86. variables that are stored:
  87. ``paste.bytes_received``
  88. This is the total number of bytes received for the given
  89. request; it can be compared with ``CONTENT_LENGTH`` to
  90. build a percentage complete. This is an integer value.
  91. ``paste.request_started``
  92. This is the time (in seconds) when the request was started
  93. as obtained from ``time.time()``. One would want to format
  94. this for presentation to the user, if necessary.
  95. ``paste.request_finished``
  96. This is the time (in seconds) when the request was finished,
  97. canceled, or otherwise disconnected. This is None while
  98. the given upload is still in-progress.
  99. TODO: turn monitor into a queue and purge queue of finished
  100. requests that have passed the timeout period.
  101. """
  102. def __init__(self, application, threshold=None, timeout=None):
  103. self.application = application
  104. self.threshold = threshold or DEFAULT_THRESHOLD
  105. self.timeout = timeout or DEFAULT_TIMEOUT
  106. self.monitor = []
  107. def __call__(self, environ, start_response):
  108. length = environ.get('CONTENT_LENGTH', 0)
  109. if length and int(length) > self.threshold:
  110. # replace input file object
  111. self.monitor.append(environ)
  112. environ[ENVIRON_RECEIVED] = 0
  113. environ[REQUEST_STARTED] = time.time()
  114. environ[REQUEST_FINISHED] = None
  115. environ['wsgi.input'] = \
  116. _ProgressFile(environ, environ['wsgi.input'])
  117. def finalizer(exc_info=None):
  118. environ[REQUEST_FINISHED] = time.time()
  119. return catch_errors(self.application, environ,
  120. start_response, finalizer, finalizer)
  121. return self.application(environ, start_response)
  122. def uploads(self):
  123. return self.monitor
  124. class UploadProgressReporter(object):
  125. """
  126. reports on the progress of uploads for a given user
  127. This reporter returns a JSON file (for use in AJAX) listing the
  128. uploads in progress for the given user. By default, this reporter
  129. uses the ``REMOTE_USER`` environment to compare between the current
  130. request and uploads in-progress. If they match, then a response
  131. record is formed.
  132. ``match()``
  133. This member function can be overriden to provide alternative
  134. matching criteria. It takes two environments, the first
  135. is the current request, the second is a current upload.
  136. ``report()``
  137. This member function takes an environment and builds a
  138. ``dict`` that will be used to create a JSON mapping for
  139. the given upload. By default, this just includes the
  140. percent complete and the request url.
  141. """
  142. def __init__(self, monitor):
  143. self.monitor = monitor
  144. def match(self, search_environ, upload_environ):
  145. if search_environ.get('REMOTE_USER', None) == \
  146. upload_environ.get('REMOTE_USER', 0):
  147. return True
  148. return False
  149. def report(self, environ):
  150. retval = { 'started': time.strftime("%Y-%m-%d %H:%M:%S",
  151. time.gmtime(environ[REQUEST_STARTED])),
  152. 'finished': '',
  153. 'content_length': environ.get('CONTENT_LENGTH'),
  154. 'bytes_received': environ[ENVIRON_RECEIVED],
  155. 'path_info': environ.get('PATH_INFO',''),
  156. 'query_string': environ.get('QUERY_STRING','')}
  157. finished = environ[REQUEST_FINISHED]
  158. if finished:
  159. retval['finished'] = time.strftime("%Y:%m:%d %H:%M:%S",
  160. time.gmtime(finished))
  161. return retval
  162. def __call__(self, environ, start_response):
  163. body = []
  164. for map in [self.report(env) for env in self.monitor.uploads()
  165. if self.match(environ, env)]:
  166. parts = []
  167. for k, v in map.items():
  168. v = str(v).replace("\\", "\\\\").replace('"', '\\"')
  169. parts.append('%s: "%s"' % (k, v))
  170. body.append("{ %s }" % ", ".join(parts))
  171. body = "[ %s ]" % ", ".join(body)
  172. start_response("200 OK", [('Content-Type', 'text/plain'),
  173. ('Content-Length', len(body))])
  174. return [body]
  175. __all__ = ['UploadProgressMonitor', 'UploadProgressReporter']
  176. if "__main__" == __name__:
  177. import doctest
  178. doctest.testmod(optionflags=doctest.ELLIPSIS)