COMP/CMSDIST/cherrypy-upload.patch

From: Lassi Tuura <lat@cern.ch>
Date: 2011-09-16 00:39:21 +0200
Subject: Add test to measure upload performance.


---
 cherrypy/test/test_upload.py |   72 ++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 72 insertions(+), 0 deletions(-)
 create mode 100644 cherrypy/test/test_upload.py

diff --git a/cherrypy/test/test_upload.py b/cherrypy/test/test_upload.py
new file mode 100644
index 0000000..f0c983c
--- /dev/null
+++ b/cherrypy/test/test_upload.py
@@ -0,0 +1,72 @@
+from cherrypy.test import test
+test.prefer_parent_path()
+
+import cherrypy, time, httplib, os, sys
+from cherrypy.test import helper
+
+def setup_server():
+  class Root:
+    @cherrypy.expose
+    def put(self, size, xtra, file):
+      assert xtra == 'foo'
+      now = time.time()
+      filelen = 0
+      while True:
+        data = file.file.read(8*1024*1024)
+        if not data: break
+        filelen += len(data)
+      cherrypy.response.headers["X-Timing"] = "%.3f" % (time.time() - now)
+      return "thanks len %d orig %d file '%s'" % (int(size), filelen, file.filename)
+
+  bufsize = int(os.environ.get("BODY_IO_SIZE", 1<<19))
+  cherrypy.tree.mount(Root(), "/small", config={'/': {}})
+  cherrypy.tree.mount(Root(), "/big", config={'/': {'request.body_io_size': bufsize}})
+  cherrypy.config.update({'server.max_request_body_size': 0, 'environment': 'test_suite'})
+
+class UploadTest(helper.CPWebCase):
+  def _encode(self, args, files, with_length):
+    body, crlf = '', '\r\n'
+    boundary = 'TEST'
+    for key, value in args.iteritems():
+      body += '--' + boundary + crlf
+      body += ('Content-Disposition: form-data; name="%s"' % key) + crlf
+      body += crlf + str(value) + crlf
+    for key, file in files.iteritems():
+      filename, filedata = file
+      body += '--' + boundary + crlf
+      body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
+               % (key, filename)) + crlf
+      body += ('Content-Type: application/octet-stream') + crlf
+      if with_length:
+        body += ('Content-Length: %d' % len(filedata)) + crlf
+      body += crlf + filedata + crlf
+    body += '--' + boundary + '--' + crlf + crlf
+    return 'multipart/form-data; boundary=' + boundary, body
+
+  def _marshall(self, name, kbytes, with_length):
+    data = ("x" * 127 + "\n") * 8 * kbytes
+    args = { 'size': len(data), 'xtra': 'foo' }
+    files = { 'file': (name, data) }
+    type, body = self._encode(args, files, with_length)
+    headers = [('Content-Type', type), ('Content-Length', len(body))]
+    return headers, body
+
+  def _run_test(self, kind, kbytes, with_length):
+    nbytes = kbytes*1024
+    headers, body = self._marshall(kind + 'file', kbytes, with_length)
+    start = time.time()
+    self.getPage("/%s/put" % kind, method='POST', headers = headers, body = body)
+    self.assertStatus(200)
+    self.assertBody("thanks len %d orig %d file '%sfile'" % (nbytes, nbytes, kind))
+    self.assertHeader("X-Timing")
+    xtiming = [v for k, v in self.headers if k.lower() == "x-timing"][0]
+    print "time(client=%.3f server=%s)" % (time.time() - start, xtiming),
+
+  def test_small_smart(self): self._run_test('small', 10, True)
+  def test_small_dumb(self):  self._run_test('small', 10, False)
+  def test_big_smart(self):   self._run_test('big', 500*1024, True)
+  def test_big_dumb(self):    self._run_test('big', 500*1024, False)
+
+if __name__ == "__main__":
+    setup_server()
+    helper.testmain()
From: Lassi Tuura <lat@cern.ch>
Date: 2011-09-15 21:38:39 +0200
Subject: Parametrise the buffer size used for reading multipart body.


---
 cherrypy/_cpcgifs.py   |    2 ++
 cherrypy/_cprequest.py |   10 +++++++++-
 2 files changed, 11 insertions(+), 1 deletions(-)

diff --git a/cherrypy/_cpcgifs.py b/cherrypy/_cpcgifs.py
index 3eb5dd8..f1c1f15 100644
--- a/cherrypy/_cpcgifs.py
+++ b/cherrypy/_cpcgifs.py
@@ -5,6 +5,8 @@ import cherrypy
 class FieldStorage(cgi.FieldStorage):
     def __init__(self, *args, **kwds):
         try:
+            size = kwds.get('environ', {}).get('_bufsize', None)
+            if size: cgi.FieldStorage.bufsize = size
             cgi.FieldStorage.__init__(self, *args, **kwds)
         except ValueError, ex:
             if str(ex) == 'Maximum content length exceeded':
diff --git a/cherrypy/_cprequest.py b/cherrypy/_cprequest.py
index 23bdad2..83e8cdf 100644
--- a/cherrypy/_cprequest.py
+++ b/cherrypy/_cprequest.py
@@ -301,6 +301,12 @@ class Request(object):
     can be sent with various HTTP method verbs). This value is set between
     the 'before_request_body' and 'before_handler' hooks (assuming that
     process_request_body is True)."""
+
+    body_io_size = _cpcgifs.FieldStorage.bufsize
+    body_io_size__doc = """
+    The I/O chunk size for reading multipart body content. The default is
+    suitable for small POST params, but for servers regularly receiving
+    large file uploads it is a good idea to increase this substantially."""
     
     # Dispatch attributes
     dispatch = cherrypy.dispatch.Dispatcher()
@@ -715,10 +721,12 @@ class Request(object):
             h = self.headers
         
         try:
+            self.rfile.rfile.default_bufsize = max(1<<16, self.body_io_size)
             forms = _cpcgifs.FieldStorage(fp=self.rfile,
                                           headers=h,
                                           # FieldStorage only recognizes POST.
-                                          environ={'REQUEST_METHOD': "POST"},
+                                          environ={'REQUEST_METHOD': "POST",
+                                                   '_bufsize': self.body_io_size},
                                           keep_blank_values=1)
         except Exception, e:
             if e.__class__.__name__ == 'MaxSizeExceeded':
From: Lassi Tuura <lat@cern.ch>
Date: 2011-09-15 21:39:31 +0200
Subject: Avoid buffer object churn when reading in request body.


---
 cherrypy/wsgiserver/__init__.py |   37 ++++++++++++++++++++-----------------
 1 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/cherrypy/wsgiserver/__init__.py b/cherrypy/wsgiserver/__init__.py
index c380e18..94d3438 100644
--- a/cherrypy/wsgiserver/__init__.py
+++ b/cherrypy/wsgiserver/__init__.py
@@ -721,6 +721,7 @@ class FatalSSLAlert(Exception):
 if not _fileobject_uses_str_type:
     class CP_fileobject(socket._fileobject):
         """Faux file object attached to a socket object."""
+        __read_pos = 0
 
         def sendall(self, data):
             """Sendall for non-blocking sockets."""
@@ -754,6 +755,7 @@ if not _fileobject_uses_str_type:
             # Use max, disallow tiny reads in a loop as they are very inefficient.
             # We never leave read() with any leftover data from a new recv() call
             # in our internal buffer.
+            pos = self.__read_pos
             rbufsize = max(self._rbufsize, self.default_bufsize)
             # Our use of StringIO rather than lists of string objects returned by
             # recv() minimizes memory usage and fragmentation that occurs when
@@ -762,24 +764,25 @@ if not _fileobject_uses_str_type:
             buf.seek(0, 2)  # seek end
             if size < 0:
                 # Read until EOF
+                self.__read_pos = 0
                 self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
                 while True:
                     data = self.recv(rbufsize)
                     if not data:
                         break
                     buf.write(data)
-                return buf.getvalue()
+                return buf.getvalue()[pos:]
             else:
                 # Read until size bytes or EOF seen, whichever comes first
-                buf_len = buf.tell()
+                buf_len = buf.tell()-pos
                 if buf_len >= size:
                     # Already have size bytes in our buffer?  Extract and return.
-                    buf.seek(0)
+                    buf.seek(pos)
                     rv = buf.read(size)
-                    self._rbuf = StringIO.StringIO()
-                    self._rbuf.write(buf.read())
+                    self.__read_pos = buf.tell()
                     return rv
 
+                self.__read_pos = 0
                 self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
                 while True:
                     left = size - buf_len
@@ -808,25 +811,26 @@ if not _fileobject_uses_str_type:
                     buf_len += n
                     del data  # explicit free
                     #assert buf_len == buf.tell()
-                return buf.getvalue()
+                return buf.getvalue()[pos:]
 
         def readline(self, size=-1):
+            pos = self.__read_pos
             buf = self._rbuf
             buf.seek(0, 2)  # seek end
-            if buf.tell() > 0:
+            if buf.tell()-pos > 0:
                 # check if we already have it in our buffer
-                buf.seek(0)
+                buf.seek(pos)
                 bline = buf.readline(size)
                 if bline.endswith('\n') or len(bline) == size:
-                    self._rbuf = StringIO.StringIO()
-                    self._rbuf.write(buf.read())
+                    self.__read_pos = buf.tell()
                     return bline
                 del bline
+            self.__read_pos = 0
             if size < 0:
                 # Read until \n or EOF, whichever comes first
                 if self._rbufsize <= 1:
                     # Speed up unbuffered case
-                    buf.seek(0)
+                    buf.seek(pos)
                     buffers = [buf.read()]
                     self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
                     data = None
@@ -852,16 +856,15 @@ if not _fileobject_uses_str_type:
                         del data
                         break
                     buf.write(data)
-                return buf.getvalue()
+                return buf.getvalue()[pos:]
             else:
                 # Read until size bytes or \n or EOF seen, whichever comes first
                 buf.seek(0, 2)  # seek end
-                buf_len = buf.tell()
+                buf_len = buf.tell()-pos
                 if buf_len >= size:
-                    buf.seek(0)
+                    buf.seek(pos)
                     rv = buf.read(size)
-                    self._rbuf = StringIO.StringIO()
-                    self._rbuf.write(buf.read())
+                    self.__read_pos = buf.tell()
                     return rv
                 self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
                 while True:
@@ -894,7 +897,7 @@ if not _fileobject_uses_str_type:
                     buf.write(data)
                     buf_len += n
                     #assert buf_len == buf.tell()
-                return buf.getvalue()
+                return buf.getvalue()[pos:]
 
 else:
     class CP_fileobject(socket._fileobject):
#	User	Rev	Content
1	lat	1.1	From: Lassi Tuura <lat@cern.ch>
2			Date: 2011-09-16 00:39:21 +0200
3			Subject: Add test to measure upload performance.
4
5
6			---
7			cherrypy/test/test_upload.py \| 72 ++++++++++++++++++++++++++++++++++++++++++
8			1 files changed, 72 insertions(+), 0 deletions(-)
9			create mode 100644 cherrypy/test/test_upload.py
10
11			diff --git a/cherrypy/test/test_upload.py b/cherrypy/test/test_upload.py
12			new file mode 100644
13			index 0000000..f0c983c
14			--- /dev/null
15			+++ b/cherrypy/test/test_upload.py
16			@@ -0,0 +1,72 @@
17			+from cherrypy.test import test
18			+test.prefer_parent_path()
19			+
20			+import cherrypy, time, httplib, os, sys
21			+from cherrypy.test import helper
22			+
23			+def setup_server():
24			+ class Root:
25			+ @cherrypy.expose
26			+ def put(self, size, xtra, file):
27			+ assert xtra == 'foo'
28			+ now = time.time()
29			+ filelen = 0
30			+ while True:
31			+ data = file.file.read(810241024)
32			+ if not data: break
33			+ filelen += len(data)
34			+ cherrypy.response.headers["X-Timing"] = "%.3f" % (time.time() - now)
35			+ return "thanks len %d orig %d file '%s'" % (int(size), filelen, file.filename)
36			+
37			+ bufsize = int(os.environ.get("BODY_IO_SIZE", 1<<19))
38			+ cherrypy.tree.mount(Root(), "/small", config={'/': {}})
39			+ cherrypy.tree.mount(Root(), "/big", config={'/': {'request.body_io_size': bufsize}})
40			+ cherrypy.config.update({'server.max_request_body_size': 0, 'environment': 'test_suite'})
41			+
42			+class UploadTest(helper.CPWebCase):
43			+ def _encode(self, args, files, with_length):
44			+ body, crlf = '', '\r\n'
45			+ boundary = 'TEST'
46			+ for key, value in args.iteritems():
47			+ body += '--' + boundary + crlf
48			+ body += ('Content-Disposition: form-data; name="%s"' % key) + crlf
49			+ body += crlf + str(value) + crlf
50			+ for key, file in files.iteritems():
51			+ filename, filedata = file
52			+ body += '--' + boundary + crlf
53			+ body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
54			+ % (key, filename)) + crlf
55			+ body += ('Content-Type: application/octet-stream') + crlf
56			+ if with_length:
57			+ body += ('Content-Length: %d' % len(filedata)) + crlf
58			+ body += crlf + filedata + crlf
59			+ body += '--' + boundary + '--' + crlf + crlf
60			+ return 'multipart/form-data; boundary=' + boundary, body
61			+
62			+ def _marshall(self, name, kbytes, with_length):
63			+ data = ("x" * 127 + "\n") * 8 * kbytes
64			+ args = { 'size': len(data), 'xtra': 'foo' }
65			+ files = { 'file': (name, data) }
66			+ type, body = self._encode(args, files, with_length)
67			+ headers = [('Content-Type', type), ('Content-Length', len(body))]
68			+ return headers, body
69			+
70			+ def _run_test(self, kind, kbytes, with_length):
71			+ nbytes = kbytes*1024
72			+ headers, body = self._marshall(kind + 'file', kbytes, with_length)
73			+ start = time.time()
74			+ self.getPage("/%s/put" % kind, method='POST', headers = headers, body = body)
75			+ self.assertStatus(200)
76			+ self.assertBody("thanks len %d orig %d file '%sfile'" % (nbytes, nbytes, kind))
77			+ self.assertHeader("X-Timing")
78			+ xtiming = [v for k, v in self.headers if k.lower() == "x-timing"][0]
79			+ print "time(client=%.3f server=%s)" % (time.time() - start, xtiming),
80			+
81			+ def test_small_smart(self): self._run_test('small', 10, True)
82			+ def test_small_dumb(self): self._run_test('small', 10, False)
83			+ def test_big_smart(self): self._run_test('big', 500*1024, True)
84			+ def test_big_dumb(self): self._run_test('big', 500*1024, False)
85			+
86			+if __name__ == "__main__":
87			+ setup_server()
88			+ helper.testmain()
89			From: Lassi Tuura <lat@cern.ch>
90			Date: 2011-09-15 21:38:39 +0200
91			Subject: Parametrise the buffer size used for reading multipart body.
92
93
94			---
95			cherrypy/_cpcgifs.py \| 2 ++
96			cherrypy/_cprequest.py \| 10 +++++++++-
97			2 files changed, 11 insertions(+), 1 deletions(-)
98
99			diff --git a/cherrypy/_cpcgifs.py b/cherrypy/_cpcgifs.py
100			index 3eb5dd8..f1c1f15 100644
101			--- a/cherrypy/_cpcgifs.py
102			+++ b/cherrypy/_cpcgifs.py
103			@@ -5,6 +5,8 @@ import cherrypy
104			class FieldStorage(cgi.FieldStorage):
105			def __init__(self, args, *kwds):
106			try:
107			+ size = kwds.get('environ', {}).get('_bufsize', None)
108			+ if size: cgi.FieldStorage.bufsize = size
109			cgi.FieldStorage.__init__(self, args, *kwds)
110			except ValueError, ex:
111			if str(ex) == 'Maximum content length exceeded':
112			diff --git a/cherrypy/_cprequest.py b/cherrypy/_cprequest.py
113			index 23bdad2..83e8cdf 100644
114			--- a/cherrypy/_cprequest.py
115			+++ b/cherrypy/_cprequest.py
116			@@ -301,6 +301,12 @@ class Request(object):
117			can be sent with various HTTP method verbs). This value is set between
118			the 'before_request_body' and 'before_handler' hooks (assuming that
119			process_request_body is True)."""
120			+
121			+ body_io_size = _cpcgifs.FieldStorage.bufsize
122			+ body_io_size__doc = """
123			+ The I/O chunk size for reading multipart body content. The default is
124			+ suitable for small POST params, but for servers regularly receiving
125			+ large file uploads it is a good idea to increase this substantially."""
126
127			# Dispatch attributes
128			dispatch = cherrypy.dispatch.Dispatcher()
129			@@ -715,10 +721,12 @@ class Request(object):
130			h = self.headers
131
132			try:
133			+ self.rfile.rfile.default_bufsize = max(1<<16, self.body_io_size)
134			forms = _cpcgifs.FieldStorage(fp=self.rfile,
135			headers=h,
136			# FieldStorage only recognizes POST.
137			- environ={'REQUEST_METHOD': "POST"},
138			+ environ={'REQUEST_METHOD': "POST",
139			+ '_bufsize': self.body_io_size},
140			keep_blank_values=1)
141			except Exception, e:
142			if e.__class__.__name__ == 'MaxSizeExceeded':
143			From: Lassi Tuura <lat@cern.ch>
144			Date: 2011-09-15 21:39:31 +0200
145			Subject: Avoid buffer object churn when reading in request body.
146
147
148			---
149			cherrypy/wsgiserver/__init__.py \| 37 ++++++++++++++++++++-----------------
150			1 files changed, 20 insertions(+), 17 deletions(-)
151
152			diff --git a/cherrypy/wsgiserver/__init__.py b/cherrypy/wsgiserver/__init__.py
153			index c380e18..94d3438 100644
154			--- a/cherrypy/wsgiserver/__init__.py
155			+++ b/cherrypy/wsgiserver/__init__.py
156			@@ -721,6 +721,7 @@ class FatalSSLAlert(Exception):
157			if not _fileobject_uses_str_type:
158			class CP_fileobject(socket._fileobject):
159			"""Faux file object attached to a socket object."""
160			+ __read_pos = 0
161
162			def sendall(self, data):
163			"""Sendall for non-blocking sockets."""
164			@@ -754,6 +755,7 @@ if not _fileobject_uses_str_type:
165			# Use max, disallow tiny reads in a loop as they are very inefficient.
166			# We never leave read() with any leftover data from a new recv() call
167			# in our internal buffer.
168			+ pos = self.__read_pos
169			rbufsize = max(self._rbufsize, self.default_bufsize)
170			# Our use of StringIO rather than lists of string objects returned by
171			# recv() minimizes memory usage and fragmentation that occurs when
172			@@ -762,24 +764,25 @@ if not _fileobject_uses_str_type:
173			buf.seek(0, 2) # seek end
174			if size < 0:
175			# Read until EOF
176			+ self.__read_pos = 0
177			self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf.
178			while True:
179			data = self.recv(rbufsize)
180			if not data:
181			break
182			buf.write(data)
183			- return buf.getvalue()
184			+ return buf.getvalue()[pos:]
185			else:
186			# Read until size bytes or EOF seen, whichever comes first
187			- buf_len = buf.tell()
188			+ buf_len = buf.tell()-pos
189			if buf_len >= size:
190			# Already have size bytes in our buffer? Extract and return.
191			- buf.seek(0)
192			+ buf.seek(pos)
193			rv = buf.read(size)
194			- self._rbuf = StringIO.StringIO()
195			- self._rbuf.write(buf.read())
196			+ self.__read_pos = buf.tell()
197			return rv
198
199			+ self.__read_pos = 0
200			self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf.
201			while True:
202			left = size - buf_len
203			@@ -808,25 +811,26 @@ if not _fileobject_uses_str_type:
204			buf_len += n
205			del data # explicit free
206			#assert buf_len == buf.tell()
207			- return buf.getvalue()
208			+ return buf.getvalue()[pos:]
209
210			def readline(self, size=-1):
211			+ pos = self.__read_pos
212			buf = self._rbuf
213			buf.seek(0, 2) # seek end
214			- if buf.tell() > 0:
215			+ if buf.tell()-pos > 0:
216			# check if we already have it in our buffer
217			- buf.seek(0)
218			+ buf.seek(pos)
219			bline = buf.readline(size)
220			if bline.endswith('\n') or len(bline) == size:
221			- self._rbuf = StringIO.StringIO()
222			- self._rbuf.write(buf.read())
223			+ self.__read_pos = buf.tell()
224			return bline
225			del bline
226			+ self.__read_pos = 0
227			if size < 0:
228			# Read until \n or EOF, whichever comes first
229			if self._rbufsize <= 1:
230			# Speed up unbuffered case
231			- buf.seek(0)
232			+ buf.seek(pos)
233			buffers = [buf.read()]
234			self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf.
235			data = None
236			@@ -852,16 +856,15 @@ if not _fileobject_uses_str_type:
237			del data
238			break
239			buf.write(data)
240			- return buf.getvalue()
241			+ return buf.getvalue()[pos:]
242			else:
243			# Read until size bytes or \n or EOF seen, whichever comes first
244			buf.seek(0, 2) # seek end
245			- buf_len = buf.tell()
246			+ buf_len = buf.tell()-pos
247			if buf_len >= size:
248			- buf.seek(0)
249			+ buf.seek(pos)
250			rv = buf.read(size)
251			- self._rbuf = StringIO.StringIO()
252			- self._rbuf.write(buf.read())
253			+ self.__read_pos = buf.tell()
254			return rv
255			self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf.
256			while True:
257			@@ -894,7 +897,7 @@ if not _fileobject_uses_str_type:
258			buf.write(data)
259			buf_len += n
260			#assert buf_len == buf.tell()
261			- return buf.getvalue()
262			+ return buf.getvalue()[pos:]
263
264			else:
265			class CP_fileobject(socket._fileobject):