From 235a6e43bcbe7587816809e75b963d5c4b025f99 Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Mon, 24 Feb 2014 18:54:49 +0400 Subject: [PATCH 1/5] fix #14 without tests --- aiohttp/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/protocol.py b/aiohttp/protocol.py index ebf9b0479ba..dfd76607448 100644 --- a/aiohttp/protocol.py +++ b/aiohttp/protocol.py @@ -253,7 +253,7 @@ def __call__(self, out, buf): class HttpPayloadParser: - def __init__(self, message, length=None, compression=True, readall=False): + def __init__(self, message, length=None, compression=True, readall=True): self.message = message self.length = length self.compression = compression From 33bc3796a3bc42e0eb2a35fe29d5f9ae62b80ae6 Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Wed, 27 Jan 2016 12:41:46 +0300 Subject: [PATCH 2/5] read_chunk body part without content-length pep8 fixes for PR fixes for PR fix typo don't search boundary twice --- aiohttp/multipart.py | 58 +++++++++++++++++++++++++++++++++++++---- aiohttp/streams.py | 14 ++++++++++ tests/test_multipart.py | 52 ++++++++++++++++++++++++++++++++---- tests/test_streams.py | 38 +++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 10 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index f5a9e95f6fb..e85a5249176 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -213,6 +213,7 @@ def __init__(self, boundary, headers, content): self._length = int(length) if length is not None else None self._read_bytes = 0 self._unread = deque() + self._prev_chunk = None @asyncio.coroutine def __aiter__(self): @@ -258,7 +259,6 @@ def read(self, *, decode=False): @asyncio.coroutine def read_chunk(self, size=chunk_size): """Reads body part content chunk of the specified size. - The body part must has `Content-Length` header with proper value. :param int size: chunk size @@ -266,17 +266,65 @@ def read_chunk(self, size=chunk_size): """ if self._at_eof: return b'' - assert self._length is not None, \ - 'Content-Length required for chunked read' - chunk_size = min(size, self._length - self._read_bytes) - chunk = yield from self._content.read(chunk_size) + if self._length: + chunk = yield from self._read_chunk_from_length(size) + else: + chunk = yield from self._read_chunk_from_stream(size) + self._read_bytes += len(chunk) if self._read_bytes == self._length: self._at_eof = True + if self._at_eof: assert b'\r\n' == (yield from self._content.readline()), \ 'reader did not read all the data or it is malformed' return chunk + @asyncio.coroutine + def _read_chunk_from_length(self, size): + """Reads body part content chunk of the specified size. + The body part must has `Content-Length` header with proper value. + + :param int size: chunk size + + :rtype: bytearray + """ + assert self._length is not None, \ + 'Content-Length required for chunked read' + chunk_size = min(size, self._length - self._read_bytes) + chunk = yield from self._content.read(chunk_size) + return chunk + + @asyncio.coroutine + def _read_chunk_from_stream(self, size): + """Reads content chunk of body part with unknown length. + The `Content-Length` header for body part is not necessary. + + :param int size: chunk size + + :rtype: bytearray + """ + assert size >= len(self._boundary) + 2, \ + 'Chunk size must be greater or equal than boundary length + 2' + if self._prev_chunk is None: + self._prev_chunk = yield from self._content.read(size) + + chunk = yield from self._content.read(size) + + window = self._prev_chunk + chunk + sub = b'\r\n' + self._boundary + idx = window.find(sub, len(self._prev_chunk) - len(sub)) + if idx >= 0: + # pushing boundary back to content + self._content.unread_data(window[idx:]) + if size > idx: + self._prev_chunk = self._prev_chunk[:idx] + chunk = window[size:idx] + if not chunk: + self._at_eof = True + result = self._prev_chunk + self._prev_chunk = chunk + return result + @asyncio.coroutine def readline(self): """Reads body part by line by line. diff --git a/aiohttp/streams.py b/aiohttp/streams.py index a732c856860..679eaeff885 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -155,6 +155,20 @@ def wait_eof(self): finally: self._eof_waiter = None + def unread_data(self, data): + """ rollback reading some data from stream, inserting it to buffer head. + """ + assert not self._eof, 'unread_data after feed_eof' + + if not data: + return + + if self._buffer_offset: + self._buffer[0] = self._buffer[0][self._buffer_offset:] + self._buffer_offset = 0 + self._buffer.appendleft(data) + self._buffer_size += len(data) + def feed_data(self, data): assert not self._eof, 'feed_data after feed_eof' diff --git a/tests/test_multipart.py b/tests/test_multipart.py index f63fbb2d698..05bda3497c8 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -73,6 +73,9 @@ def read(self, size=None): def readline(self): return self.content.readline() + def unread_data(self, data): + self.content = io.BytesIO(data + self.content.read()) + class StreamWithShortenRead(Stream): @@ -156,11 +159,23 @@ def test_read_chunk_at_eof(self): result = yield from obj.read_chunk() self.assertEqual(b'', result) - def test_read_chunk_requires_content_length(self): + def test_read_chunk_without_content_length(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream(b'Hello, world!\r\n--:')) - with self.assertRaises(AssertionError): - yield from obj.read_chunk() + c1 = yield from obj.read_chunk(8) + c2 = yield from obj.read_chunk(8) + c3 = yield from obj.read_chunk(8) + self.assertEqual(c1 + c2, b'Hello, world!') + self.assertEqual(c3, b'') + + def test_multi_read_chunk(self): + stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') + obj = aiohttp.multipart.BodyPartReader(self.boundary, {}, stream) + result = yield from obj.read_chunk(8) + self.assertEqual(b'Hello,', result) + result = yield from obj.read_chunk(8) + self.assertEqual(b'', result) + self.assertTrue(obj.at_eof()) def test_read_chunk_properly_counts_read_bytes(self): expected = b'.' * 10 @@ -557,7 +572,7 @@ def test_release_without_read_the_last_object(self): self.assertTrue(second.at_eof()) self.assertIsNone(third) - def test_read_chunk_doesnt_breaks_reader(self): + def test_read_chunk_by_length_doesnt_breaks_reader(self): reader = aiohttp.multipart.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, Stream(b'--:\r\n' @@ -567,12 +582,39 @@ def test_read_chunk_doesnt_breaks_reader(self): b'Content-Length: 6\r\n\r\n' b'passed' b'\r\n--:--')) + body_parts = [] + while True: + read_part = b'' + part = yield from reader.next() + if part is None: + break + while not part.at_eof(): + read_part += yield from part.read_chunk(3) + body_parts.append(read_part) + self.assertListEqual(body_parts, [b'test', b'passed']) + + def test_read_chunk_from_stream_doesnt_breaks_reader(self): + reader = aiohttp.multipart.MultipartReader( + {CONTENT_TYPE: 'multipart/related;boundary=":"'}, + Stream(b'--:\r\n' + b'\r\n' + b'chunk' + b'\r\n--:\r\n' + b'\r\n' + b'two_chunks' + b'\r\n--:--')) + body_parts = [] while True: + read_part = b'' part = yield from reader.next() if part is None: break while not part.at_eof(): - yield from part.read_chunk(3) + chunk = yield from part.read_chunk(5) + self.assertTrue(chunk) + read_part += chunk + body_parts.append(read_part) + self.assertListEqual(body_parts, [b'chunk', b'two_chunks']) class BodyPartWriterTestCase(unittest.TestCase): diff --git a/tests/test_streams.py b/tests/test_streams.py index b760e15b81c..8fab257b233 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -371,6 +371,44 @@ def test_readexactly_exception(self): self.assertRaises( ValueError, self.loop.run_until_complete, stream.readexactly(2)) + def test_unread_data(self): + stream = self._make_one() + stream.feed_data(b'line1') + stream.feed_data(b'line2') + stream.feed_data(b'onemoreline') + + data = self.loop.run_until_complete(stream.read(5)) + self.assertEqual(b'line1', data) + + stream.unread_data(data) + + data = self.loop.run_until_complete(stream.read(5)) + self.assertEqual(b'line1', data) + + data = self.loop.run_until_complete(stream.read(4)) + self.assertEqual(b'line', data) + + stream.unread_data(b'line1line') + + data = b'' + while len(data) < 10: + data += self.loop.run_until_complete(stream.read(10)) + self.assertEqual(b'line1line2', data) + + data = self.loop.run_until_complete(stream.read(7)) + self.assertEqual(b'onemore', data) + + stream.unread_data(data) + + data = b'' + while len(data) < 11: + data += self.loop.run_until_complete(stream.read(11)) + self.assertEqual(b'onemoreline', data) + + stream.unread_data(b'line') + data = self.loop.run_until_complete(stream.read(4)) + self.assertEqual(b'line', data) + def test_exception(self): stream = self._make_one() self.assertIsNone(stream.exception()) From 46ed000ca69b055c5bd6c38797a7896a7aa11a04 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Jan 2016 16:22:46 +0200 Subject: [PATCH 3/5] Update CHANGES --- CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index d16fe518107..203ee1c5d80 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -62,3 +62,5 @@ CHANGES - ClientSession.close and Connector.close are coroutines now - Close client connection on exception in ClientResponse.release() + +- Allow to read multipart parts without content-length specified #750 From ffe977ace686a27ebe7ebd3a28766632afa7ac49 Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Sun, 31 Jan 2016 12:51:14 +0300 Subject: [PATCH 4/5] allow unread_data to stream at eof state --- aiohttp/streams.py | 2 -- tests/test_streams.py | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index d7b998ece73..1cfab77ca84 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -159,8 +159,6 @@ def wait_eof(self): def unread_data(self, data): """ rollback reading some data from stream, inserting it to buffer head. """ - assert not self._eof, 'unread_data after feed_eof' - if not data: return diff --git a/tests/test_streams.py b/tests/test_streams.py index 8fab257b233..8ff025120bd 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -409,6 +409,11 @@ def test_unread_data(self): data = self.loop.run_until_complete(stream.read(4)) self.assertEqual(b'line', data) + stream.feed_eof() + stream.unread_data(b'at_eof') + data = self.loop.run_until_complete(stream.read(6)) + self.assertEqual(b'at_eof', data) + def test_exception(self): stream = self._make_one() self.assertIsNone(stream.exception()) From ce5b597a8cf3da29adb64b4f737d0fb6f79022fc Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Sun, 31 Jan 2016 12:51:43 +0300 Subject: [PATCH 5/5] fix result cutting for incomplete prev_chunk in multipart --- aiohttp/multipart.py | 2 +- tests/test_multipart.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index e85a5249176..005257761d2 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -318,7 +318,7 @@ def _read_chunk_from_stream(self, size): self._content.unread_data(window[idx:]) if size > idx: self._prev_chunk = self._prev_chunk[:idx] - chunk = window[size:idx] + chunk = window[len(self._prev_chunk):idx] if not chunk: self._at_eof = True result = self._prev_chunk diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 05bda3497c8..f03b996f1ec 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -168,6 +168,27 @@ def test_read_chunk_without_content_length(self): self.assertEqual(c1 + c2, b'Hello, world!') self.assertEqual(c3, b'') + def test_read_incomplete_chunk(self): + stream = Stream(b'') + def prepare(data): + f = asyncio.Future(loop=self.loop) + f.set_result(data) + return f + with mock.patch.object(stream, 'read', side_effect=[ + prepare(b'Hello, '), + prepare(b'World'), + prepare(b'!\r\n--:'), + prepare(b'') + ]): + obj = aiohttp.multipart.BodyPartReader( + self.boundary, {}, stream) + c1 = yield from obj.read_chunk(8) + self.assertEqual(c1, b'Hello, ') + c2 = yield from obj.read_chunk(8) + self.assertEqual(c2, b'World') + c3 = yield from obj.read_chunk(8) + self.assertEqual(c3, b'!') + def test_multi_read_chunk(self): stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') obj = aiohttp.multipart.BodyPartReader(self.boundary, {}, stream)