From 848582df0e58d53176bb545f21799afe5ff76d9b Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 13:07:38 -0700 Subject: [PATCH 1/9] add PyCharm venv dir to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6540510..ccea939 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ docs/build .pytest_cache # PyCharm -.idea/ \ No newline at end of file +.idea/ +venv/ \ No newline at end of file From 3105277cfe7cd7c219ca0894a1a31defc9c2f99a Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 13:14:16 -0700 Subject: [PATCH 2/9] add isort and autoformat --- setup.cfg | 11 +++++++++++ tox.ini | 29 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/setup.cfg b/setup.cfg index 02db1ba..eb61762 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,3 +39,14 @@ ignore = # Doc8 Configuration [doc8] max-line-length = 120 + +[isort] +line_length = 120 +# https://github.com/timothycrosley/isort#multi-line-output-modes +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +combine_as_imports = True +not_skip = __init__.py +known_first_party = base64io +known_third_party =base64io,mock,pytest,setuptools diff --git a/tox.ini b/tox.ini index 6881c3e..b05c9be 100644 --- a/tox.ini +++ b/tox.ini @@ -129,6 +129,35 @@ deps = commands = {[testenv:blacken-src]commands} --diff +[testenv:isort-seed] +basepython = python3 +deps = seed-isort-config +commands = seed-isort-config + +[testenv:isort] +basepython = python3 +deps = isort +commands = isort -rc \ + src \ + test \ + doc \ + setup.py \ + {posargs} + +[testenv:isort-check] +basepython = python3 +deps = {[testenv:isort]deps} +commands = {[testenv:isort]commands} -c + +[testenv:autoformat] +basepython = python3 +deps = + {[testenv:blacken]deps} + {[testenv:isort]deps} +commands = + {[testenv:blacken]commands} + {[testenv:isort]commands} + [testenv:pylint] basepython = {[testenv:default-python]basepython} deps = From 63a267f628beb1d76200e9e07ae9c588b34fe0d0 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 13:14:40 -0700 Subject: [PATCH 3/9] apply autoformat --- doc/conf.py | 2 +- src/base64io/__init__.py | 7 ++++--- test/unit/test_base64_stream.py | 6 +++--- test/unit/test_base64io.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index ef6f7ab..d3c5d1b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,9 +1,9 @@ # pylint: disable=invalid-name """Sphinx configuration.""" -from datetime import datetime import io import os import re +from datetime import datetime VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""") HERE = os.path.abspath(os.path.dirname(__file__)) diff --git a/src/base64io/__init__.py b/src/base64io/__init__.py index d399080..4822523 100644 --- a/src/base64io/__init__.py +++ b/src/base64io/__init__.py @@ -227,7 +227,7 @@ def _read_additional_data_removing_whitespace(self, data, total_bytes_to_read): return data _data_buffer = io.BytesIO() if isinstance(data, bytes) else io.StringIO() - join_char = b'' if isinstance(data, bytes) else u'' + join_char = b"" if isinstance(data, bytes) else u"" _data_buffer.write(join_char.join(data.split())) # type: ignore _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell() # type: ignore @@ -274,8 +274,9 @@ def read(self, b=-1): data = self.__wrapped.read(_bytes_to_read) # Remove whitespace from read data and attempt to read more data to get the desired # number of bytes. - whitespace = string.whitespace.encode("utf-8") if isinstance(data, bytes) \ - else string.whitespace # type: Union[bytes, str] + whitespace = ( + string.whitespace.encode("utf-8") if isinstance(data, bytes) else string.whitespace + ) # type: Union[bytes, str] if any([char in data for char in whitespace]): data = self._read_additional_data_removing_whitespace(data, _bytes_to_read) diff --git a/test/unit/test_base64_stream.py b/test/unit/test_base64_stream.py index 4492516..59fe701 100644 --- a/test/unit/test_base64_stream.py +++ b/test/unit/test_base64_stream.py @@ -19,8 +19,8 @@ import math import os -from mock import MagicMock, sentinel import pytest +from mock import MagicMock, sentinel from base64io import Base64IO @@ -159,7 +159,7 @@ def test_base64io_decode(bytes_to_generate, bytes_per_round, number_of_rounds, t ) def test_base64io_decode_str(bytes_to_generate, bytes_per_round, number_of_rounds, total_bytes_to_expect): plaintext_source = os.urandom(bytes_to_generate) - plaintext_b64 = io.StringIO(base64.b64encode(plaintext_source).decode('ascii')) + plaintext_b64 = io.StringIO(base64.b64encode(plaintext_source).decode("ascii")) plaintext_wrapped = Base64IO(plaintext_b64) test = b"" @@ -315,7 +315,7 @@ def test_base64io_decode_with_whitespace(plaintext_source, b64_plaintext_with_wh @pytest.mark.parametrize("plaintext_source, b64_plaintext_with_whitespace, read_bytes", build_whitespace_testcases()) def test_base64io_decode_with_whitespace_str(plaintext_source, b64_plaintext_with_whitespace, read_bytes): - with Base64IO(io.StringIO(b64_plaintext_with_whitespace.decode('ascii'))) as decoder: + with Base64IO(io.StringIO(b64_plaintext_with_whitespace.decode("ascii"))) as decoder: test = decoder.read(read_bytes) assert test == plaintext_source[:read_bytes] diff --git a/test/unit/test_base64io.py b/test/unit/test_base64io.py index dfb9d20..07528b7 100644 --- a/test/unit/test_base64io.py +++ b/test/unit/test_base64io.py @@ -32,7 +32,7 @@ def test_file(): if is_python2: # If we are in Python 2, the "file" assignment should not # happen because it is a builtin object. - assert not hasattr(base64io, 'file') + assert not hasattr(base64io, "file") else: # If we are in Python 3, the "file" assignment should happen # to provide a concrete definition of the "file" name. From 56e9bf45fd6b9ebe1f610e1dcb9651e353d2debf Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 13:18:38 -0700 Subject: [PATCH 4/9] run string tests with both ascii and utf-8 decoding --- test/unit/test_base64_stream.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/unit/test_base64_stream.py b/test/unit/test_base64_stream.py index 59fe701..1924796 100644 --- a/test/unit/test_base64_stream.py +++ b/test/unit/test_base64_stream.py @@ -154,12 +154,13 @@ def test_base64io_decode(bytes_to_generate, bytes_per_round, number_of_rounds, t assert test == plaintext_source[:total_bytes_to_expect] +@pytest.mark.parametrize("encoding", ("ascii", "utf-8")) @pytest.mark.parametrize( "bytes_to_generate, bytes_per_round, number_of_rounds, total_bytes_to_expect", build_test_cases() ) -def test_base64io_decode_str(bytes_to_generate, bytes_per_round, number_of_rounds, total_bytes_to_expect): +def test_base64io_decode_str(encoding, bytes_to_generate, bytes_per_round, number_of_rounds, total_bytes_to_expect): plaintext_source = os.urandom(bytes_to_generate) - plaintext_b64 = io.StringIO(base64.b64encode(plaintext_source).decode("ascii")) + plaintext_b64 = io.StringIO(base64.b64encode(plaintext_source).decode(encoding)) plaintext_wrapped = Base64IO(plaintext_b64) test = b"" @@ -313,9 +314,10 @@ def test_base64io_decode_with_whitespace(plaintext_source, b64_plaintext_with_wh assert test == plaintext_source[:read_bytes] +@pytest.mark.parametrize("encoding", ("ascii", "utf-8")) @pytest.mark.parametrize("plaintext_source, b64_plaintext_with_whitespace, read_bytes", build_whitespace_testcases()) -def test_base64io_decode_with_whitespace_str(plaintext_source, b64_plaintext_with_whitespace, read_bytes): - with Base64IO(io.StringIO(b64_plaintext_with_whitespace.decode("ascii"))) as decoder: +def test_base64io_decode_with_whitespace_str(encoding, plaintext_source, b64_plaintext_with_whitespace, read_bytes): + with Base64IO(io.StringIO(b64_plaintext_with_whitespace.decode(encoding))) as decoder: test = decoder.read(read_bytes) assert test == plaintext_source[:read_bytes] From 52f2c9824ba8940e24795c12bc6b72ecf4657960 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 16:09:54 -0700 Subject: [PATCH 5/9] streamline string handling: single-source logic by simply converting data read from wrapped stream to bytes immediately --- src/base64io/__init__.py | 38 ++++++++++++++++++++++++-------------- test/unit/test_base64io.py | 9 +++++++++ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/base64io/__init__.py b/src/base64io/__init__.py index 4822523..f99382d 100644 --- a/src/base64io/__init__.py +++ b/src/base64io/__init__.py @@ -52,6 +52,19 @@ def _py2(): file = NotImplemented # pylint: disable=invalid-name +def _to_bytes(data): + # type: (AnyStr) -> bytes + """Convert input data from either string or bytes to bytes. + + :param data: Data to convert + :returns: ``data`` converted to bytes + :rtype: bytes + """ + if isinstance(data, bytes): + return data + return data.encode("utf-8") + + class Base64IO(io.IOBase): """Base64 stream with context manager support. @@ -209,7 +222,7 @@ def writelines(self, lines): self.write(line) def _read_additional_data_removing_whitespace(self, data, total_bytes_to_read): - # type: (AnyStr, int) -> AnyStr + # type: (bytes, int) -> bytes """Read additional data from wrapped stream until we reach the desired number of bytes. .. note:: @@ -226,20 +239,20 @@ def _read_additional_data_removing_whitespace(self, data, total_bytes_to_read): # case the base64 module happily removes any whitespace. return data - _data_buffer = io.BytesIO() if isinstance(data, bytes) else io.StringIO() - join_char = b"" if isinstance(data, bytes) else u"" - _data_buffer.write(join_char.join(data.split())) # type: ignore - _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell() # type: ignore + _data_buffer = io.BytesIO() + + _data_buffer.write(b"".join(data.split())) + _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell() while _remaining_bytes_to_read > 0: - _raw_additional_data = self.__wrapped.read(_remaining_bytes_to_read) + _raw_additional_data = _to_bytes(self.__wrapped.read(_remaining_bytes_to_read)) if not _raw_additional_data: # No more data to read from wrapped stream. break - _data_buffer.write(join_char.join(_raw_additional_data.split())) # type: ignore - _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell() # type: ignore - return _data_buffer.getvalue() # type: ignore + _data_buffer.write(b"".join(_raw_additional_data.split())) + _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell() + return _data_buffer.getvalue() def read(self, b=-1): # type: (int) -> bytes @@ -271,14 +284,11 @@ def read(self, b=-1): _bytes_to_read += 4 - _bytes_to_read % 4 # Read encoded bytes from wrapped stream. - data = self.__wrapped.read(_bytes_to_read) + data = _to_bytes(self.__wrapped.read(_bytes_to_read)) # Remove whitespace from read data and attempt to read more data to get the desired # number of bytes. - whitespace = ( - string.whitespace.encode("utf-8") if isinstance(data, bytes) else string.whitespace - ) # type: Union[bytes, str] - if any([char in data for char in whitespace]): + if any([char in data for char in string.whitespace.encode("utf-8")]): data = self._read_additional_data_removing_whitespace(data, _bytes_to_read) results = io.BytesIO() diff --git a/test/unit/test_base64io.py b/test/unit/test_base64io.py index 07528b7..c93462d 100644 --- a/test/unit/test_base64io.py +++ b/test/unit/test_base64io.py @@ -37,3 +37,12 @@ def test_file(): # If we are in Python 3, the "file" assignment should happen # to provide a concrete definition of the "file" name. assert base64io.file is NotImplemented + + +@pytest.mark.parametrize("source, expected", ( + ('asdf', b'asdf'), + (b'\x00\x01\x02\x03', b'\x00\x01\x02\x03'), + (u'\u1111\u2222', b'\xe1\x84\x91\xe2\x88\xa2') +)) +def test_to_bytes(source, expected): + assert base64io._to_bytes(source) == expected From 8b50da954017cf6ec43f114c6dc53d313c07c109 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 17:54:18 -0700 Subject: [PATCH 6/9] add docs autobuild --- tox.ini | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tox.ini b/tox.ini index b05c9be..6d63c5d 100644 --- a/tox.ini +++ b/tox.ini @@ -233,6 +233,14 @@ deps = -rdoc/requirements.txt commands = sphinx-build -E -c doc/ -b html doc/ doc/build/html +[testenv:docs-autobuild] +basepython = {[testenv:default-python]basepython} +deps = + {[testenv:docs]deps} + sphinx-autobuild +commands = + sphinx-autobuild -E -c {toxinidir}/doc/ -b html {toxinidir}/doc/ {toxinidir}/doc/build/html + [testenv:serve-docs] basepython = {[testenv:default-python]basepython} skip_install = true From 9eda346937191fe9781c4d8f9a464e760ffaaf16 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 17:54:31 -0700 Subject: [PATCH 7/9] add changelog notes for string support --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6bff46e..1940d87 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,14 @@ Changelog ********* +1.0.3 -- 2018-xx-xx +=================== + +* Add support for strings on input for decoding to match functionality of :class:`base64.b64decode`. + `#21 `_ + `#23 `_ + `#24 `_ + 1.0.2 -- 2018-08-01 =================== From 6275472c83c4632eee07a43e42cab4b7ed0788eb Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 18:01:32 -0700 Subject: [PATCH 8/9] move flake8 to isort --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6d63c5d..9c9b125 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ basepython = {[testenv:default-python]basepython} deps = flake8 flake8-docstrings - flake8-import-order + flake8-isort # https://github.com/JBKahn/flake8-print/pull/30 flake8-print>=3.1.0 commands = From 1001a94b6675bb2d453e877cbcb604622c6eca97 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 2 Nov 2018 18:01:41 -0700 Subject: [PATCH 9/9] autoformat --- test/unit/test_base64io.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/unit/test_base64io.py b/test/unit/test_base64io.py index c93462d..b9a0df0 100644 --- a/test/unit/test_base64io.py +++ b/test/unit/test_base64io.py @@ -39,10 +39,9 @@ def test_file(): assert base64io.file is NotImplemented -@pytest.mark.parametrize("source, expected", ( - ('asdf', b'asdf'), - (b'\x00\x01\x02\x03', b'\x00\x01\x02\x03'), - (u'\u1111\u2222', b'\xe1\x84\x91\xe2\x88\xa2') -)) +@pytest.mark.parametrize( + "source, expected", + (("asdf", b"asdf"), (b"\x00\x01\x02\x03", b"\x00\x01\x02\x03"), (u"\u1111\u2222", b"\xe1\x84\x91\xe2\x88\xa2")), +) def test_to_bytes(source, expected): assert base64io._to_bytes(source) == expected