Merge pull request #24 from mattsb42-aws/unicode

Cleaning up string support and expanding test coverage.

.gitignore

@@ -19,4 +19,5 @@ docs/build
 .pytest_cache
 
 # PyCharm
-.idea/
+.idea/
+venv/

CHANGELOG.rst

@@ -2,6 +2,14 @@
 Changelog
 *********
 
+1.0.3 -- 2018-xx-xx
+===================
+
+* Add support for strings on input for decoding to match functionality of :class:`base64.b64decode`.
+  `#21 <https://github.com/aws/base64io-python/issues/21>`_
+  `#23 <https://github.com/aws/base64io-python/pull/23>`_
+  `#24 <https://github.com/aws/base64io-python/pull/24>`_
+
 1.0.2 -- 2018-08-01
 ===================
 

doc/conf.py

@@ -1,9 +1,9 @@
 # pylint: disable=invalid-name
 """Sphinx configuration."""
-from datetime import datetime
 import io
 import os
 import re
+from datetime import datetime
 
 VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""")
 HERE = os.path.abspath(os.path.dirname(__file__))

setup.cfg

@@ -39,3 +39,14 @@ ignore =
 # Doc8 Configuration
 [doc8]
 max-line-length = 120
+
+[isort]
+line_length = 120
+# https://github.com/timothycrosley/isort#multi-line-output-modes
+multi_line_output = 3
+include_trailing_comma = True
+force_grid_wrap = 0
+combine_as_imports = True
+not_skip = __init__.py
+known_first_party = base64io
+known_third_party =base64io,mock,pytest,setuptools

src/base64io/__init__.py

@@ -52,6 +52,19 @@ def _py2():
     file = NotImplemented  # pylint: disable=invalid-name
 
 
+def _to_bytes(data):
+    # type: (AnyStr) -> bytes
+    """Convert input data from either string or bytes to bytes.
+
+    :param data: Data to convert
+    :returns: ``data`` converted to bytes
+    :rtype: bytes
+    """
+    if isinstance(data, bytes):
+        return data
+    return data.encode("utf-8")
+
+
 class Base64IO(io.IOBase):
     """Base64 stream with context manager support.
 
@@ -209,7 +222,7 @@ def writelines(self, lines):
             self.write(line)
 
     def _read_additional_data_removing_whitespace(self, data, total_bytes_to_read):
-        # type: (AnyStr, int) -> AnyStr
+        # type: (bytes, int) -> bytes
         """Read additional data from wrapped stream until we reach the desired number of bytes.
 
         .. note::
@@ -226,20 +239,20 @@ def _read_additional_data_removing_whitespace(self, data, total_bytes_to_read):
             # case the base64 module happily removes any whitespace.
             return data
 
-        _data_buffer = io.BytesIO() if isinstance(data, bytes) else io.StringIO()
-        join_char = b'' if isinstance(data, bytes) else u''
-        _data_buffer.write(join_char.join(data.split()))  # type: ignore
-        _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell()  # type: ignore
+        _data_buffer = io.BytesIO()
+
+        _data_buffer.write(b"".join(data.split()))
+        _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell()
 
         while _remaining_bytes_to_read > 0:
-            _raw_additional_data = self.__wrapped.read(_remaining_bytes_to_read)
+            _raw_additional_data = _to_bytes(self.__wrapped.read(_remaining_bytes_to_read))
             if not _raw_additional_data:
                 # No more data to read from wrapped stream.
                 break
 
-            _data_buffer.write(join_char.join(_raw_additional_data.split()))  # type: ignore
-            _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell()  # type: ignore
-        return _data_buffer.getvalue()  # type: ignore
+            _data_buffer.write(b"".join(_raw_additional_data.split()))
+            _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell()
+        return _data_buffer.getvalue()
 
     def read(self, b=-1):
         # type: (int) -> bytes
@@ -271,13 +284,11 @@ def read(self, b=-1):
             _bytes_to_read += 4 - _bytes_to_read % 4
 
         # Read encoded bytes from wrapped stream.
-        data = self.__wrapped.read(_bytes_to_read)
+        data = _to_bytes(self.__wrapped.read(_bytes_to_read))
         # Remove whitespace from read data and attempt to read more data to get the desired
         # number of bytes.
-        whitespace = string.whitespace.encode("utf-8") if isinstance(data, bytes) \
-            else string.whitespace  # type: Union[bytes, str]
 
-        if any([char in data for char in whitespace]):
+        if any([char in data for char in string.whitespace.encode("utf-8")]):
             data = self._read_additional_data_removing_whitespace(data, _bytes_to_read)
 
         results = io.BytesIO()

test/unit/test_base64_stream.py

@@ -19,8 +19,8 @@
 import math
 import os
 
-from mock import MagicMock, sentinel
 import pytest
+from mock import MagicMock, sentinel
 
 from base64io import Base64IO
 
@@ -154,12 +154,13 @@ def test_base64io_decode(bytes_to_generate, bytes_per_round, number_of_rounds, t
     assert test == plaintext_source[:total_bytes_to_expect]
 
 
+@pytest.mark.parametrize("encoding", ("ascii", "utf-8"))
 @pytest.mark.parametrize(
     "bytes_to_generate, bytes_per_round, number_of_rounds, total_bytes_to_expect", build_test_cases()
 )
-def test_base64io_decode_str(bytes_to_generate, bytes_per_round, number_of_rounds, total_bytes_to_expect):
+def test_base64io_decode_str(encoding, bytes_to_generate, bytes_per_round, number_of_rounds, total_bytes_to_expect):
     plaintext_source = os.urandom(bytes_to_generate)
-    plaintext_b64 = io.StringIO(base64.b64encode(plaintext_source).decode('ascii'))
+    plaintext_b64 = io.StringIO(base64.b64encode(plaintext_source).decode(encoding))
     plaintext_wrapped = Base64IO(plaintext_b64)
 
     test = b""
@@ -313,9 +314,10 @@ def test_base64io_decode_with_whitespace(plaintext_source, b64_plaintext_with_wh
     assert test == plaintext_source[:read_bytes]
 
 
+@pytest.mark.parametrize("encoding", ("ascii", "utf-8"))
 @pytest.mark.parametrize("plaintext_source, b64_plaintext_with_whitespace, read_bytes", build_whitespace_testcases())
-def test_base64io_decode_with_whitespace_str(plaintext_source, b64_plaintext_with_whitespace, read_bytes):
-    with Base64IO(io.StringIO(b64_plaintext_with_whitespace.decode('ascii'))) as decoder:
+def test_base64io_decode_with_whitespace_str(encoding, plaintext_source, b64_plaintext_with_whitespace, read_bytes):
+    with Base64IO(io.StringIO(b64_plaintext_with_whitespace.decode(encoding))) as decoder:
         test = decoder.read(read_bytes)
 
     assert test == plaintext_source[:read_bytes]

test/unit/test_base64io.py

@@ -32,8 +32,16 @@ def test_file():
     if is_python2:
         # If we are in Python 2, the "file" assignment should not
         # happen because it is a builtin object.
-        assert not hasattr(base64io, 'file')
+        assert not hasattr(base64io, "file")
     else:
         # If we are in Python 3, the "file" assignment should happen
         # to provide a concrete definition of the "file" name.
         assert base64io.file is NotImplemented
+
+
+@pytest.mark.parametrize(
+    "source, expected",
+    (("asdf", b"asdf"), (b"\x00\x01\x02\x03", b"\x00\x01\x02\x03"), (u"\u1111\u2222", b"\xe1\x84\x91\xe2\x88\xa2")),
+)
+def test_to_bytes(source, expected):
+    assert base64io._to_bytes(source) == expected

tox.ini

@@ -74,7 +74,7 @@ basepython = {[testenv:default-python]basepython}
 deps =
     flake8
     flake8-docstrings
-    flake8-import-order
+    flake8-isort
     # https://github.com/JBKahn/flake8-print/pull/30
     flake8-print>=3.1.0
 commands =
@@ -129,6 +129,35 @@ deps =
 commands =
     {[testenv:blacken-src]commands} --diff
 
+[testenv:isort-seed]
+basepython = python3
+deps = seed-isort-config
+commands = seed-isort-config
+
+[testenv:isort]
+basepython = python3
+deps = isort
+commands = isort -rc \
+    src \
+    test \
+    doc \
+    setup.py \
+    {posargs}
+
+[testenv:isort-check]
+basepython = python3
+deps = {[testenv:isort]deps}
+commands = {[testenv:isort]commands} -c
+
+[testenv:autoformat]
+basepython = python3
+deps =
+    {[testenv:blacken]deps}
+    {[testenv:isort]deps}
+commands =
+    {[testenv:blacken]commands}
+    {[testenv:isort]commands}
+
 [testenv:pylint]
 basepython = {[testenv:default-python]basepython}
 deps =
@@ -204,6 +233,14 @@ deps = -rdoc/requirements.txt
 commands =
     sphinx-build -E -c doc/ -b html doc/ doc/build/html
 
+[testenv:docs-autobuild]
+basepython = {[testenv:default-python]basepython}
+deps =
+    {[testenv:docs]deps}
+    sphinx-autobuild
+commands =
+    sphinx-autobuild -E -c {toxinidir}/doc/ -b html {toxinidir}/doc/ {toxinidir}/doc/build/html
+
 [testenv:serve-docs]
 basepython = {[testenv:default-python]basepython}
 skip_install = true