Skip to content

Commit

Permalink
Merge pull request #437 from mpenkov/fixup
Browse files Browse the repository at this point in the history
Fixup
  • Loading branch information
mpenkov committed Mar 22, 2020
2 parents 90d4f72 + 7b2ec25 commit 9c505a7
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 28 deletions.
25 changes: 13 additions & 12 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -385,21 +385,22 @@ Otherwise, the transparent decompression will not occur.
Drop-in replacement of ``pathlib.Path.open``
--------------------------------------------

Now you can natively use ``smart_open.open`` with your ``Path`` objects.
You can't transparently read text from compressed file with original ``Path.open``, but can after ``patch_pathlib``.
``smart_open.open`` can also be used with ``Path`` objects.
The built-in `Path.open()` is not able to read text from compressed files, so use ``patch_pathlib`` to replace it with `smart_open.open()` instead.
This can be helpful when e.g. working with compressed files.

.. code-block:: python
>>> from pathlib import Path
>>> from smart_open.smart_open_lib import patch_pathlib
>>>
>>> _ = patch_pathlib() # replace `Path.open` with `smart_open.open`
>>>
>>> path = Path("smart_open/tests/test_data/crime-and-punishment.txt.gz")
>>>
>>> with path.open("r") as infile:
... print(infile.readline()[:41])
В начале июля, в чрезвычайно жаркое время
>>> from pathlib import Path
>>> from smart_open.smart_open_lib import patch_pathlib
>>>
>>> _ = patch_pathlib() # replace `Path.open` with `smart_open.open`
>>>
>>> path = Path("smart_open/tests/test_data/crime-and-punishment.txt.gz")
>>>
>>> with path.open("r") as infile:
... print(infile.readline()[:41])
В начале июля, в чрезвычайно жаркое время
Comments, bug reports
=====================
Expand Down
9 changes: 4 additions & 5 deletions smart_open/smart_open_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -945,11 +945,10 @@ def __exit__(self, exc_type, exc_val, exc_tb):

def _patch_pathlib(func):
"""Replace `Path.open` with `func`"""
pathlib = sys.modules.get("pathlib", None)

if not pathlib:
raise RuntimeError("Can't patch 'pathlib.Path.open', you should import 'pathlib' first")

if not PATHLIB_SUPPORT:
raise RuntimeError('install pathlib (or pathlib2) before using this function')
if six.PY2:
raise RuntimeError('this monkey patch does not work on Py2')
old_impl = pathlib.Path.open
pathlib.Path.open = func
return old_impl
32 changes: 21 additions & 11 deletions smart_open/tests/test_smart_open.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import tempfile
import os
import hashlib
import pathlib

import boto3
import mock
Expand Down Expand Up @@ -289,7 +288,10 @@ def test_gs_uri_contains_slash(self):
self.assertEqual(parsed_uri.bucket_id, "mybucket")
self.assertEqual(parsed_uri.blob_id, "mydir/myblob")

def test_pathlib_monkeypath(self):
@unittest.skipUnless(smart_open_lib.six.PY3, "our monkey patch only works on Py3")
def test_pathlib_monkeypatch(self):
from smart_open.smart_open_lib import pathlib

assert pathlib.Path.open != smart_open.open

with patch_pathlib():
Expand All @@ -303,20 +305,30 @@ def test_pathlib_monkeypath(self):
_patch_pathlib(obj.old_impl)
assert pathlib.Path.open != smart_open.open

@unittest.skipUnless(smart_open_lib.six.PY3, "our monkey patch only works on Py3")
def test_pathlib_monkeypath_read_gz(self):
from smart_open.smart_open_lib import pathlib

path = pathlib.Path(CURR_DIR) / 'test_data' / 'crime-and-punishment.txt.gz'

# Check that standart implementation can't work with gzip
# Check that standard implementation can't work with gzip
with path.open("r") as infile:
with self.assertRaises(Exception) as context:
with self.assertRaises(Exception):
lines = infile.readlines()

# Check that out implementation works with gzip
# Check that our implementation works with gzip
obj = patch_pathlib()
with path.open("r") as infile:
lines = infile.readlines()
try:
with path.open("r") as infile:
lines = infile.readlines()
self.assertEqual(len(lines), 3)
finally:
_patch_pathlib(obj.old_impl)

_patch_pathlib(obj.old_impl)
@unittest.skipUnless(smart_open_lib.six.PY2, 'this test is for Py2 only')
def test_monkey_patch_raises_exception_py2(self):
with self.assertRaises(RuntimeError):
patch_pathlib()


class SmartOpenHttpTest(unittest.TestCase):
Expand Down Expand Up @@ -637,9 +649,7 @@ def test_open_with_keywords_explicit_r(self):
actual = fin.read()
self.assertEqual(expected, actual)

@unittest.skipUnless(
smart_open_lib.PATHLIB_SUPPORT,
"do not test pathlib support if pathlib or backport are not available")
@unittest.skipUnless(smart_open_lib.PATHLIB_SUPPORT, "this test requires pathlib")
def test_open_and_read_pathlib_path(self):
"""If ``pathlib.Path`` is available we should be able to open and read."""
from smart_open.smart_open_lib import pathlib
Expand Down

0 comments on commit 9c505a7

Please sign in to comment.