Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add progress to streaming download #1268

Merged
merged 11 commits into from
Sep 10, 2020
22 changes: 22 additions & 0 deletions docs/advanced.md
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,28 @@ with httpx.Client(headers=headers) as client:
...
```

## Monitoring download progress

If you need to monitor download progress, you can stream with using `response.last_raw_chunk_size` property.
cdeler marked this conversation as resolved.
Show resolved Hide resolved

tomchristie marked this conversation as resolved.
Show resolved Hide resolved
For example, you can build a nice progress bar using the `tqdm` library:
cdeler marked this conversation as resolved.
Show resolved Hide resolved

```python
import tempfile

import httpx
from tqdm import tqdm

with tempfile.NamedTemporaryFile() as download_file:
data = b"@" * 1000000
with httpx.stream("POST", "https://httpbin.org/anything", data=data) as response:
florimondmanca marked this conversation as resolved.
Show resolved Hide resolved
content_length = int(response.headers["Content-Length"])
florimondmanca marked this conversation as resolved.
Show resolved Hide resolved
with tqdm(total=content_length) as progress:
florimondmanca marked this conversation as resolved.
Show resolved Hide resolved
for chunk in response.iter_bytes():
download_file.write(chunk)
progress.update(response.last_raw_chunk_size)
```

## .netrc Support

HTTPX supports .netrc file. In `trust_env=True` cases, if auth parameter is
Expand Down
10 changes: 10 additions & 0 deletions httpx/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,8 @@ def __init__(
self._raw_stream = ByteStream(body=content or b"")
self.read()

self._last_raw_chunk_size = 0

@property
def elapsed(self) -> datetime.timedelta:
"""
Expand Down Expand Up @@ -885,6 +887,10 @@ def links(self) -> typing.Dict[typing.Optional[str], typing.Dict[str, str]]:
ldict[key] = link
return ldict

@property
def last_raw_chunk_size(self) -> int:
return self._last_raw_chunk_size

def __repr__(self) -> str:
return f"<Response [{self.status_code} {self.reason_phrase}]>"

Expand Down Expand Up @@ -951,8 +957,10 @@ def iter_raw(self) -> typing.Iterator[bytes]:
raise ResponseClosed()

self.is_stream_consumed = True
self._last_raw_chunk_size = 0
with map_exceptions(HTTPCORE_EXC_MAP, request=self._request):
for part in self._raw_stream:
self._last_raw_chunk_size = len(part)
yield part
self.close()

Expand Down Expand Up @@ -1032,8 +1040,10 @@ async def aiter_raw(self) -> typing.AsyncIterator[bytes]:
raise ResponseClosed()

self.is_stream_consumed = True
self._last_raw_chunk_size = 0
with map_exceptions(HTTPCORE_EXC_MAP, request=self._request):
async for part in self._raw_stream:
self._last_raw_chunk_size = len(part)
yield part
await self.aclose()

Expand Down
25 changes: 25 additions & 0 deletions tests/models/test_responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,18 @@ def test_iter_raw():
assert raw == b"Hello, world!"


def test_iter_raw_increments_updates_counter():
stream = IteratorStream(iterator=streaming_body())

response = httpx.Response(
200,
stream=stream,
)

for part in response.iter_raw():
assert len(part) == response.last_raw_chunk_size


@pytest.mark.asyncio
async def test_aiter_raw():
stream = AsyncIteratorStream(aiterator=async_streaming_body())
Expand All @@ -241,6 +253,19 @@ async def test_aiter_raw():
assert raw == b"Hello, world!"


@pytest.mark.asyncio
async def test_aiter_raw_increments_updates_counter():
stream = AsyncIteratorStream(aiterator=async_streaming_body())

response = httpx.Response(
200,
stream=stream,
)

async for part in response.aiter_raw():
assert len(part) == response.last_raw_chunk_size


def test_iter_bytes():
response = httpx.Response(
200,
Expand Down