diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 3f6eac405..e093967e8 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -389,17 +389,28 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file, disable=False): + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"} + if disable: - urllib.request.urlretrieve(url, file) + response = requests.get(url, headers=headers, stream=True) + with open(file, "wb") as f: + f.write(response.content) else: - with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t: - - def update_to(b=1, bsize=1, tsize=None): - if tsize is not None: - t.total = tsize - t.update(b * bsize - t.n) - - urllib.request.urlretrieve(url, file, reporthook=update_to) + response = requests.get(url, headers=headers, stream=True) + total_size = int(response.headers.get("content-length", 0)) + chunk_size = 1024 + + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=file, + ) as t: + with open(file, "wb") as f: + for data in response.iter_content(chunk_size=chunk_size): + f.write(data) + t.update(len(data)) def mock_snakemake( diff --git a/scripts/retrieve_jrc_idees.py b/scripts/retrieve_jrc_idees.py index 0e23b82c6..1e56e6363 100644 --- a/scripts/retrieve_jrc_idees.py +++ b/scripts/retrieve_jrc_idees.py @@ -35,7 +35,7 @@ to_fn_zp = to_fn + ".zip" # download .zip file - logger.info(f"Downloading JRC IDEES from {url_jrc}.") + logger.info(f"Downloading JRC IDEES from '{url_jrc}'.") progress_retrieve(url_jrc, to_fn_zp, disable=disable_progress) # extract