Skip to content

Commit

Permalink
Hmm
Browse files Browse the repository at this point in the history
  • Loading branch information
palewire committed May 27, 2022
1 parent f5fd8b7 commit 11b2f1a
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 11 deletions.
2 changes: 2 additions & 0 deletions Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ click = "*"
"flake8" = "*"
twine = "*"
pylint = "*"
bs4 = "*"
html5lib = "*"

[requires]
python_version = "3.9"
Expand Down
39 changes: 35 additions & 4 deletions Pipfile.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 9 additions & 6 deletions archiveis/api.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env python
import logging
import urllib.parse

import click
import requests
Expand All @@ -17,26 +18,28 @@ def capture(
Returns the URL where the capture is stored.
"""
# The site changes its TLD all the time. Cache what we get currently.
domain = requests.get("https://archive.is").url

# Put together the URL that will save our request
domain = "https://archive.is"
save_url = domain + "/submit/"
save_url = domain + "submit/"

# Configure the request headers
headers = {
"User-Agent": user_agent,
"host": "archive.is",
"host": urllib.parse.urlparse(domain).netloc,
}

# Request a unique identifier for our activity
logger.debug("Requesting {}".format(domain + "/"))
logger.debug(f"Requesting {domain}")
get_kwargs = dict(
timeout=120,
allow_redirects=True,
headers=headers,
)
if proxies:
get_kwargs["proxies"] = proxies
response = requests.get(domain + "/", **get_kwargs)
response = requests.get(domain, **get_kwargs)
response.raise_for_status()

# It will need to be parsed from the homepage response headers
Expand All @@ -47,7 +50,7 @@ def capture(
)
logger.debug(f"Unique identifier: {unique_id}")
except IndexError:
logger.warn(
logger.warning(
"Unable to extract unique identifier from archive.is. Submitting without it."
)
unique_id = None
Expand Down
5 changes: 4 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@ def read(fname):
"Programming Language :: Python :: 3.8",
"License :: OSI Approved :: MIT License",
],
install_requires=["requests>=2.22.0", "click"],
install_requires=[
"requests",
"click",
],
entry_points="""
[console_scripts]
archiveis=archiveis.api:cli
Expand Down

0 comments on commit 11b2f1a

Please sign in to comment.