Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for gzip content-encoding #776

Merged
merged 6 commits into from
Mar 9, 2022
Merged

Add support for gzip content-encoding #776

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
291894f
Add support for gzip Content-Encoding
ivan-valkov Feb 17, 2022
c2dc1aa
remove the case insensitive check for accept header
ivan-valkov Feb 24, 2022
0f05084
Add option to disable compression
ivan-valkov Feb 24, 2022
a46aaf4
sMake disable_compression an argument instead of env var
ivan-valkov Mar 2, 2022
9974c46
Update readme
ivan-valkov Mar 2, 2022
245c3be
Fix linters
ivan-valkov Mar 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions prometheus_client/asgi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,25 @@ async def prometheus_app(scope, receive, send):
params = parse_qs(scope.get('query_string', b''))
accept_header = "Accept: " + ",".join([
value.decode("utf8") for (name, value) in scope.get('headers')
if name.decode("utf8") == 'accept'
if name.decode("utf8").lower() == 'accept'
])
accept_encoding_header = ",".join([
value.decode("utf8") for (name, value) in scope.get('headers')
if name.decode("utf8").lower() == 'accept-encoding'
])
# Bake output
status, header, output = _bake_output(registry, accept_header, params)
status, headers, output = _bake_output(registry, accept_header, accept_encoding_header, params)
formatted_headers = []
for header in headers:
formatted_headers.append(tuple(x.encode('utf8') for x in header))
# Return output
payload = await receive()
if payload.get("type") == "http.request":
await send(
{
"type": "http.response.start",
"status": int(status.split(' ')[0]),
"headers": [
tuple(x.encode('utf8') for x in header)
]
"headers": formatted_headers,
}
)
await send({"type": "http.response.body", "body": output})
Expand Down
144 changes: 86 additions & 58 deletions prometheus_client/exposition.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import base64
from contextlib import closing
import gzip
from http.server import BaseHTTPRequestHandler
import os
import socket
Expand Down Expand Up @@ -38,6 +39,13 @@
PYTHON376_OR_NEWER = sys.version_info > (3, 7, 5)


def _get_disable_compression() -> bool:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of an environment variable what would you think of passing an additional argument to make_(w|a)sgi_app? My thought is that most users should have control over creating the apps so an env var is not necessary. We try to only use env vars in places where users might not have access to the code, such as registering metrics.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, makes sense. I suppose we need to expose this option in start_http_server as well?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that would be nice, though not completely required as start_http_server doesn't need to support all configuration for advanced user.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

awesome, yeah, this makes sense. I have updated the PR now and added documentation to the README. PTAL when you get some time. I don't think there is a release notes file we need to update but maybe worth mentioning this change when releasing the next version of the client (:

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! I plan to review this early next week. I will definitely mention it in the release notes for the next version!

return os.environ.get("PROMETHEUS_DISABLE_COMPRESSION", 'False').lower() in ('true', '1', 't')


_disable_compression = _get_disable_compression()


class _PrometheusRedirectHandler(HTTPRedirectHandler):
"""
Allow additional methods (e.g. PUT) and data forwarding in redirects.
Expand Down Expand Up @@ -93,13 +101,19 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
return new_request


def _bake_output(registry, accept_header, params):
def _bake_output(registry, accept_header, accept_encoding_header, params):
"""Bake output for metrics output."""
encoder, content_type = choose_encoder(accept_header)
# Choose the correct plain text format of the output.
formatter, content_type = choose_formatter(accept_header)
if 'name[]' in params:
registry = registry.restricted_registry(params['name[]'])
output = encoder(registry)
return '200 OK', ('Content-Type', content_type), output
output = formatter(registry)
headers = [('Content-Type', content_type)]
# If gzip encoding required, gzip the output.
if gzip_accepted(accept_encoding_header):
output = gzip.compress(output)
headers.append(('Content-Encoding', 'gzip'))
return '200 OK', headers, output


def make_wsgi_app(registry: CollectorRegistry = REGISTRY) -> Callable:
Expand All @@ -108,17 +122,18 @@ def make_wsgi_app(registry: CollectorRegistry = REGISTRY) -> Callable:
def prometheus_app(environ, start_response):
# Prepare parameters
accept_header = environ.get('HTTP_ACCEPT')
accept_encoding_header = environ.get('HTTP_ACCEPT_ENCODING')
params = parse_qs(environ.get('QUERY_STRING', ''))
if environ['PATH_INFO'] == '/favicon.ico':
# Serve empty response for browsers
status = '200 OK'
header = ('', '')
headers = [('', '')]
output = b''
else:
# Bake output
status, header, output = _bake_output(registry, accept_header, params)
status, headers, output = _bake_output(registry, accept_header, accept_encoding_header, params)
# Return output
start_response(status, [header])
start_response(status, headers)
return [output]

return prometheus_app
Expand Down Expand Up @@ -152,8 +167,10 @@ def _get_best_family(address, port):

def start_wsgi_server(port: int, addr: str = '0.0.0.0', registry: CollectorRegistry = REGISTRY) -> None:
"""Starts a WSGI server for prometheus metrics as a daemon thread."""

class TmpServer(ThreadingWSGIServer):
"""Copy of ThreadingWSGIServer to update address_family locally"""

TmpServer.address_family, addr = _get_best_family(addr, port)
app = make_wsgi_app(registry)
httpd = make_server(addr, port, app, TmpServer, handler_class=_SilentHandler)
Expand Down Expand Up @@ -227,7 +244,7 @@ def sample_line(line):
return ''.join(output).encode('utf-8')


def choose_encoder(accept_header: str) -> Tuple[Callable[[CollectorRegistry], bytes], str]:
def choose_formatter(accept_header: str) -> Tuple[Callable[[CollectorRegistry], bytes], str]:
accept_header = accept_header or ''
for accepted in accept_header.split(','):
if accepted.split(';')[0].strip() == 'application/openmetrics-text':
Expand All @@ -236,6 +253,16 @@ def choose_encoder(accept_header: str) -> Tuple[Callable[[CollectorRegistry], by
return generate_latest, CONTENT_TYPE_LATEST


def gzip_accepted(accept_encoding_header: str) -> bool:
if _disable_compression:
return False
accept_encoding_header = accept_encoding_header or ''
for accepted in accept_encoding_header.split(','):
if accepted.split(';')[0].strip().lower() == 'gzip':
return True
return False


class MetricsHandler(BaseHTTPRequestHandler):
"""HTTP handler that gives metrics from ``REGISTRY``."""
registry: CollectorRegistry = REGISTRY
Expand All @@ -244,12 +271,14 @@ def do_GET(self) -> None:
# Prepare parameters
registry = self.registry
accept_header = self.headers.get('Accept')
accept_encoding_header = self.headers.get('Accept-Encoding')
params = parse_qs(urlparse(self.path).query)
# Bake output
status, header, output = _bake_output(registry, accept_header, params)
status, headers, output = _bake_output(registry, accept_header, accept_encoding_header, params)
# Return output
self.send_response(int(status.split(' ')[0]))
self.send_header(*header)
for header in headers:
self.send_header(*header)
self.end_headers()
self.wfile.write(output)

Expand Down Expand Up @@ -289,14 +318,13 @@ def write_to_textfile(path: str, registry: CollectorRegistry) -> None:


def _make_handler(
url: str,
method: str,
timeout: Optional[float],
headers: Sequence[Tuple[str, str]],
data: bytes,
base_handler: type,
url: str,
method: str,
timeout: Optional[float],
headers: Sequence[Tuple[str, str]],
data: bytes,
base_handler: type,
) -> Callable[[], None]:

def handle() -> None:
request = Request(url, data=data)
request.get_method = lambda: method # type: ignore
Expand All @@ -310,11 +338,11 @@ def handle() -> None:


def default_handler(
url: str,
method: str,
timeout: Optional[float],
headers: List[Tuple[str, str]],
data: bytes,
url: str,
method: str,
timeout: Optional[float],
headers: List[Tuple[str, str]],
data: bytes,
) -> Callable[[], None]:
"""Default handler that implements HTTP/HTTPS connections.

Expand All @@ -324,11 +352,11 @@ def default_handler(


def passthrough_redirect_handler(
url: str,
method: str,
timeout: Optional[float],
headers: List[Tuple[str, str]],
data: bytes,
url: str,
method: str,
timeout: Optional[float],
headers: List[Tuple[str, str]],
data: bytes,
) -> Callable[[], None]:
"""
Handler that automatically trusts redirect responses for all HTTP methods.
Expand All @@ -344,13 +372,13 @@ def passthrough_redirect_handler(


def basic_auth_handler(
url: str,
method: str,
timeout: Optional[float],
headers: List[Tuple[str, str]],
data: bytes,
username: str = None,
password: str = None,
url: str,
method: str,
timeout: Optional[float],
headers: List[Tuple[str, str]],
data: bytes,
username: str = None,
password: str = None,
) -> Callable[[], None]:
"""Handler that implements HTTP/HTTPS connections with Basic Auth.

Expand All @@ -371,12 +399,12 @@ def handle():


def push_to_gateway(
gateway: str,
job: str,
registry: CollectorRegistry,
grouping_key: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = 30,
handler: Callable = default_handler,
gateway: str,
job: str,
registry: CollectorRegistry,
grouping_key: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = 30,
handler: Callable = default_handler,
) -> None:
"""Push metrics to the given pushgateway.

Expand Down Expand Up @@ -420,12 +448,12 @@ def push_to_gateway(


def pushadd_to_gateway(
gateway: str,
job: str,
registry: Optional[CollectorRegistry],
grouping_key: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = 30,
handler: Callable = default_handler,
gateway: str,
job: str,
registry: Optional[CollectorRegistry],
grouping_key: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = 30,
handler: Callable = default_handler,
) -> None:
"""PushAdd metrics to the given pushgateway.

Expand All @@ -451,11 +479,11 @@ def pushadd_to_gateway(


def delete_from_gateway(
gateway: str,
job: str,
grouping_key: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = 30,
handler: Callable = default_handler,
gateway: str,
job: str,
grouping_key: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = 30,
handler: Callable = default_handler,
) -> None:
"""Delete metrics from the given pushgateway.

Expand All @@ -480,13 +508,13 @@ def delete_from_gateway(


def _use_gateway(
method: str,
gateway: str,
job: str,
registry: Optional[CollectorRegistry],
grouping_key: Optional[Dict[str, Any]],
timeout: Optional[float],
handler: Callable,
method: str,
gateway: str,
job: str,
registry: Optional[CollectorRegistry],
grouping_key: Optional[Dict[str, Any]],
timeout: Optional[float],
handler: Callable,
) -> None:
gateway_url = urlparse(gateway)
# See https://bugs.python.org/issue27657 for details on urlparse in py>=3.7.6.
Expand Down
Loading