diff --git a/lib/req/response.ex b/lib/req/response.ex index 16b08d87..82732894 100644 --- a/lib/req/response.ex +++ b/lib/req/response.ex @@ -141,6 +141,32 @@ defmodule Req.Response do when is_binary(key) and is_binary(value) do %{response | headers: List.keystore(response.headers, key, 0, {key, value})} end + + @doc """ + Deletes the header given by `key` + + All occurences of the header are deleted, in case the header is repeated multiple times. + + ## Examples + + iex> Req.Response.get_header(resp, "cache-control") + ["max-age=600", "no-transform"] + iex> resp = Req.Response.delete_header(resp, "cache-control") + iex> Req.Response.get_header(resp, "cache-control") + [] + + """ + def delete_header(%Req.Response{} = response, key) when is_binary(key) do + %Req.Response{ + response + | headers: + for( + {name, value} <- response.headers, + String.downcase(name) != String.downcase(key), + do: {name, value} + ) + } + end @doc """ Returns the `retry-after` header delay value or nil if not found. @@ -193,7 +219,7 @@ defmodule Req.Response do valid_datetime {:error, reason} -> - raise "could not parse \"Retry-After\" header #{datetime} - #{reason}" + raise "cannot parse \"retry-after\" header value #{inspect(datetime)} as datetime, reason: #{reason}" end end -end +end \ No newline at end of file diff --git a/lib/req/steps.ex b/lib/req/steps.ex index 193f8869..d43dc0b0 100644 --- a/lib/req/steps.ex +++ b/lib/req/steps.ex @@ -862,7 +862,9 @@ defmodule Req.Steps do | _other_ | Returns data as is | This step updates the following headers to reflect the changes: - - `content-length` is set to the length of the decompressed body + + * `content-length` is set to the length of the decompressed body + * `content-encoding` is removed ## Options @@ -904,49 +906,59 @@ defmodule Req.Steps do if request.options[:raw] do {request, response} else - compression_algorithms = get_content_encoding_header(response.headers) - decompressed_body = decompress_body(response.body, compression_algorithms) + codecs = get_content_encoding_header(response.headers) + {decompressed_body, unknown_codecs} = decompress_body(codecs, response.body, []) decompressed_content_length = decompressed_body |> byte_size() |> to_string() response = %Req.Response{response | body: decompressed_body} |> Req.Response.put_header("content-length", decompressed_content_length) + response = + if unknown_codecs == [] do + Req.Response.delete_header(response, "content-encoding") + else + Req.Response.put_header(response, "content-encoding", Enum.join(unknown_codecs, ", ")) + end + {request, response} end end - defp decompress_body(body, algorithms) do - Enum.reduce(algorithms, body, &decompress_with_algorithm(&1, &2)) + defp decompress_body([gzip | rest], body, acc) when gzip in ["gzip", "x-gzip"] do + decompress_body(rest, :zlib.gunzip(body), acc) end - defp decompress_with_algorithm(gzip, body) when gzip in ["gzip", "x-gzip"] do - :zlib.gunzip(body) - end - - defp decompress_with_algorithm("br", body) do + defp decompress_body(["br" | rest], body, acc) do if brotli_loaded?() do {:ok, decompressed} = :brotli.decode(body) - decompressed + decompress_body(rest, decompressed, acc) else - raise("`:brotli` decompression library not loaded") + Logger.debug("decompress_body: :brotli library not loaded, skipping brotli decompression") + decompress_body(rest, body, ["br" | acc]) end end - defp decompress_with_algorithm("zstd", body) do + defp decompress_body(["zstd" | rest], body, acc) do if ezstd_loaded?() do - :ezstd.decompress(body) + decompress_body(rest, :ezstd.decompress(body), acc) else - raise("`:ezstd` decompression library not loaded") + Logger.debug("decompress_body: :ezstd library not loaded, skipping zstd decompression") + decompress_body(rest, body, ["zstd" | acc]) end end - defp decompress_with_algorithm("identity", body) do - body + defp decompress_body(["identity" | rest], body, acc) do + decompress_body(rest, body, acc) + end + + defp decompress_body([codec | rest], body, acc) do + Logger.debug("decompress_body: algorithm #{inspect(codec)} is not supported") + decompress_body(rest, body, [codec | acc]) end - defp decompress_with_algorithm(_algorithm, body) do - body + defp decompress_body([], body, acc) do + {body, acc} end defmacrop nimble_csv_loaded? do diff --git a/test/req/response_test.exs b/test/req/response_test.exs index c477c2cc..27c89e4e 100644 --- a/test/req/response_test.exs +++ b/test/req/response_test.exs @@ -1,4 +1,4 @@ defmodule Req.ResponseTest do use ExUnit.Case, async: true - doctest Req.Response, except: [get_header: 2, put_header: 3] + doctest Req.Response, except: [get_header: 2, put_header: 3, delete_header: 2] end diff --git a/test/req/steps_test.exs b/test/req/steps_test.exs index f57fff68..625d3f7b 100644 --- a/test/req/steps_test.exs +++ b/test/req/steps_test.exs @@ -349,15 +349,16 @@ defmodule Req.StepsTest do assert Req.get!(c.url).body == "foo" end - test "unknown codec", c do + @tag :capture_log + test "unknown codecs", c do Bypass.expect(c.bypass, "GET", "/", fn conn -> conn - |> Plug.Conn.put_resp_header("content-encoding", "unknown") + |> Plug.Conn.put_resp_header("content-encoding", "unknown1, unknown2") |> Plug.Conn.send_resp(200, <<1, 2, 3>>) end) resp = Req.get!(c.url) - assert Req.Response.get_header(resp, "content-encoding") == ["unknown"] + assert Req.Response.get_header(resp, "content-encoding") == ["unknown1, unknown2"] assert resp.body == <<1, 2, 3>> end @@ -371,7 +372,7 @@ defmodule Req.StepsTest do assert Req.head!(c.url).body == "" end - test "recalculate content-length when decompressing", c do + test "recalculate content-length header", c do body = "foo" gzipped_body = :zlib.gzip(body) @@ -384,10 +385,21 @@ defmodule Req.StepsTest do |> Plug.Conn.send_resp(200, gzipped_body) end) - response = Req.get!(c.url) - [content_length] = Req.Response.get_header(response, "content-length") + resp = Req.get!(c.url) + [content_length] = Req.Response.get_header(resp, "content-length") assert String.to_integer(content_length) == byte_size(body) end + + test "delete content-encoding header", c do + Bypass.expect(c.bypass, "GET", "/", fn conn -> + conn + |> Plug.Conn.put_resp_header("content-encoding", "x-gzip") + |> Plug.Conn.send_resp(200, :zlib.gzip("foo")) + end) + + resp = Req.get!(c.url) + assert [] = Req.Response.get_header(resp, "content-encoding") + end end describe "output" do