Merge branch 'main' into 2-arity-retry-fun

lib/req/response.ex

@@ -141,6 +141,32 @@ defmodule Req.Response do
       when is_binary(key) and is_binary(value) do
     %{response | headers: List.keystore(response.headers, key, 0, {key, value})}
   end
+
+  @doc """
+  Deletes the header given by `key`
+
+  All occurences of the header are deleted, in case the header is repeated multiple times.
+
+  ## Examples
+
+      iex> Req.Response.get_header(resp, "cache-control")
+      ["max-age=600", "no-transform"]
+      iex> resp = Req.Response.delete_header(resp, "cache-control")
+      iex> Req.Response.get_header(resp, "cache-control")
+      []
+
+  """
+  def delete_header(%Req.Response{} = response, key) when is_binary(key) do
+    %Req.Response{
+      response
+      | headers:
+          for(
+            {name, value} <- response.headers,
+            String.downcase(name) != String.downcase(key),
+            do: {name, value}
+          )
+    }
+  end
 
   @doc """
   Returns the `retry-after` header delay value or nil if not found.
@@ -193,7 +219,7 @@ defmodule Req.Response do
         valid_datetime
 
       {:error, reason} ->
-        raise "could not parse \"Retry-After\" header #{datetime} - #{reason}"
+        raise "cannot parse \"retry-after\" header value #{inspect(datetime)} as datetime, reason: #{reason}"
     end
   end
-end
+end

lib/req/steps.ex

@@ -862,7 +862,9 @@ defmodule Req.Steps do
   | _other_       | Returns data as is                              |
 
   This step updates the following headers to reflect the changes:
-  - `content-length` is set to the length of the decompressed body
+
+    * `content-length` is set to the length of the decompressed body
+    * `content-encoding` is removed
 
   ## Options
 
@@ -904,49 +906,59 @@ defmodule Req.Steps do
     if request.options[:raw] do
       {request, response}
     else
-      compression_algorithms = get_content_encoding_header(response.headers)
-      decompressed_body = decompress_body(response.body, compression_algorithms)
+      codecs = get_content_encoding_header(response.headers)
+      {decompressed_body, unknown_codecs} = decompress_body(codecs, response.body, [])
       decompressed_content_length = decompressed_body |> byte_size() |> to_string()
 
       response =
         %Req.Response{response | body: decompressed_body}
         |> Req.Response.put_header("content-length", decompressed_content_length)
 
+      response =
+        if unknown_codecs == [] do
+          Req.Response.delete_header(response, "content-encoding")
+        else
+          Req.Response.put_header(response, "content-encoding", Enum.join(unknown_codecs, ", "))
+        end
+
       {request, response}
     end
   end
 
-  defp decompress_body(body, algorithms) do
-    Enum.reduce(algorithms, body, &decompress_with_algorithm(&1, &2))
+  defp decompress_body([gzip | rest], body, acc) when gzip in ["gzip", "x-gzip"] do
+    decompress_body(rest, :zlib.gunzip(body), acc)
   end
 
-  defp decompress_with_algorithm(gzip, body) when gzip in ["gzip", "x-gzip"] do
-    :zlib.gunzip(body)
-  end
-
-  defp decompress_with_algorithm("br", body) do
+  defp decompress_body(["br" | rest], body, acc) do
     if brotli_loaded?() do
       {:ok, decompressed} = :brotli.decode(body)
-      decompressed
+      decompress_body(rest, decompressed, acc)
     else
-      raise("`:brotli` decompression library not loaded")
+      Logger.debug("decompress_body: :brotli library not loaded, skipping brotli decompression")
+      decompress_body(rest, body, ["br" | acc])
     end
   end
 
-  defp decompress_with_algorithm("zstd", body) do
+  defp decompress_body(["zstd" | rest], body, acc) do
     if ezstd_loaded?() do
-      :ezstd.decompress(body)
+      decompress_body(rest, :ezstd.decompress(body), acc)
     else
-      raise("`:ezstd` decompression library not loaded")
+      Logger.debug("decompress_body: :ezstd library not loaded, skipping zstd decompression")
+      decompress_body(rest, body, ["zstd" | acc])
     end
   end
 
-  defp decompress_with_algorithm("identity", body) do
-    body
+  defp decompress_body(["identity" | rest], body, acc) do
+    decompress_body(rest, body, acc)
+  end
+
+  defp decompress_body([codec | rest], body, acc) do
+    Logger.debug("decompress_body: algorithm #{inspect(codec)} is not supported")
+    decompress_body(rest, body, [codec | acc])
   end
 
-  defp decompress_with_algorithm(_algorithm, body) do
-    body
+  defp decompress_body([], body, acc) do
+    {body, acc}
   end
 
   defmacrop nimble_csv_loaded? do

test/req/response_test.exs

@@ -1,4 +1,4 @@
 defmodule Req.ResponseTest do
   use ExUnit.Case, async: true
-  doctest Req.Response, except: [get_header: 2, put_header: 3]
+  doctest Req.Response, except: [get_header: 2, put_header: 3, delete_header: 2]
 end

test/req/steps_test.exs

@@ -349,15 +349,16 @@ defmodule Req.StepsTest do
       assert Req.get!(c.url).body == "foo"
     end
 
-    test "unknown codec", c do
+    @tag :capture_log
+    test "unknown codecs", c do
       Bypass.expect(c.bypass, "GET", "/", fn conn ->
         conn
-        |> Plug.Conn.put_resp_header("content-encoding", "unknown")
+        |> Plug.Conn.put_resp_header("content-encoding", "unknown1, unknown2")
         |> Plug.Conn.send_resp(200, <<1, 2, 3>>)
       end)
 
       resp = Req.get!(c.url)
-      assert Req.Response.get_header(resp, "content-encoding") == ["unknown"]
+      assert Req.Response.get_header(resp, "content-encoding") == ["unknown1, unknown2"]
       assert resp.body == <<1, 2, 3>>
     end
 
@@ -371,7 +372,7 @@ defmodule Req.StepsTest do
       assert Req.head!(c.url).body == ""
     end
 
-    test "recalculate content-length when decompressing", c do
+    test "recalculate content-length header", c do
       body = "foo"
       gzipped_body = :zlib.gzip(body)
 
@@ -384,10 +385,21 @@ defmodule Req.StepsTest do
         |> Plug.Conn.send_resp(200, gzipped_body)
       end)
 
-      response = Req.get!(c.url)
-      [content_length] = Req.Response.get_header(response, "content-length")
+      resp = Req.get!(c.url)
+      [content_length] = Req.Response.get_header(resp, "content-length")
       assert String.to_integer(content_length) == byte_size(body)
     end
+
+    test "delete content-encoding header", c do
+      Bypass.expect(c.bypass, "GET", "/", fn conn ->
+        conn
+        |> Plug.Conn.put_resp_header("content-encoding", "x-gzip")
+        |> Plug.Conn.send_resp(200, :zlib.gzip("foo"))
+      end)
+
+      resp = Req.get!(c.url)
+      assert [] = Req.Response.get_header(resp, "content-encoding")
+    end
   end
 
   describe "output" do