Merge branch 'main' into main

config/config.exs

@@ -37,7 +37,7 @@ config :bob,
   agent_schedule: [
     [
       module: Bob.Job.Clean,
-      period: {24, :hour},
+      period: {1, :hour},
       queue: true
     ]
   ]

lib/bob/docker_hub.ex

@@ -17,6 +17,11 @@ defmodule Bob.DockerHub do
   end
 
   def fetch_repo_tags(repo) do
+    (@dockerhub_url <> "v2/repositories/#{repo}/tags?page=${page}&page_size=100")
+    |> dockerhub_request()
+  end
+
+  def fetch_repo_tags_from_cache(repo) do
     Bob.DockerHub.Cache.lookup(repo, fn ->
       (@dockerhub_url <> "v2/repositories/#{repo}/tags?page=${page}&page_size=100")
       |> dockerhub_request()
@@ -44,6 +49,22 @@ defmodule Bob.DockerHub do
     end
   end
 
+  def delete_tag(repo, tag) do
+    url = @dockerhub_url <> "v2/repositories/#{repo}/tags/#{tag}"
+    headers = headers()
+    opts = [:with_body, recv_timeout: 20_000]
+
+    result =
+      Bob.HTTP.retry("DockerHub #{url}", fn ->
+        :hackney.request(:delete, url, headers, "", opts)
+      end)
+
+    case result do
+      {:ok, 204, _headers, _body} -> :ok
+      {:ok, 404, _headers, _body} -> :ok
+    end
+  end
+
   def headers() do
     if token = Application.get_env(:bob, :dockerhub_token) do
       [{"authorization", "JWT #{token}"}]

lib/bob/docker_hub/auth.ex

@@ -1,7 +1,7 @@
 defmodule Bob.DockerHub.Auth do
   use GenServer
 
-  @timeout 24 * 60 * 60 * 1000
+  @timeout 60 * 60 * 1000
 
   def start_link([]) do
     GenServer.start_link(__MODULE__, [])

lib/bob/http.ex

@@ -12,7 +12,7 @@ defmodule Bob.HTTP do
   defp retry(name, fun, times) do
     case fun.() do
       {:error, reason} ->
-        Logger.warn("#{name} ERROR: #{inspect(reason)}")
+        Logger.warning("#{name} ERROR: #{inspect(reason)}")
 
         if times + 1 < @max_retry_times do
           sleep = trunc(:math.pow(3, times) * @error_sleep_time)
@@ -23,7 +23,7 @@ defmodule Bob.HTTP do
         end
 
       {:ok, 429, _headers, _body} = result ->
-        Logger.warn("#{name} RATE LIMIT")
+        Logger.warning("#{name} RATE LIMIT")
 
         if times + 1 < @max_retry_times do
           sleep = trunc(:math.pow(3, times) * @rate_limit_sleep_time)

lib/bob/job/docker_checker.ex

@@ -4,34 +4,60 @@ defmodule Bob.Job.DockerChecker do
 
   @archs ["amd64", "arm64"]
 
-  # TODO: Automate picking the OS versions
-
-  @builds %{
-    "alpine" => [
-      "3.16.6",
-      "3.17.4",
-      "3.18.2"
-    ],
-    "ubuntu" => [
-      # 22.04
-      "jammy-20230126",
-      # 20.04
-      "focal-20230126",
-      # 18.04
-      "bionic-20230126"
-    ],
-    "debian" => [
-      # 12
-      "bookworm-20230612",
-      "bookworm-20230612-slim",
-      # 11
-      "bullseye-20230612",
-      "bullseye-20230612-slim",
-      # 10
-      "buster-20230612",
-      "buster-20230612-slim"
+  def builds() do
+    [
+      {"alpine",
+       [
+         ~r/^3\.19\.\d+$/,
+         ~r/^3\.18\.\d+$/,
+         ~r/^3\.17\.\d+$/,
+         ~r/^3\.16\.\d+$/
+       ]},
+      {"ubuntu",
+       [
+         # 24.04
+         ~r/^noble-\d{8}$/,
+         # 22.04
+         ~r/^jammy-\d{8}$/,
+         # 20.04
+         ~r/^focal-\d{8}$/
+       ]},
+      {"debian",
+       [
+         # 12
+         ~r/^bookworm-\d{8}$/,
+         ~r/^bookworm-\d{8}-slim$/,
+         # 11
+         ~r/^bullseye-\d{8}$/,
+         ~r/^bullseye-\d{8}-slim$/,
+         # 10
+         ~r/^buster-\d{8}$/,
+         ~r/^buster-\d{8}-slim$/
+       ]}
     ]
-  }
+    |> Task.async_stream(
+      fn {repo, regexes} ->
+        {repo, tags(repo, regexes)}
+      end,
+      ordered: false,
+      timeout: 300_000
+    )
+    |> Enum.map(fn {:ok, repo_and_tags} -> repo_and_tags end)
+    |> Map.new()
+  end
+
+  defp tags(repo, regexes) do
+    tags =
+      ("library/" <> repo)
+      |> Bob.DockerHub.fetch_repo_tags()
+      |> Enum.filter(fn {_tag, archs} -> Enum.all?(@archs, &(&1 in archs)) end)
+      |> Enum.map(fn {tag, _archs} -> tag end)
+      |> Enum.sort(&(&1 >= &2))
+
+    Enum.map(regexes, fn regex ->
+      Enum.find(tags, &(&1 =~ regex))
+    end)
+  end
 
   def run() do
     erlang()
@@ -59,7 +85,7 @@ defmodule Bob.Job.DockerChecker do
   def expected_erlang_tags() do
     refs = erlang_refs()
 
-    Stream.flat_map(@builds, fn {os, os_versions} ->
+    Stream.flat_map(builds(), fn {os, os_versions} ->
       Stream.flat_map(refs, fn ref ->
         if build_erlang_ref?(os, ref) do
           Stream.flat_map(os_versions, fn os_version ->
@@ -104,6 +130,9 @@ defmodule Bob.Job.DockerChecker do
   defp build_erlang_ref?("ubuntu", "jammy-" <> _, "OTP-" <> version),
     do: build_openssl_3?(version)
 
+  defp build_erlang_ref?("ubuntu", "noble-" <> _, "OTP-" <> version),
+    do: build_openssl_3?(version)
+
   defp build_erlang_ref?(_os, _os_version, _ref), do: true
 
   defp build_erlang_ref?("arm64", "ubuntu", "trusty-" <> _, "OTP-17" <> _), do: false
@@ -207,7 +236,7 @@ defmodule Bob.Job.DockerChecker do
 
   def erlang_tags(arch) do
     "hexpm/erlang-#{arch}"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Stream.map(fn {tag, [^arch]} ->
       [erlang, os, os_version] = Regex.run(@erlang_tag_regex, tag, capture: :all_but_first)
       {erlang, os, os_version, arch}
@@ -224,10 +253,11 @@ defmodule Bob.Job.DockerChecker do
   end
 
   def expected_elixir_tags() do
+    builds = builds()
     refs = elixir_builds()
 
     Stream.flat_map(erlang_tags(), fn {erlang, os, os_version, erlang_arch} ->
-      if not skip_elixir_for_erlang?(erlang) and os_version in @builds[os] do
+      if not skip_elixir_for_erlang?(erlang) and os_version in builds[os] do
         Stream.flat_map(refs, fn {"v" <> elixir, otp_major} ->
           if not skip_elixir?(elixir) and compatible_elixir_and_erlang?(otp_major, erlang) do
             [{elixir, erlang, os, os_version, erlang_arch}]
@@ -280,7 +310,7 @@ defmodule Bob.Job.DockerChecker do
 
   def elixir_tags(arch) do
     "hexpm/elixir-#{arch}"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Enum.map(fn {tag, [^arch]} ->
       [elixir, erlang, os, os_version] =
         Regex.run(@elixir_tag_regex, tag, capture: :all_but_first)
@@ -356,7 +386,7 @@ defmodule Bob.Job.DockerChecker do
 
   def erlang_manifest_tags() do
     "hexpm/erlang"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Map.new(fn {tag, archs} ->
       [erlang, os, os_version] = Regex.run(@erlang_tag_regex, tag, capture: :all_but_first)
       {{erlang, os, os_version}, archs}
@@ -365,7 +395,7 @@ defmodule Bob.Job.DockerChecker do
 
   def elixir_manifest_tags() do
     "hexpm/elixir"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Map.new(fn {tag, archs} ->
       [elixir, erlang, os, os_version] =
         Regex.run(@elixir_tag_regex, tag, capture: :all_but_first)

lib/bob/job/elixir_checker.ex

@@ -13,7 +13,6 @@ defmodule Bob.Job.ElixirChecker do
   def weight(), do: 1
   def concurrency(), do: :shared
 
-  defp build_ref?("main"), do: true
   defp build_ref?("v0." <> _), do: false
 
   defp build_ref?("v" <> version) do

lib/bob/job/otp_checker.ex

@@ -1,31 +1,32 @@
 defmodule Bob.Job.OTPChecker do
   @repo "erlang/otp"
-  @linuxes ["ubuntu-18.04", "ubuntu-20.04", "ubuntu-22.04"]
+  @linuxes ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04"]
   @arches ["amd64", "arm64"]
 
   def run(_type) do
     for linux <- @linuxes,
         arch <- @arches,
         {ref_name, ref} <- Bob.GitHub.diff(@repo, "builds/otp/#{arch}/#{linux}"),
-        build_ref?(linux, arch, ref_name),
+        build_ref?(linux, ref_name),
         do: Bob.Queue.add({Bob.Job.BuildOTP, arch}, [ref_name, ref, linux])
   end
 
   def priority(), do: 1
   def weight(), do: 1
   def concurrency(), do: :shared
 
-  defp build_ref?(_linux, _, "OTP-18.0-rc2"), do: false
-  defp build_ref?(_linux, _, "maint-r" <> _), do: false
-  defp build_ref?("ubuntu-18.04", "arm64", _), do: false
-  defp build_ref?("ubuntu-20.04", _, "OTP-" <> version), do: build_ubuntu_20?(version)
-  defp build_ref?("ubuntu-20.04", _, "maint-" <> version), do: build_ubuntu_20?(version)
-  defp build_ref?("ubuntu-22.04", _, "OTP-" <> version), do: build_ubuntu_22?(version)
-  defp build_ref?("ubuntu-22.04", _, "maint-" <> version), do: build_ubuntu_22?(version)
-  defp build_ref?(_linux, _, "OTP-" <> _), do: true
-  defp build_ref?(_linux, _, "maint" <> _), do: true
-  defp build_ref?(_linux, _, "master" <> _), do: true
-  defp build_ref?(_linux, _, _ref), do: false
+  defp build_ref?(_linux, "OTP-18.0-rc2"), do: false
+  defp build_ref?(_linux, "maint-r" <> _), do: false
+  defp build_ref?("ubuntu-20.04", "OTP-" <> version), do: build_ubuntu_20?(version)
+  defp build_ref?("ubuntu-20.04", "maint-" <> version), do: build_ubuntu_20?(version)
+  defp build_ref?("ubuntu-22.04", "OTP-" <> version), do: build_ubuntu_22?(version)
+  defp build_ref?("ubuntu-22.04", "maint-" <> version), do: build_ubuntu_22?(version)
+  defp build_ref?("ubuntu-24.04", "OTP-" <> version), do: build_ubuntu_22?(version)
+  defp build_ref?("ubuntu-24.04", "maint-" <> version), do: build_ubuntu_22?(version)
+  defp build_ref?(_linux, "OTP-" <> _), do: true
+  defp build_ref?(_linux, "maint" <> _), do: true
+  defp build_ref?(_linux, "master" <> _), do: true
+  defp build_ref?(_linux, _ref), do: false
 
   defp build_ubuntu_20?(erlang_version) do
     erlang_version = parse_otp_ref(erlang_version)

lib/bob/plugs/forwarded.ex

@@ -22,7 +22,7 @@ defmodule Bob.Plug.Forwarded do
         parts = Enum.map(parts, &elem(&1, 0))
         List.to_tuple(parts)
       else
-        Logger.warn("Invalid IP: #{inspect(ip)}")
+        Logger.warning("Invalid IP: #{inspect(ip)}")
         nil
       end
     end

priv/scripts/docker/erlang-ubuntu-bionic.dockerfile → priv/scripts/docker/erlang-ubuntu-noble.dockerfile

@@ -2,8 +2,6 @@ ARG OS_VERSION
 
 FROM ubuntu:${OS_VERSION} AS build
 
-ARG ERLANG
-
 RUN apt-get update
 RUN apt-get -y --no-install-recommends install \
   autoconf \
@@ -13,21 +11,23 @@ RUN apt-get -y --no-install-recommends install \
   make \
   libncurses-dev \
   unixodbc-dev \
-  $(bash -c 'if [ "${ERLANG:0:1}" = "1" ]; then echo "libssl1.0-dev"; else echo "libssl-dev"; fi') \
+  libssl-dev \
   libsctp-dev \
   wget \
   ca-certificates \
   pax-utils
 
+ARG ERLANG
+
 RUN mkdir -p /OTP/subdir
 RUN wget -nv "https://github.com/erlang/otp/archive/OTP-${ERLANG}.tar.gz" && tar -zxf "OTP-${ERLANG}.tar.gz" -C /OTP/subdir --strip-components=1
 WORKDIR /OTP/subdir
 RUN ./otp_build autoconf
 RUN ./configure --with-ssl --enable-dirty-schedulers
 RUN make -j$(getconf _NPROCESSORS_ONLN)
 RUN make -j$(getconf _NPROCESSORS_ONLN) install
-RUN bash -c 'if [ "${ERLANG:0:2}" -ge "23" ]; then make -j$(getconf _NPROCESSORS_ONLN) docs DOC_TARGETS=chunks; else true; fi'
-RUN bash -c 'if [ "${ERLANG:0:2}" -ge "23" ]; then make -j$(getconf _NPROCESSORS_ONLN) install-docs DOC_TARGETS=chunks; else true; fi'
+RUN make -j$(getconf _NPROCESSORS_ONLN) docs DOC_TARGETS=chunks
+RUN make -j$(getconf _NPROCESSORS_ONLN) install-docs DOC_TARGETS=chunks
 RUN find /usr/local -regex '/usr/local/lib/erlang/\(lib/\|erts-\).*/\(man\|obj\|c_src\|emacs\|info\|examples\)' | xargs rm -rf
 RUN find /usr/local -name src | xargs -r find | grep -v '\.hrl$' | xargs rm -v || true
 RUN find /usr/local -name src | xargs -r find | xargs rmdir -vp || true
@@ -36,16 +36,27 @@ RUN scanelf --nobanner -E ET_DYN -BF '%F' --recursive /usr/local | xargs -r stri
 
 FROM ubuntu:${OS_VERSION} AS final
 
-ARG ERLANG
-
-RUN apt-get update && \
-  apt-get -y --no-install-recommends install \
-    ca-certificates \
-    libodbc1 \
-    $(bash -c 'if [ "${ERLANG:0:1}" = "1" ]; then echo "libssl1.0.0"; else echo "libssl1.1"; fi') \
-    libsctp1 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+ARG ARCH
+
+RUN if [ "${ARCH}" = "amd64" ]; then \
+      apt-get update; \
+      apt-get -y --no-install-recommends install \
+        ca-certificates \
+        libodbc2 \
+        libssl3t \
+        libsctp1; \
+        apt-get clean; \
+        rm -rf /var/lib/apt/lists/*; \
+    elif [ "${ARCH}" = "arm64" ]; then \
+      apt-get update; \
+      apt-get -y --no-install-recommends install \
+        ca-certificates \
+        libodbc2 \
+        libssl3t64 \
+        libsctp1; \
+        apt-get clean; \
+        rm -rf /var/lib/apt/lists/*; \
+    fi
 
 COPY --from=build /usr/local /usr/local
 ENV LANG=C.UTF-8

priv/scripts/docker/erlang.sh

@@ -40,6 +40,7 @@ fi
 docker build \
   --ulimit nofile=1024:1024 \
   -t hexpm/erlang-${arch}:${tag} \
+  --build-arg ARCH=${arch} \
   --build-arg ERLANG=${erlang} \
   --build-arg OS_VERSION=${os_version} \
   --build-arg PIE_CFLAGS=${pie_cflags} \