Auto merge of rust-lang#90457 - rusticstuff:use-curl-ca-bundle, r=Mar…

…k-Simulacrum

CI: Use ubuntu image to download openssl, curl sources, cacert.pem for x86 dist builds

The dist-x86_64 and dist-i686 docker builds are failing again (see [try build](https://github.com/rust-lang-ci/rust/runs/4060836540?check_suite_focus=true)) because python.org renewed its certificate with a CA cert that is too new for debian:6.

In order to solve this once and for all this PR moves the curl and openssl downloads to a new  ubuntu:20.04 "stage-0" docker build and copies the downloaded tarballs over to build them in the "stage-1" debian:6 context. It also downloads the cacert.pem file from the curl website and uses it by pointing the `CURL_CA_BUNDLE` environment variable to it.

src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile

@@ -1,3 +1,15 @@
+# We need recent curl, OpenSSL and CA certificates, so we can download further
+# dependencies in the debian:6 image. We use an ubuntu 20.04 image download
+# those.
+FROM ubuntu:20.04
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates
+WORKDIR /tmp
+COPY host-x86_64/dist-x86_64-linux/download-openssl-curl.sh /tmp/
+RUN ./download-openssl-curl.sh
+
 # We use Debian 6 (glibc 2.11, kernel 2.6.32) as a common base for other
 # distros that still need Rust support: RHEL 6 (glibc 2.12, kernel 2.6.32) and
 # SLES 11 SP4 (glibc 2.11, kernel 3.0).
@@ -14,8 +26,6 @@ RUN apt-get update && \
     apt-get install --allow-unauthenticated -y --no-install-recommends \
       automake \
       bzip2 \
-      ca-certificates \
-      curl \
       file \
       g++ \
       g++-multilib \
@@ -34,11 +44,6 @@ RUN apt-get update && \
       xz-utils \
       zlib1g-dev
 
-# Install new Let's Encrypt root CA certificate and remove the expired one.
-COPY host-x86_64/shared/ISRG_Root_X1.crt /usr/local/share/ca-certificates/ISRG_Root_X1.crt
-RUN sed -i '/mozilla\/DST_Root_CA_X3\.crt/d' /etc/ca-certificates.conf
-RUN /usr/sbin/update-ca-certificates
-
 ENV PATH=/rustroot/bin:$PATH
 ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
@@ -50,6 +55,7 @@ COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
 # static.rust-lang.org. This'll be used to link into libcurl below (and used
 # later as well), so build a copy of OpenSSL with dynamic libraries into our
 # generic root.
+COPY --from=0 /tmp/openssl.tar.gz /tmp/openssl.tar.gz
 COPY host-x86_64/dist-x86_64-linux/build-openssl.sh /tmp/
 RUN ./build-openssl.sh
 
@@ -59,8 +65,13 @@ RUN ./build-openssl.sh
 #
 # Note that we also disable a bunch of optional features of curl that we don't
 # really need.
+COPY --from=0 /tmp/curl.tar.xz /tmp/curl.tar.xz
 COPY host-x86_64/dist-x86_64-linux/build-curl.sh /tmp/
-RUN ./build-curl.sh && apt-get remove -y curl
+RUN ./build-curl.sh
+
+# Use up-to-date curl CA bundle
+COPY --from=0 /tmp/cacert.pem /tmp/cacert.pem
+ENV CURL_CA_BUNDLE /tmp/cacert.pem
 
 # binutils < 2.22 has a bug where the 32-bit executables it generates
 # immediately segfault in Rust, so we need to install our own binutils.

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -1,3 +1,15 @@
+# We need recent curl, OpenSSL and CA certificates, so we can download further
+# dependencies in the debian:6 image. We use an ubuntu 20.04 image download
+# those.
+FROM ubuntu:20.04
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates
+WORKDIR /tmp
+COPY host-x86_64/dist-x86_64-linux/download-openssl-curl.sh /tmp/
+RUN ./download-openssl-curl.sh
+
 # We use Debian 6 (glibc 2.11, kernel 2.6.32) as a common base for other
 # distros that still need Rust support: RHEL 6 (glibc 2.12, kernel 2.6.32) and
 # SLES 11 SP4 (glibc 2.11, kernel 3.0).
@@ -14,8 +26,6 @@ RUN apt-get update && \
     apt-get install --allow-unauthenticated -y --no-install-recommends \
       automake \
       bzip2 \
-      ca-certificates \
-      curl \
       file \
       g++ \
       g++-multilib \
@@ -34,11 +44,6 @@ RUN apt-get update && \
       xz-utils \
       zlib1g-dev
 
-# Install new Let's Encrypt root CA certificate and remove the expired one.
-COPY host-x86_64/shared/ISRG_Root_X1.crt /usr/local/share/ca-certificates/ISRG_Root_X1.crt
-RUN sed -i '/mozilla\/DST_Root_CA_X3\.crt/d' /etc/ca-certificates.conf
-RUN /usr/sbin/update-ca-certificates
-
 ENV PATH=/rustroot/bin:$PATH
 ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
@@ -50,6 +55,7 @@ COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
 # static.rust-lang.org. This'll be used to link into libcurl below (and used
 # later as well), so build a copy of OpenSSL with dynamic libraries into our
 # generic root.
+COPY --from=0 /tmp/openssl.tar.gz /tmp/openssl.tar.gz
 COPY host-x86_64/dist-x86_64-linux/build-openssl.sh /tmp/
 RUN ./build-openssl.sh
 
@@ -59,8 +65,13 @@ RUN ./build-openssl.sh
 #
 # Note that we also disable a bunch of optional features of curl that we don't
 # really need.
+COPY --from=0 /tmp/curl.tar.xz /tmp/curl.tar.xz
 COPY host-x86_64/dist-x86_64-linux/build-curl.sh /tmp/
-RUN ./build-curl.sh && apt-get remove -y curl
+RUN ./build-curl.sh
+
+# Use up-to-date curl CA bundle
+COPY --from=0 /tmp/cacert.pem /tmp/cacert.pem
+ENV CURL_CA_BUNDLE /tmp/cacert.pem
 
 # binutils < 2.22 has a bug where the 32-bit executables it generates
 # immediately segfault in Rust, so we need to install our own binutils.

src/ci/docker/host-x86_64/dist-x86_64-linux/build-curl.sh

@@ -3,18 +3,11 @@
 set -ex
 source shared.sh
 
-VERSION=7.66.0
-
-# This needs to be downloaded directly from S3, it can't go through the CDN.
-# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1
-# (without paying an absurd amount of money).
-curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \
-  | xz --decompress \
-  | tar xf -
+tar xJf curl.tar.xz
 
 mkdir curl-build
 cd curl-build
-hide_output ../curl-$VERSION/configure \
+hide_output ../curl-*/configure \
       --prefix=/rustroot \
       --with-ssl=/rustroot \
       --disable-sspi \
@@ -35,4 +28,4 @@ hide_output make install
 
 cd ..
 rm -rf curl-build
-rm -rf curl-$VERSION
+rm -rf curl-*

src/ci/docker/host-x86_64/dist-x86_64-linux/build-openssl.sh

@@ -3,21 +3,14 @@
 set -ex
 source shared.sh
 
-VERSION=1.0.2k
+tar xzf openssl.tar.gz
 
-# This needs to be downloaded directly from S3, it can't go through the CDN.
-# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1
-# (without paying an absurd amount of money).
-URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz
-
-curl $URL | tar xzf -
-
-cd openssl-$VERSION
+cd openssl-*
 hide_output ./config --prefix=/rustroot shared -fPIC
 hide_output make -j$(nproc)
 hide_output make install
 cd ..
-rm -rf openssl-$VERSION
+rm -rf openssl-*
 
 # Make the system cert collection available to the new install.
 ln -nsf /etc/pki/tls/cert.pem /rustroot/ssl/

src/ci/docker/host-x86_64/dist-x86_64-linux/download-openssl-curl.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -ex
+
+OPENSSL_VERSION=1.0.2k
+CURL_VERSION=7.66.0
+
+curl -f https://ci-mirrors.rust-lang.org/rustc/openssl-$OPENSSL_VERSION.tar.gz -o openssl.tar.gz
+curl -f https://ci-mirrors.rust-lang.org/rustc/curl-$CURL_VERSION.tar.xz -o curl.tar.xz
+curl -f https://curl.se/ca/cacert.pem -o cacert.pem

src/ci/docker/run.sh

@@ -50,7 +50,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       # Look for all source files involves in the COPY command
       copied_files=/tmp/.docker-copied-files.txt
       rm -f "$copied_files"
-      for i in $(sed -n -e 's/^COPY \(.*\) .*$/\1/p' "$docker_dir/$image/Dockerfile"); do
+      for i in $(sed -n -e '/^COPY --from=/! s/^COPY \(.*\) .*$/\1/p' \
+          "$docker_dir/$image/Dockerfile"); do
         # List the file names
         find "$script_dir/$i" -type f >> $copied_files
       done