From 75dfdcb065231811bc9977da1aaf0e66889cbbce Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Sat, 27 Jul 2019 22:18:40 +0200
Subject: [PATCH] ci: download awscli from our mirror

This fixes multiple network issues we had when downloading awscli from
PyPI on Azure Pipelines by vendoring awscli itself and its dependencies
in our S3 bucket. Instructions on how to update the cache are present at
the top of src/ci/install-awscli.sh
---
 .azure-pipelines/steps/run.yml | 42 ++++------------------------------
 src/ci/awscli-requirements.txt | 13 -----------
 src/ci/install-awscli.sh       | 35 ++++++++++++++++++++++++++++
 3 files changed, 40 insertions(+), 50 deletions(-)
 delete mode 100644 src/ci/awscli-requirements.txt
 create mode 100755 src/ci/install-awscli.sh

diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml
index 015fd83488550..1e49cc00921cd 100644
--- a/.azure-pipelines/steps/run.yml
+++ b/.azure-pipelines/steps/run.yml
@@ -138,43 +138,11 @@ steps:
 
 # Ensure the `aws` CLI is installed so we can deploy later on, cache docker
 # images, etc.
-- bash: |
-    set -e
-    # Temporary code to debug #62967.
-    debug_failed_connections() {
-        echo "trying to ping pypi.org"
-        ping pypi.org -c10 || true
-        echo "trying to ping google.com"
-        ping google.com -c10 || true
-        echo "trying to ping 8.8.8.8"
-        ping 8.8.8.8 -c10 || true
-        echo "trying to download pypi.org"
-        curl https://pypi.org || true
-        echo "trying to download from our S3 bucket"
-        curl https://rust-lang-ci2.s3.amazonaws.com || true
-        echo "trying to dig pypi.org"
-        dig pypi.org || true
-        echo "trying to dig files.pythonhosted.org"
-        dig files.pythonhosted.org || true
-        echo "trying to connect to pypi.org with openssl"
-        echo | openssl s_client -connect pypi.org:443 || true
-        echo "trying to connect to files.pythonhosted.org with openssl"
-        echo | openssl s_client -connect files.pythonhosted.org:443 || true
-    }
-    debug_failed_connections_and_fail() {
-        debug_failed_connections
-        return 1
-    }
-    source src/ci/shared.sh
-    sudo apt-get install -y python3-setuptools
-    debug_failed_connections
-    retry pip3 install -r src/ci/awscli-requirements.txt --upgrade --user || debug_failed_connections_and_fail
-    echo "##vso[task.prependpath]$HOME/.local/bin"
-  displayName: Install awscli (Linux)
-  condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Linux'))
-- script: pip install -r src/ci/awscli-requirements.txt
-  displayName: Install awscli (non-Linux)
-  condition: and(succeeded(), not(variables.SKIP_JOB), ne(variables['Agent.OS'], 'Linux'))
+- bash: src/ci/install-awscli.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+  displayName: Install awscli
 
 # Configure our CI_JOB_NAME variable which log analyzers can use for the main
 # step to see what's going on.
diff --git a/src/ci/awscli-requirements.txt b/src/ci/awscli-requirements.txt
deleted file mode 100644
index c1ffa525a1b41..0000000000000
--- a/src/ci/awscli-requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-awscli==1.16.201
-botocore==1.12.191
-colorama==0.3.9
-docutils==0.14
-jmespath==0.9.4
-pyasn1==0.4.5
-python-dateutil==2.8.0
-PyYAML==5.1
-rsa==3.4.2
-s3transfer==0.2.1
-six==1.12.0
-urllib3==1.25.3
-futures==3.3.0; python_version < '3.0'
diff --git a/src/ci/install-awscli.sh b/src/ci/install-awscli.sh
new file mode 100755
index 0000000000000..d491b9fbcdcf8
--- /dev/null
+++ b/src/ci/install-awscli.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# This script downloads and installs awscli from the packages mirrored in our
+# own S3 bucket. This follows the recommendations at:
+#
+#    https://packaging.python.org/guides/index-mirrors-and-caches/#caching-with-pip
+#
+# To create a new mirrored copy you can run the command:
+#
+#    pip wheel awscli
+#
+# Before compressing please make sure all the wheels end with `-none-any.whl`.
+# If that's not the case you'll need to remove the non-cross-platform ones and
+# replace them with the .tar.gz downloaded from https://pypi.org. Also make
+# sure it's possible to call this script with both Python 2 and Python 3.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+MIRROR="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-07-27-awscli.tar"
+DEPS_DIR="/tmp/awscli-deps"
+
+pip="pip"
+pipflags=""
+if [[ "${AGENT_OS}" == "Linux" ]]; then
+    pip="pip3"
+    pipflags="--user"
+
+    sudo apt-get install -y python3-setuptools
+    echo "##vso[task.prependpath]$HOME/.local/bin"
+fi
+
+mkdir -p "${DEPS_DIR}"
+curl "${MIRROR}" | tar xf - -C "${DEPS_DIR}"
+"${pip}" install ${pipflags} --no-index "--find-links=${DEPS_DIR}" awscli
+rm -rf "${DEPS_DIR}"