Add Proxy for Fetch in download.ts - Resolves issue 1280

docs/guides/compile-contracts.md

@@ -30,6 +30,10 @@ module.exports = {
 
 We recommend always setting a compiler version to avoid unexpected behavior or compiling errors as new releases of Solidity are published.
 
+:::: warning
+Hardhat will automatically download the versions of `solc` that you set up. If you are behind an HTTP proxy, you may need to set the `HTTP_PROXY` or `HTTPS_PROXY` environment variable to the URL of your proxy.
+::::
+
 The expanded usage allows for more control of the compiler:
 
 ```js

packages/hardhat-core/package.json

@@ -52,36 +52,37 @@
     "console.sol"
   ],
   "devDependencies": {
-    "@types/chai-as-promised": "^7.1.3",
     "@types/chai": "^4.2.0",
+    "@types/chai-as-promised": "^7.1.3",
     "@types/ci-info": "^2.0.0",
     "@types/debug": "^4.1.4",
     "@types/find-up": "^2.1.1",
     "@types/fs-extra": "^5.1.0",
     "@types/glob": "^7.1.1",
     "@types/lodash": "^4.14.123",
     "@types/mocha": "^5.2.6",
-    "@types/node-fetch": "^2.3.7",
     "@types/node": "^10.17.24",
+    "@types/node-fetch": "^2.3.7",
     "@types/qs": "^6.5.3",
     "@types/resolve": "^1.17.1",
     "@types/semver": "^6.0.2",
     "@types/sinon": "^9.0.8",
     "@types/uuid": "^3.4.5",
     "@types/ws": "^7.2.1",
-    "chai-as-promised": "^7.1.1",
     "chai": "^4.2.0",
+    "chai-as-promised": "^7.1.1",
     "ethers": "^5.0.0",
     "mocha": "^7.1.2",
     "prettier": "2.0.5",
+    "proxy": "^1.0.2",
     "rimraf": "^3.0.2",
     "sinon": "^9.0.0",
     "time-require": "^0.1.2",
     "ts-node": "^8.1.0",
+    "tslint": "^5.16.0",
     "tslint-config-prettier": "^1.18.0",
     "tslint-consistent-codestyle": "^1.15.1",
     "tslint-plugin-prettier": "^2.0.1",
-    "tslint": "^5.16.0",
     "typescript": "~4.0.3"
   },
   "dependencies": {
@@ -111,6 +112,7 @@
     "fp-ts": "1.19.3",
     "fs-extra": "^7.0.1",
     "glob": "^7.1.3",
+    "https-proxy-agent": "^5.0.0",
     "immutable": "^4.0.0-rc.12",
     "io-ts": "1.10.4",
     "lodash": "^4.17.11",

packages/hardhat-core/src/internal/util/download.ts

@@ -1,8 +1,14 @@
 import fs from "fs";
 import fsExtra from "fs-extra";
+import HttpsProxyAgent from "https-proxy-agent";
 import path from "path";
 import util from "util";
 
+interface FetchOptions {
+  timeout: number;
+  agent?: undefined | HttpsProxyAgent.HttpsProxyAgent;
+}
+
 export async function download(
   url: string,
   filePath: string,
@@ -11,8 +17,30 @@ export async function download(
   const { pipeline } = await import("stream");
   const { default: fetch } = await import("node-fetch");
   const streamPipeline = util.promisify(pipeline);
+  const fetchOptions: FetchOptions = {
+    timeout: timeoutMillis,
+    agent: undefined,
+  };
+
+  // Check if Proxy is set https
+  if (process.env.HTTPS_PROXY !== undefined) {
+    // Create the proxy from the environment variables
+    const proxy: string = process.env.HTTPS_PROXY;
+    fetchOptions.agent = new HttpsProxyAgent.HttpsProxyAgent(proxy);
+  }
+
+  // Check if Proxy is set http and `fetchOptions.agent` was not already set for https
+  if (
+    process.env.HTTP_PROXY !== undefined &&
+    fetchOptions.agent === undefined
+  ) {
+    // Create the proxy from the environment variables
+    const proxy: string = process.env.HTTP_PROXY;
+    fetchOptions.agent = new HttpsProxyAgent.HttpsProxyAgent(proxy);
+  }
 
-  const response = await fetch(url, { timeout: timeoutMillis });
+  // Fetch the url
+  const response = await fetch(url, fetchOptions);
 
   if (response.ok && response.body !== null) {
     await fsExtra.ensureDir(path.dirname(filePath));

packages/hardhat-core/test/internal/util/download.ts

@@ -0,0 +1,99 @@
+import { assert } from "chai";
+import fsExtra from "fs-extra";
+import path from "path";
+// @ts-ignore
+// tslint:disable-next-line no-implicit-dependencies
+import Proxy from "proxy";
+
+import { download } from "../../../src/internal/util/download";
+import { useTmpDir } from "../../helpers/fs";
+
+describe("Compiler List download", function () {
+  useTmpDir("compiler-downloader");
+
+  describe("Compilers list download", function () {
+    it("Should call download with the right params", async function () {
+      const compilersDir = this.tmpDir;
+      const downloadPath = path.join(compilersDir, "downloadedCompiler");
+      const expectedUrl = `https://solc-bin.ethereum.org/wasm/list.json`;
+
+      // download the file
+      await download(expectedUrl, downloadPath);
+      // Assert that the file exists
+      assert.isTrue(await fsExtra.pathExists(downloadPath));
+    });
+  });
+});
+
+describe("Compiler List download with proxy", function () {
+  let env: typeof process.env;
+  let proxy: any;
+  let proxyPort: number;
+
+  useTmpDir("compiler-downloader");
+
+  before(function (done) {
+    // Setup Proxy Server
+    proxy = new Proxy();
+    proxy.listen(function () {
+      proxyPort = proxy.address().port;
+      done();
+    });
+  });
+
+  describe("Compilers list download with HTTPS_PROXY", function () {
+    before(function () {
+      // Save the Environment Settings and Set
+      env = process.env;
+      process.env.HTTPS_PROXY = `http://127.0.0.1:${proxyPort}`;
+    });
+
+    it("Should call download with the right params", async function () {
+      const compilersDir = this.tmpDir;
+      const downloadPath = path.join(compilersDir, "downloadedCompilerProxy");
+      const expectedUrl = `https://solc-bin.ethereum.org/wasm/list.json`;
+
+      // download the file
+      await download(expectedUrl, downloadPath);
+      // Assert that the file exists
+      assert.isTrue(await fsExtra.pathExists(downloadPath));
+    });
+
+    after(function () {
+      // restoring everything back to the environment
+      process.env = env;
+    });
+  });
+
+  describe("Compilers list download with HTTP_PROXY", function () {
+    before(function () {
+      // Save the Environment Settings and Set
+      env = process.env;
+      process.env.HTTP_PROXY = `http://127.0.0.1:${proxyPort}`;
+    });
+
+    it("Should call download with the right params", async function () {
+      const compilersDir = this.tmpDir;
+      const downloadPath = path.join(compilersDir, "downloadedCompilerProxy");
+      const expectedUrl = `https://solc-bin.ethereum.org/wasm/list.json`;
+
+      // download the file
+      await download(expectedUrl, downloadPath);
+      // Assert that the file exists
+      assert.isTrue(await fsExtra.pathExists(downloadPath));
+    });
+
+    after(function () {
+      // restoring everything back to the environment
+      process.env = env;
+    });
+  });
+
+  after(function (done) {
+    // Shutdown Proxy Server
+    proxy.once("close", function () {
+      done();
+    });
+    proxy.close();
+  });
+});

yarn.lock

@@ -1331,6 +1331,16 @@ argparse@^1.0.7:
   dependencies:
     sprintf-js "~1.0.2"
 
+args@5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/args/-/args-5.0.1.tgz#4bf298df90a4799a09521362c579278cc2fdd761"
+  integrity sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==
+  dependencies:
+    camelcase "5.0.0"
+    chalk "2.4.2"
+    leven "2.1.0"
+    mri "1.1.4"
+
 arr-diff@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520"
@@ -2039,6 +2049,11 @@ base@^0.11.1:
     mixin-deep "^1.2.0"
     pascalcase "^0.1.1"
 
+basic-auth-parser@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/basic-auth-parser/-/basic-auth-parser-0.0.2.tgz#ce9e71a77f23c1279eecd2659b2a46244c156e41"
+  integrity sha1-zp5xp38jwSee7NJlmypGJEwVbkE=
+
 bcrypt-pbkdf@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
@@ -2433,6 +2448,11 @@ callsites@^3.0.0:
   resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
   integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
 
+camelcase@5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42"
+  integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==
+
 camelcase@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
@@ -2485,6 +2505,15 @@ chai@^4.2.0:
     pathval "^1.1.1"
     type-detect "^4.0.5"
 
+chalk@2.4.2, chalk@^2.0.0, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2:
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
+  integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
+  dependencies:
+    ansi-styles "^3.2.1"
+    escape-string-regexp "^1.0.5"
+    supports-color "^5.3.0"
+
 chalk@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-0.4.0.tgz#5199a3ddcd0c1efe23bc08c1b027b06176e0c64f"
@@ -2505,15 +2534,6 @@ chalk@^1.1.3:
     strip-ansi "^3.0.0"
     supports-color "^2.0.0"
 
-chalk@^2.0.0, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2:
-  version "2.4.2"
-  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
-  integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
-  dependencies:
-    ansi-styles "^3.2.1"
-    escape-string-regexp "^1.0.5"
-    supports-color "^5.3.0"
-
 chardet@^0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
@@ -5841,6 +5861,11 @@ levelup@^4.3.2:
     level-supports "~1.0.0"
     xtend "~4.0.0"
 
+leven@2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/leven/-/leven-2.1.0.tgz#c2e7a9f772094dee9d34202ae8acce4687875580"
+  integrity sha1-wuep93IJTe6dNCAq6KzORoeHVYA=
+
 levn@^0.3.0, levn@~0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee"
@@ -6322,6 +6347,11 @@ mock-fs@^4.1.0:
   resolved "https://registry.yarnpkg.com/mock-fs/-/mock-fs-4.14.0.tgz#ce5124d2c601421255985e6e94da80a7357b1b18"
   integrity sha512-qYvlv/exQ4+svI3UOvPUpLDF0OMX5euvUH0Ny4N5QyRyhNdgAgUrVH3iUINSzEPLvx0kbo/Bp28GJKIqvE7URw==
 
+mri@1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a"
+  integrity sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==
+
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -7099,6 +7129,15 @@ proxy-addr@~2.0.5:
     forwarded "~0.1.2"
     ipaddr.js "1.9.1"
 
+proxy@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/proxy/-/proxy-1.0.2.tgz#e0cfbe11c0a7a8b238fd2d7134de4e2867578e7f"
+  integrity sha512-KNac2ueWRpjbUh77OAFPZuNdfEqNynm9DD4xHT14CccGpW8wKZwEkN0yjlb7X9G9Z9F55N0Q+1z+WfgAhwYdzQ==
+  dependencies:
+    args "5.0.1"
+    basic-auth-parser "0.0.2"
+    debug "^4.1.1"
+
 prr@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"