From 13080e8981a09b787fcc7d5f6feaf9a489190e50 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 4 Sep 2018 10:19:52 -0700 Subject: [PATCH 1/2] Retry npm install in CI --- .circleci/config.yml | 109 +++++++++++++++++++++++++-------- .circleci/npm-install-retry.js | 60 ++++++++++++++++++ 2 files changed, 142 insertions(+), 27 deletions(-) create mode 100755 .circleci/npm-install-retry.js diff --git a/.circleci/config.yml b/.circleci/config.yml index edbfdb7d1..80dcf7e67 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,23 +11,36 @@ workflows: filters: *all_commits - node10: filters: *all_commits - - docs: - filters: *all_commits - lint: + requires: + - node6 + - node8 + - node10 + filters: *all_commits + - docs: + requires: + - node6 + - node8 + - node10 filters: *all_commits - system_tests: - filters: + requires: + - lint + - docs + filters: &master_and_releases branches: only: master tags: &releases only: '/^v[\d.]+$/' + - sample_tests: + requires: + - lint + - docs + filters: *master_and_releases - publish_npm: requires: - - node6 - - node8 - - node10 - system_tests - - docs + - sample_tests filters: branches: ignore: /.*/ @@ -46,20 +59,17 @@ jobs: - image: 'node:6' user: node steps: &unit_tests_steps - - checkout + - checkout - run: &npm_install_and_link name: Install and link the module command: |- mkdir -p /home/node/.npm-global - npm install + ./.circleci/npm-install-retry.js environment: NPM_CONFIG_PREFIX: /home/node/.npm-global - - run: - name: Run unit tests - command: npm run test-only - environment: - GOOGLE_APPLICATION_CREDENTIALS: build/test/fake-certificate.json - - run: npm run codecov + - run: npm test + - run: node_modules/.bin/codecov + node8: docker: - image: 'node:8' @@ -72,51 +82,96 @@ jobs: steps: *unit_tests_steps lint: docker: - - image: 'node:10' + - image: 'node:8' user: node steps: - checkout - run: *npm_install_and_link - - run: npm run check + - run: &samples_npm_install_and_link + name: Link the module being tested to the samples. + command: | + cd samples/ + npm link ../ + ./../.circleci/npm-install-retry.js + environment: + NPM_CONFIG_PREFIX: /home/node/.npm-global + - run: + name: Run linting. + command: npm run lint + environment: + NPM_CONFIG_PREFIX: /home/node/.npm-global docs: docker: - - image: 'node:10' + - image: 'node:8' user: node steps: - checkout - run: *npm_install_and_link - run: npm run docs + sample_tests: + docker: + - image: 'node:8' + user: node + steps: + - checkout + - run: + name: Decrypt credentials. + command: | + if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then + openssl aes-256-cbc -d -in .circleci/key.json.enc \ + -out .circleci/key.json \ + -k "${SYSTEM_TESTS_ENCRYPTION_KEY}" + fi + - run: *npm_install_and_link + - run: *samples_npm_install_and_link + - run: + name: Run sample tests. + command: npm run samples-test + environment: + GCLOUD_PROJECT: long-door-651 + GOOGLE_APPLICATION_CREDENTIALS: /home/node/samples/.circleci/key.json + NPM_CONFIG_PREFIX: /home/node/.npm-global + - run: + name: Remove unencrypted key. + command: | + if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then + rm .circleci/key.json + fi + when: always + working_directory: /home/node/samples/ system_tests: docker: - - image: 'node:10' + - image: 'node:8' user: node steps: - checkout - run: name: Decrypt credentials. command: | - openssl aes-256-cbc -d -in .circleci/key.json.enc \ + if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then + openssl aes-256-cbc -d -in .circleci/key.json.enc \ -out .circleci/key.json \ -k "${SYSTEM_TESTS_ENCRYPTION_KEY}" + fi - run: *npm_install_and_link - run: name: Run system tests. command: npm run system-test environment: GOOGLE_APPLICATION_CREDENTIALS: .circleci/key.json - GCLOUD_PROJECT: node-gcloud-ci - GCLOUD_TESTS_KEY: .circleci/key.json - GCLOUD_TESTS_PROJECT_ID: node-gcloud-ci - run: name: Remove unencrypted key. - command: rm .circleci/key.json + command: | + if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then + rm .circleci/key.json + fi when: always publish_npm: docker: - - image: 'node:10' + - image: 'node:8' user: node steps: - checkout + - run: ./.circleci/npm-install-retry.js - run: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc - - run: *npm_install_and_link - - run: npm publish + - run: npm publish --access=public diff --git a/.circleci/npm-install-retry.js b/.circleci/npm-install-retry.js new file mode 100755 index 000000000..ae3220d73 --- /dev/null +++ b/.circleci/npm-install-retry.js @@ -0,0 +1,60 @@ +#!/usr/bin/env node + +let spawn = require('child_process').spawn; + +// +//USE: ./index.js [... NPM ARGS] +// + +let timeout = process.argv[2] || 60000; +let attempts = process.argv[3] || 3; +let args = process.argv.slice(4); +if (args.length === 0) { + args = ['install']; +} + +(function npm() { + let timer; + args.push('--verbose'); + let proc = spawn('npm', args); + proc.stdout.pipe(process.stdout); + proc.stderr.pipe(process.stderr); + proc.stdin.end(); + proc.stdout.on('data', () => { + setTimer(); + }); + proc.stderr.on('data', () => { + setTimer(); + }); + + // side effect: this also restarts when npm exits with a bad code even if it + // didnt timeout + proc.on('close', (code, signal) => { + clearTimeout(timer); + if (code || signal) { + console.log('[npm-are-you-sleeping] npm exited with code ' + code + ''); + + if (--attempts) { + console.log('[npm-are-you-sleeping] restarting'); + npm(); + } else { + console.log('[npm-are-you-sleeping] i tried lots of times. giving up.'); + throw new Error("npm install fails"); + } + } + }); + + function setTimer() { + clearTimeout(timer); + timer = setTimeout(() => { + console.log('[npm-are-you-sleeping] killing npm with SIGTERM'); + proc.kill('SIGTERM'); + // wait a couple seconds + timer = setTimeout(() => { + // its it's still not closed sigkill + console.log('[npm-are-you-sleeping] killing npm with SIGKILL'); + proc.kill('SIGKILL'); + }, 2000); + }, timeout); + } +})(); From 7a0d3ea660286feb8e110c251055359f6d8ad73d Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 4 Sep 2018 17:03:21 -0700 Subject: [PATCH 2/2] fixy --- package.json | 4 ++-- synth.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index c963d05d1..4c04742d6 100644 --- a/package.json +++ b/package.json @@ -44,13 +44,13 @@ "conformance": "mocha build/conformance", "test-only": "nyc mocha build/test", "test": "npm run test-only", - "check": "gts check", + "lint": "gts check", "clean": "gts clean", "compile": "tsc -p . && cp -r dev/protos build && cp -r dev/test/fake-certificate.json build/test/fake-certificate.json && cp dev/src/v1beta1/firestore_client_config.json build/src/v1beta1/ && cp dev/conformance/test-definition.proto build/conformance && cp dev/conformance/test-suite.binproto build/conformance", "fix": "gts fix", "prepare": "npm run compile", "pretest-only": "npm run compile", - "posttest": "npm run check" + "posttest": "npm run lint" }, "dependencies": { "@google-cloud/projectify": "^0.3.0", diff --git a/synth.py b/synth.py index df8f5f1ed..1b812abe2 100644 --- a/synth.py +++ b/synth.py @@ -24,7 +24,11 @@ 'new firestoreModule.v1beta1.FirestoreClient\(', 'new firestoreModule.v1beta1(') +# Copy template files +common_templates = gcp.CommonTemplates() +templates = common_templates.node_library() +s.copy(templates) + # Node.js specific cleanup subprocess.run(['npm', 'install']) -subprocess.run(['npm', 'run', 'prettier']) -subprocess.run(['npm', 'run', 'lint']) +subprocess.run(['npm', 'run', 'fix'])