Skip to content

Commit

Permalink
feat: Release of the Extensible CLI via npm
Browse files Browse the repository at this point in the history
  • Loading branch information
PeterSchafer committed Mar 8, 2023
1 parent b6e056b commit e7e558c
Show file tree
Hide file tree
Showing 25 changed files with 7,645 additions and 256 deletions.
35 changes: 31 additions & 4 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -278,13 +278,16 @@ jobs:
- setup_npm
- run:
name: Building project
command: npm run build:prod
command: |
make pre-build
npm run build:prod
- persist_to_workspace:
root: .
paths:
- dist/
- packages/*/dist
- pysrc
- ts-binary-wrapper/
version:
executor: docker-node
steps:
Expand Down Expand Up @@ -575,7 +578,10 @@ jobs:
cp cliv2/bin/snyk-linux-arm64 binary-releases/snyk-linux-arm64
cp cliv2/bin/snyk-linux-arm64.sha256 binary-releases/snyk-linux-arm64.sha256
- run:
name: Signing shasums
name: Making TS-Binary-Wrapper (snyk.tgz)
command: make binary-releases/snyk.tgz
- run:
name: Signing all shasums
command: make binary-releases/sha256sums.txt.asc
- run:
name: Making release.json
Expand Down Expand Up @@ -984,6 +990,22 @@ jobs:
- cliv2/bin/snyk_windows_amd64.exe
- cliv2/bin/snyk_windows_amd64.exe.sha256

test-in-proxy-environment-linux-amd64:
executor: linux
working_directory: /home/circleci/snyk
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Run acceptance tests in a proxied environment
working_directory: ./
environment:
TEST_SNYK_COMMAND: ./bin/snyk-linux
command: |
export SNYK_TOKEN="${SNYK_API_KEY}"
make acceptance-test-with-proxy
workflows:
version: 2
test_and_release:
Expand Down Expand Up @@ -1037,7 +1059,6 @@ workflows:
matrix:
parameters:
artifact:
- snyk.tgz
- snyk-fix.tgz
- snyk-protect.tgz
- snyk-alpine
Expand Down Expand Up @@ -1098,7 +1119,6 @@ workflows:
context:
- snyk-cli-pgp-signing
requires:
- Build (snyk.tgz)
- Build (snyk-fix.tgz)
- Build (snyk-protect.tgz)
- Build (snyk-alpine)
Expand Down Expand Up @@ -1318,3 +1338,10 @@ workflows:
- v2 / Rename linux/arm64 artifact
- v2 / Rename macos/amd64 artifact
- v2 / Rename windows/amd64 artifact
- test-in-proxy-environment-linux-amd64:
filters:
branches:
ignore: master
name: v2 / Acceptance Tests With Proxy (linux/amd64)
requires:
- v2 / Rename linux/amd64 artifact
2 changes: 1 addition & 1 deletion .github/workflows/smoke-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ jobs:
run: |
which snyk
snyk version
shellspec -f d --skip-message quiet
shellspec -f d --skip-message quiet --no-warning-as-failure
- name: Run shellspec tests - Windows
if: ${{ matrix.os == 'windows' }}
Expand Down
66 changes: 53 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@ help:
@echo 'Use `npm run` for CLIv1 scripts.'

$(BINARY_RELEASES_FOLDER_TS_CLI):
@mkdir $(BINARY_RELEASES_FOLDER_TS_CLI)
@mkdir -p $(BINARY_RELEASES_FOLDER_TS_CLI)

$(BINARY_RELEASES_FOLDER_TS_CLI)/version: | $(BINARY_RELEASES_FOLDER_TS_CLI)
./release-scripts/next-version.sh > $(BINARY_RELEASES_FOLDER_TS_CLI)/version

ifneq ($(BINARY_OUTPUT_FOLDER), $(BINARY_RELEASES_FOLDER_TS_CLI))
$(BINARY_OUTPUT_FOLDER)/version: $(BINARY_RELEASES_FOLDER_TS_CLI)/version
$(BINARY_OUTPUT_FOLDER):
@mkdir -p $(BINARY_OUTPUT_FOLDER)

$(BINARY_OUTPUT_FOLDER)/version: $(BINARY_OUTPUT_FOLDER) $(BINARY_RELEASES_FOLDER_TS_CLI)/version
@cp $(BINARY_RELEASES_FOLDER_TS_CLI)/version $(BINARY_OUTPUT_FOLDER)/version
endif

Expand All @@ -38,25 +41,26 @@ endif
# Only removing "prepack" is not enough. We need to do additional cleanup (see clean-prepack).
.INTERMEDIATE: prepack
.SECONDARY: prepack
prepack: $(BINARY_RELEASES_FOLDER_TS_CLI)/version
prepack: $(BINARY_OUTPUT_FOLDER)/version
@echo "'make prepack' was run. Run 'make clean-prepack' to rollback your package.json changes and this file." > prepack
npm version "$(shell cat $(BINARY_RELEASES_FOLDER_TS_CLI)/version)" --no-git-tag-version --workspaces --include-workspace-root
cd $(BINARY_WRAPPER_DIR) && npm version "$(shell cat $(CURDIR)/$(BINARY_RELEASES_FOLDER_TS_CLI)/version)" --no-git-tag-version --include-workspace-root
npx ts-node ./release-scripts/prune-dependencies-in-packagejson.ts

.PHONY: clean-prepack
clean-prepack:
git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json
git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json $(BINARY_WRAPPER_DIR)/package.json $(BINARY_WRAPPER_DIR)/package-lock.json
rm -f prepack

.PHONY: clean-ts
clean-ts:
npm run clean
rm -f -r $(BINARY_RELEASES_FOLDER_TS_CLI)

$(BINARY_OUTPUT_FOLDER)/sha256sums.txt.asc: $(wildcard $(BINARY_OUTPUT_FOLDER)/*.sha256)
$(BINARY_OUTPUT_FOLDER)/sha256sums.txt.asc:
./release-scripts/sha256sums.txt.asc.sh

$(BINARY_OUTPUT_FOLDER)/release.json: $(BINARY_OUTPUT_FOLDER)/version $(wildcard $(BINARY_OUTPUT_FOLDER)/*.sha256)
$(BINARY_OUTPUT_FOLDER)/release.json: $(BINARY_OUTPUT_FOLDER)/version
./release-scripts/release.json.sh

# --commit-path is forwarded to `git log <path>`.
Expand All @@ -71,8 +75,8 @@ $(BINARY_OUTPUT_FOLDER)/RELEASE_NOTES.md: prepack | $(BINARY_RELEASES_FOLDER_TS_
%.sha256: %
cd $(@D); shasum -a 256 $(<F) > $(@F); shasum -a 256 -c $(@F)

$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
mv $(shell npm pack) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
$(MAKE) pack-binary-wrapper
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz.sha256

$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-fix.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
Expand Down Expand Up @@ -121,8 +125,20 @@ $(BINARY_RELEASES_FOLDER_TS_CLI)/docker-mac-signed-bundle.tar.gz: prepack | $(BI
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/docker-mac-signed-bundle.tar.gz.sha256

# targets responsible for the Wrapper CLI (TS around Golang)
$(BINARY_WRAPPER_DIR)/README.md:
@cp ./README.md $(BINARY_WRAPPER_DIR)/README.md

$(BINARY_WRAPPER_DIR)/SECURITY.md:
@cp ./SECURITY.md $(BINARY_WRAPPER_DIR)/SECURITY.md

$(BINARY_WRAPPER_DIR)/LICENSE:
@cp ./LICENSE $(BINARY_WRAPPER_DIR)/LICENSE

$(BINARY_WRAPPER_DIR)/src/generated/binary-deployments.json: $(BINARY_WRAPPER_DIR)/src/generated
@cp ./binary-deployments.json $(BINARY_WRAPPER_DIR)/src/generated/binary-deployments.json

$(BINARY_WRAPPER_DIR)/src/generated:
@mkdir $(BINARY_WRAPPER_DIR)/src/generated/
@mkdir -p $(BINARY_WRAPPER_DIR)/src/generated/

$(BINARY_WRAPPER_DIR)/src/generated/version: $(BINARY_WRAPPER_DIR)/src/generated $(BINARY_RELEASES_FOLDER_TS_CLI)/version
@cp $(BINARY_RELEASES_FOLDER_TS_CLI)/version $(BINARY_WRAPPER_DIR)/src/generated/version
Expand All @@ -132,35 +148,59 @@ $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt:
@cat $(BINARY_OUTPUT_FOLDER)/*.sha256 > $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt

.PHONY: build-binary-wrapper
build-binary-wrapper: $(BINARY_WRAPPER_DIR)/src/generated/version $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt
build-binary-wrapper: pre-build-binary-wrapper $(BINARY_WRAPPER_DIR)/src/generated/version $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt
@echo "-- Building Typescript Binary Wrapper ($(BINARY_WRAPPER_DIR)/dist/)"
@cd $(BINARY_WRAPPER_DIR) && npm run build

.PHONY: clean-binary-wrapper
clean-binary-wrapper:
@rm -f $(BINARY_WRAPPER_DIR)/config.default.json
@rm -f $(BINARY_WRAPPER_DIR)/src/generated/binary-deployments.json
@rm -f $(BINARY_WRAPPER_DIR)/README.md
@rm -f $(BINARY_WRAPPER_DIR)/SECURITY.md
@rm -f $(BINARY_WRAPPER_DIR)/LICENSE
@rm -rf $(BINARY_WRAPPER_DIR)/src/generated
@rm -rf $(BINARY_WRAPPER_DIR)/help
@rm -rf $(BINARY_WRAPPER_DIR)/pysrc
@cd $(BINARY_WRAPPER_DIR) && npm run clean

.PHONY: pre-build-binary-wrapper
pre-build-binary-wrapper: $(BINARY_WRAPPER_DIR)/README.md $(BINARY_WRAPPER_DIR)/SECURITY.md $(BINARY_WRAPPER_DIR)/LICENSE $(BINARY_WRAPPER_DIR)/src/generated/binary-deployments.json

# for compatibility reasons, we pack the legacy and the ts-binary-wrapper next to each other
.PHONY: pack-binary-wrapper
pack-binary-wrapper: build-binary-wrapper
@echo "-- Packaging tarball ($(BINARY_OUTPUT_FOLDER)/snyk.tgz)"
@mv $(BINARY_WRAPPER_DIR)/$(shell cd $(BINARY_WRAPPER_DIR) && npm pack) $(BINARY_OUTPUT_FOLDER)/snyk.tgz
release-scripts/create-npm-artifact.sh $(BINARY_OUTPUT_FOLDER) $(BINARY_WRAPPER_DIR)


.PHONY: test-binary-wrapper
test-binary-wrapper:
test-binary-wrapper: build-binary-wrapper
@echo "-- Testing binary wrapper"
@cd $(BINARY_WRAPPER_DIR) && npm run test


# targets responsible for the complete CLI build
.PHONY: pre-build
pre-build: pre-build-binary-wrapper $(BINARY_RELEASES_FOLDER_TS_CLI)

.PHONY: build
build:
build: pre-build
@cd $(EXTENSIBLE_CLI_DIR) && $(MAKE) build-full install bindir=$(CURDIR)/$(BINARY_OUTPUT_FOLDER) USE_LEGACY_EXECUTABLE_NAME=1

.PHONY: clean
clean:
@cd $(EXTENSIBLE_CLI_DIR) && $(MAKE) clean-full
$(MAKE) clean-prepack

# targets responsible for the testing of CLI build
.PHONY: acceptance-test-with-proxy
acceptance-test-with-proxy:
@echo "-- Running acceptance tests in a proxied environment"
@docker build -t acceptance-test-with-proxy -f ./test/acceptance/environments/proxy/Dockerfile .
@docker run --rm --cap-add=NET_ADMIN acceptance-test-with-proxy ./node_modules/.bin/jest ./ts-binary-wrapper/test/acceptance/basic.spec.ts
# TODO: Run all acceptance tests behind a proxy using npm run test:acceptance

# targets responsible for the CLI release
.PHONY: release-pre
release-pre:
Expand Down
16 changes: 16 additions & 0 deletions binary-deployments.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"darwin": {
"amd64": "snyk-macos",
"arm64": "snyk-macos"
},
"windows": {
"amd64": "snyk-win.exe"
},
"alpine": {
"amd64": "snyk-alpine"
},
"linux": {
"amd64": "snyk-linux",
"arm64": "snyk-linux-arm64"
}
}
29 changes: 29 additions & 0 deletions release-scripts/create-npm-artifact.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail

BINARY_OUTPUT_FOLDER=${1}
BINARY_WRAPPER_DIR=${2}
ROOT=$(pwd)

# create legacy TS tarball
mv "$(npm pack)" "${BINARY_OUTPUT_FOLDER}/snyk_legacy.tgz"

# create TS binary wrapper tarball
pushd .
cd "${BINARY_WRAPPER_DIR}"
mv "$(npm pack)" "${ROOT}/${BINARY_OUTPUT_FOLDER}/snyk_wrapper.tgz"
popd

# merge the two tarballs and repack them to the final file
pushd .
cd "${BINARY_OUTPUT_FOLDER}"
tar -xf snyk_legacy.tgz
tar -xf snyk_wrapper.tgz
cd package
mv "$(npm pack)" "${ROOT}/${BINARY_OUTPUT_FOLDER}/snyk.tgz"
popd

# cleanup intermediate files and folders
rm -rf "${ROOT}/${BINARY_OUTPUT_FOLDER}/package"
rm -rf "${ROOT}/${BINARY_OUTPUT_FOLDER}/snyk_legacy.tgz"
rm -rf "${ROOT}/${BINARY_OUTPUT_FOLDER}/snyk_wrapper.tgz"
41 changes: 41 additions & 0 deletions test/acceptance/environments/proxy/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Copyright 2022 Snyk Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM node:gallium
RUN apt-get update
RUN DEBIAN_FRONTEND=noninteractive apt-get install -y sudo git squid curl git traceroute net-tools iptables vim make golang

RUN apt-get auto-remove -y && apt-get clean -y && rm -rf /var/lib/apt/
RUN update-alternatives --set iptables /usr/sbin/iptables-legacy

RUN useradd snyk --create-home
RUN echo "snyk ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers

RUN mkdir -p /app
COPY . /app

RUN rm -rf /app/build /app/.bin
RUN cp /app/test/acceptance/environments/proxy/entrypoint.sh /bin

RUN chmod +x /bin/entrypoint.sh
RUN chmod 777 /app && chown -R snyk /app

ENV http_proxy="http://localhost:3128"
ENV https_proxy="http://localhost:3128"
ENV no_proxy "localhost,127.0.0.1"

RUN env
USER snyk
WORKDIR /app
ENTRYPOINT ["/bin/entrypoint.sh"]
31 changes: 31 additions & 0 deletions test/acceptance/environments/proxy/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash
#
# © 2022 Snyk Limited All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

set -ex

echo "Starting squid..."
sudo squid -f /etc/squid/squid.conf -NYCd 1 &
## configure firewall to only work through proxy
sudo iptables -A OUTPUT -m owner --uid-owner root -j ACCEPT
sudo iptables -A OUTPUT -m owner --uid-owner proxy -j ACCEPT
sudo iptables -A OUTPUT -p tcp --dport 80 -j REJECT
sudo iptables -A OUTPUT -p tcp --dport 443 -j REJECT
sudo Xvfb -ac :99 -screen 0 1280x1024x24 > /dev/null 2>&1 &
sleep 5 # wait for squid to startup
export PATH=$PATH:$PWD/.bin/pact/bin:$PWD/.bin/
export DISPLAY=:99
exec "$@"
27 changes: 27 additions & 0 deletions test/acceptance/environments/proxy/settings.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<!--
~ © 2022 Snyk Limited All rights reserved.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<settings>
<proxies>
<proxy>
<id>cli-acceptance-test</id>
<active>true</active>
<protocol>http</protocol>
<host>localhost</host>
<port>3128</port>
</proxy>
</proxies>
</settings>
2 changes: 1 addition & 1 deletion test/smoke/docker-root/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM node:12
FROM node:16

COPY ./smoke/ /snyk/smoke/
COPY ./fixtures/ /snyk/fixtures/
Expand Down
Loading

0 comments on commit e7e558c

Please sign in to comment.