Skip to content

Commit

Permalink
Interchaintest for e2e tests (#174)
Browse files Browse the repository at this point in the history
* locked down dockerfiles

* begin migration to interchaintest

* upgrade to horcrux test

* run e2e tests in matrix

* TestDownedSigners and TestLeaderElection

* TestChainPureHorcrux

* TestMultipleChainHorcrux, remove rest of old framework

* tidy

* handle feedback

* fix nil pre-genesis

* fix cosigner start vs chain start race

* fix no chains in WaitForBlocks

* remove sleep in remote signer
  • Loading branch information
agouin authored Jul 18, 2023
1 parent 7c21a7e commit 5252b60
Show file tree
Hide file tree
Showing 20 changed files with 4,525 additions and 2,599 deletions.
5 changes: 4 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,7 @@ test/
scripts/
docs/
data/
build/
build/
docker/
go.work
go.work.sum
47 changes: 36 additions & 11 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Run Go Tests
name: Tests

on:
pull_request:
Expand All @@ -13,9 +13,8 @@ on:
- 'Makefile'

jobs:
test:
name: run tests
runs-on: [self-hosted, linux]
unit:
runs-on: ubuntu-latest
steps:
# Install and setup go
- name: Set up Go 1.20
Expand All @@ -27,17 +26,43 @@ jobs:
- name: checkout horcrux
uses: actions/checkout@v3

# cleanup docker environment on self-hosted test runner
- name: prepare fresh docker environment
run: |
docker stop $(docker ps -a -q) || true
docker rm -f $(docker ps -a -q) || true
docker network prune -f || true
# make sure proto files are up to date
- name: generate fresh signer proto .go files
run: make signer-proto

# run tests
- name: run horcrux tests
run: make test
e2e:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- TestMultipleChainHorcrux
- TestChainPureHorcrux
- TestDownedSigners2of3
- TestDownedSigners3of5
- TestLeaderElection2of3
- Test2Of3SignerThreeSentries
- Test2Of3SignerThreeSentriesUniqueConnection
- TestUpgradeValidatorToHorcrux
- TestSingleSignerTwoSentries
steps:
# Install and setup go
- name: Set up Go 1.20
uses: actions/setup-go@v4
with:
go-version: '^1.20.0'

# checkout horcrux
- name: checkout horcrux
uses: actions/checkout@v3

# make sure proto files are up to date
- name: generate fresh signer proto .go files
run: make signer-proto

# run test matrix
- name: run test
run: cd test && go test -v -timeout 30m -run ^${{ matrix.test }}$ .
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ all: install
LD_FLAGS = -X github.com/strangelove-ventures/horcrux/cmd/horcrux/cmd.Version=$(VERSION) \
-X github.com/strangelove-ventures/horcrux/cmd/horcrux/cmd.Commit=$(COMMIT)

LD_FLAGS += $(LDFLAGS)
LD_FLAGS := $(strip $(LD_FLAGS))

BUILD_FLAGS := -ldflags '$(LD_FLAGS)'

build:
Expand Down
72 changes: 60 additions & 12 deletions docker/horcrux/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,24 +1,72 @@
FROM --platform=$BUILDPLATFORM golang:1.20-alpine AS build-env
FROM --platform=$BUILDPLATFORM golang:1.20-alpine3.16 AS build-env

ENV PACKAGES make git
RUN apk add --update --no-cache curl make git libc-dev bash gcc linux-headers eudev-dev

RUN apk add --no-cache $PACKAGES
ARG TARGETARCH
ARG BUILDARCH

RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
wget -c https://musl.cc/aarch64-linux-musl-cross.tgz -O - | tar -xzvv --strip-components 1 -C /usr; \
elif [ "${TARGETARCH}" = "amd64" ] && [ "${BUILDARCH}" != "amd64" ]; then \
wget -c https://musl.cc/x86_64-linux-musl-cross.tgz -O - | tar -xzvv --strip-components 1 -C /usr; \
fi

WORKDIR /go/src/github.com/strangelove-ventures/horcrux
WORKDIR /horcrux

ADD . .

ARG TARGETARCH
ARG TARGETOS
RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
export CC=aarch64-linux-musl-gcc CXX=aarch64-linux-musl-g++;\
elif [ "${TARGETARCH}" = "amd64" ] && [ "${BUILDARCH}" != "amd64" ]; then \
export CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++; \
fi; \
GOOS=linux GOARCH=$TARGETARCH CGO_ENABLED=1 LDFLAGS='-linkmode external -extldflags "-static"' make install

RUN if [ -d "/go/bin/linux_${TARGETARCH}" ]; then mv /go/bin/linux_${TARGETARCH}/* /go/bin/; fi

# Use minimal busybox from infra-toolkit image for final scratch image
FROM ghcr.io/strangelove-ventures/infra-toolkit:v0.0.6 AS busybox-min
RUN addgroup --gid 2345 -S horcrux && adduser --uid 2345 -S horcrux -G horcrux

# Use ln and rm from full featured busybox for assembling final image
FROM busybox:1.34.1-musl AS busybox-full

# Build final image from scratch
FROM scratch

LABEL org.opencontainers.image.source="https://github.com/cosmos/horcrux"

WORKDIR /bin

# Install ln (for making hard links) and rm (for cleanup) from full busybox image (will be deleted, only needed for image assembly)
COPY --from=busybox-full /bin/ln /bin/rm ./

RUN export GOOS=${TARGETOS} GOARCH=${TARGETARCH} && make build
# Install minimal busybox image as shell binary (will create hardlinks for the rest of the binaries to this data)
COPY --from=busybox-min /busybox/busybox /bin/sh

FROM alpine:edge
# Add hard links for read-only utils, then remove ln and rm
# Will then only have one copy of the busybox minimal binary file with all utils pointing to the same underlying inode
RUN ln sh pwd && \
ln sh ls && \
ln sh cat && \
ln sh less && \
ln sh grep && \
ln sh sleep && \
ln sh env && \
ln sh tar && \
ln sh tee && \
ln sh du && \
rm ln rm

RUN apk add --no-cache ca-certificates
# Install chain binaries
COPY --from=build-env /go/bin/horcrux /bin

WORKDIR /root
# Install trusted CA certificates
COPY --from=busybox-min /etc/ssl/cert.pem /etc/ssl/cert.pem

COPY --from=build-env /go/src/github.com/strangelove-ventures/horcrux/build/horcrux /usr/bin/horcrux
# Install horcrux user
COPY --from=busybox-min /etc/passwd /etc/passwd
COPY --from=busybox-min --chown=2345:2345 /home/horcrux /home/horcrux

CMD ["horcrux"]
WORKDIR /home/horcrux
USER horcrux
59 changes: 49 additions & 10 deletions docker/horcrux/native.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,21 +1,60 @@
FROM golang:1.20-alpine AS build-env
FROM golang:1.20-alpine3.16 AS build-env

ENV PACKAGES make git
RUN apk add --update --no-cache curl make git libc-dev bash gcc linux-headers eudev-dev

RUN apk add --no-cache $PACKAGES
WORKDIR /horcrux

WORKDIR /go/src/github.com/strangelove-ventures/horcrux
ADD go.mod go.sum ./

RUN go mod download

ADD . .

RUN make build
RUN CGO_ENABLED=1 LDFLAGS='-linkmode external -extldflags "-static"' make install

# Use minimal busybox from infra-toolkit image for final scratch image
FROM ghcr.io/strangelove-ventures/infra-toolkit:v0.0.6 AS busybox-min
RUN addgroup --gid 2345 -S horcrux && adduser --uid 2345 -S horcrux -G horcrux

# Use ln and rm from full featured busybox for assembling final image
FROM busybox:1.34.1-musl AS busybox-full

# Build final image from scratch
FROM scratch

LABEL org.opencontainers.image.source="https://github.com/strangelove-ventures/horcrux"

WORKDIR /bin

# Install ln (for making hard links) and rm (for cleanup) from full busybox image (will be deleted, only needed for image assembly)
COPY --from=busybox-full /bin/ln /bin/rm ./

# Install minimal busybox image as shell binary (will create hardlinks for the rest of the binaries to this data)
COPY --from=busybox-min /busybox/busybox /bin/sh

FROM alpine:edge
# Add hard links for read-only utils, then remove ln and rm
# Will then only have one copy of the busybox minimal binary file with all utils pointing to the same underlying inode
RUN ln sh pwd && \
ln sh ls && \
ln sh cat && \
ln sh less && \
ln sh grep && \
ln sh sleep && \
ln sh env && \
ln sh tar && \
ln sh tee && \
ln sh du && \
rm ln rm

RUN apk add --no-cache ca-certificates
# Install chain binaries
COPY --from=build-env /go/bin/horcrux /bin

WORKDIR /root
# Install trusted CA certificates
COPY --from=busybox-min /etc/ssl/cert.pem /etc/ssl/cert.pem

COPY --from=build-env /go/src/github.com/strangelove-ventures/horcrux/build/horcrux /usr/bin/horcrux
# Install horcrux user
COPY --from=busybox-min /etc/passwd /etc/passwd
COPY --from=busybox-min --chown=2345:2345 /home/horcrux /home/horcrux

CMD ["horcrux"]
WORKDIR /home/horcrux
USER horcrux
36 changes: 0 additions & 36 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ require (
github.com/Jille/raft-grpc-transport v1.4.0
github.com/Jille/raftadmin v1.2.0
github.com/armon/go-metrics v0.4.1
github.com/avast/retry-go v3.0.0+incompatible
github.com/cometbft/cometbft v0.37.2
github.com/cosmos/cosmos-sdk v0.47.3
github.com/ecies/go/v2 v2.0.6
Expand All @@ -17,7 +16,6 @@ require (
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/kraken-hpc/go-fork v0.1.1
github.com/mitchellh/go-homedir v1.1.0
github.com/ory/dockertest v3.3.5+incompatible
github.com/prometheus/client_golang v1.14.0
github.com/spf13/cobra v1.6.1
github.com/spf13/viper v1.14.0
Expand All @@ -32,59 +30,39 @@ require (
)

require (
cosmossdk.io/api v0.3.1 // indirect
cosmossdk.io/core v0.5.1 // indirect
cosmossdk.io/depinject v1.0.0-alpha.3 // indirect
cosmossdk.io/errors v1.0.0-beta.7 // indirect
cosmossdk.io/math v1.0.1 // indirect
filippo.io/edwards25519 v1.0.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cometbft/cometbft-db v0.7.0 // indirect
github.com/confio/ics23/go v0.9.0 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/cosmos/btcutil v1.0.5 // indirect
github.com/cosmos/cosmos-proto v1.0.0-beta.2 // indirect
github.com/cosmos/go-bip39 v1.0.0 // indirect
github.com/cosmos/gogoproto v1.4.10 // indirect
github.com/cosmos/iavl v0.20.0 // indirect
github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
github.com/ethereum/go-ethereum v1.11.5 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-kit/kit v0.12.0 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/gtank/merlin v0.1.1 // indirect
github.com/gtank/ristretto255 v0.1.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
Expand All @@ -95,7 +73,6 @@ require (
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hdevalence/ed25519consensus v0.1.0 // indirect
github.com/huandu/skiplist v1.2.0 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/jmhodges/levigo v1.0.0 // indirect
github.com/klauspost/compress v1.16.3 // indirect
Expand All @@ -106,10 +83,6 @@ require (
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/opencontainers/runc v1.1.5 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
Expand All @@ -118,32 +91,23 @@ require (
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/sasha-s/go-deadlock v0.3.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.4.1 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
github.com/tidwall/btree v1.6.0 // indirect
github.com/zondax/hid v0.9.1 // indirect
github.com/zondax/ledger-go v0.14.1 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
golang.org/x/crypto v0.8.0 // indirect
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/tools v0.6.0 // indirect
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
pgregory.net/rapid v0.5.5 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

Expand Down
Loading

0 comments on commit 5252b60

Please sign in to comment.