Skip to content

Commit

Permalink
Merge branch 'main' into is-nethermind-libs
Browse files Browse the repository at this point in the history
  • Loading branch information
isaacsolo committed Jan 23, 2023
2 parents d331b05 + 2457196 commit dbfe4c1
Show file tree
Hide file tree
Showing 390 changed files with 65,366 additions and 55,188 deletions.
162 changes: 58 additions & 104 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ jobs:
--storage-location=us
- slack-fail:
branch_pattern: main
slack_mentions_user: "@dheeraj"
slack_mentions_user: "@ray"

publish-sdk:
docker:
Expand Down Expand Up @@ -370,6 +370,20 @@ jobs:
event: always
branch_pattern: main
slack_mentions_user: << parameters.slack_mentions_user >>
- restore_cache:
keys:
- sdk-cache-{{ checksum "libs/package-lock.json" }}
# fallback to using the latest cache if no exact match is found
- sdk-cache-
- run:
name: Install sdk dependencies
command: |
cd libs
npm i
- save_cache:
key: sdk-cache-{{ checksum "libs/package-lock.json" }}
paths:
- ./libs/node_modules
- run:
name: Bump and Publish Libs
command: |
Expand Down Expand Up @@ -735,7 +749,7 @@ jobs:
environment:
POSTGRES_USER: postgres
POSTGRES_DB: audius_identity_service_test
- image: redis:5.0.4
- image: redis:7.0
steps:
- checkout
- diff-if-necessary:
Expand Down Expand Up @@ -854,70 +868,6 @@ jobs:
# - solana-programs/track_listen_count/target
# key: track-listen-count-deps-{{ checksum "solana-programs/track_listen_count/Cargo.toml" }}

test-solana-programs-anchor:
resource_class: large # 4vcpu/8gb
machine:
image: ubuntu-2004:202107-02 # Ubuntu 20.04, Docker v20.10.7, Docker Compose v1.29.2 - see https://circleci.com/docs/2.0/configuration-reference/#machine
docker_layer_caching: true
steps:
# TODO implement cache for faster build
- checkout
- diff-if-necessary:
service: solana-programs
- run:
name: Install node
command: |
export PROTOCOL_DIR="/home/circleci/project"
export NVM_VERSION="v0.35.3"
export NODE_VERSION="v14.18.1"
sudo apt update -y
sudo DEBIAN_FRONTEND=noninteractive apt upgrade -y
sudo apt install curl -y
curl -o- "https://raw.githubusercontent.com/nvm-sh/nvm/$NVM_VERSION/install.sh" | bash
source ~/.bashrc
nvm install "$NODE_VERSION"
- restore_cache:
keys:
- anchor-node-deps-{{ checksum "solana-programs/anchor/audius-data/yarn.lock" }}
- anchor-node-deps-
- run:
name: Setup dev dependencies
command: |
export NODE_VERSION="v14.18.1"
source ~/.profile
source ~/.bashrc
nvm use "$NODE_VERSION"
export CARGO_HOME="cargo"
export PROTOCOL_DIR="/home/circleci/project"
cd "$PROTOCOL_DIR/solana-programs/anchor/audius-data"
source /opt/circleci/.nvm/nvm.sh
npm run install-dev
- run:
name: anchor tests
command: |
export PROTOCOL_DIR="/home/circleci/project"
export PATH="/home/circleci/.local/share/solana/install/active_release/bin:/usr/local/lbin:$PATH"
cd "$PROTOCOL_DIR/solana-programs/anchor/audius-data"
npm test
- save_cache:
paths:
- solana-programs/anchor/audius-data/node_modules
key: anchor-node-deps-{{ checksum "solana-programs/anchor/audius-data/yarn.lock" }}
- run:
name: anchor program seed CLI tests
command: |
export NODE_VERSION="v14.18.1"
source ~/.profile
source ~/.bashrc
nvm use "$NODE_VERSION"
export PROTOCOL_DIR="/home/circleci/project"
cd "$PROTOCOL_DIR/solana-programs/anchor/audius-data"
solana-keygen new --no-bip39-passphrase --force -o "/home/circleci/.config/solana/id.json"
yarn
source /opt/circleci/.nvm/nvm.sh
npm run localnet-up-ci
npm run dev-setup
docker-build-and-push:
docker:
- image: circleci/buildpack-deps:stretch
Expand Down Expand Up @@ -1273,22 +1223,23 @@ jobs:
type: string
default: ""
steps:
- checkout
- add_ssh_keys:
fingerprints:
# github.com
- "d0:0b:a0:19:ac:46:58:e4:6c:ac:34:99:f6:1b:31:bb"
- setup_remote_docker:
version: 20.10.14
docker_layer_caching: true
- run: |
ssh-keyscan -H github.com >> ~/.ssh/known_hosts
git clone git@github.com:AudiusProject/audius-tooling.git
cd audius-tooling
mkdir -p ssh_keys
cp ~/.ssh/id_rsa_d00ba019ac4658e46cac3499f61b31bb ssh_keys/
SLASH_ADDRESS=<< parameters.slash_address >> \
SLASH_AMOUNT=<< parameters.slash_amount >> \
release-scripts/<< parameters.target >>.sh
- run:
no_output_timeout: 30m
command: |
ssh-keyscan -H github.com >> ~/.ssh/known_hosts
mkdir -p ssh_keys
cp ~/.ssh/id_rsa_d00ba019ac4658e46cac3499f61b31bb ssh_keys/
SLASH_ADDRESS=<< parameters.slash_address >> \
SLASH_AMOUNT=<< parameters.slash_amount >> \
.circleci/release-scripts/<< parameters.target >>.sh
- slack-fail
- when:
condition:
Expand Down Expand Up @@ -1361,6 +1312,8 @@ workflows:
name: creator
- noop:
name: discovery
- noop:
name: comms
- noop:
name: identity

Expand Down Expand Up @@ -1408,9 +1361,8 @@ workflows:
- test-libs
- test-contracts
- test-solana-programs
- test-solana-programs-anchor
- test-eth-contracts
- test-mad-dog-e2e
# - test-mad-dog-e2e
- test-sdk

# Auto-deploy `main` to staging
Expand Down Expand Up @@ -1781,6 +1733,12 @@ workflows:
repo: discovery-provider
requires:
- discovery

- docker-build-and-push-updated:
name: build-comms
repo: comms
requires:
- comms

- test-identity:
name: test-identity
Expand All @@ -1796,17 +1754,13 @@ workflows:
name: test-solana-programs
requires:
- tests
- test-solana-programs-anchor:
name: test-solana-programs-anchor
requires:
- tests

- test-mad-dog-e2e:
context:
- GCP2
mad-dog-type: test
requires:
- tests
# - test-mad-dog-e2e:
# context:
# - GCP2
# mad-dog-type: test
# requires:
# - tests

- test-sdk:
context:
Expand All @@ -1830,9 +1784,9 @@ workflows:
context:
- Audius Client
- slack-secrets
filters:
branches:
only: /(^main$)/
# filters:
# branches:
# only: /(^main$)/

# in order to trigger this job
# 1. go to the CircleCI dashboard
Expand Down Expand Up @@ -2003,15 +1957,15 @@ workflows:
hold-workflows:
when: << pipeline.parameters.full_ci >>
jobs:
- hold-test-mad-dog-e2e-full:
type: approval
- test-mad-dog-e2e:
context:
- GCP2
name: test-mad-dog-e2e-full
mad-dog-type: test-nightly
requires:
- hold-test-mad-dog-e2e-full
# - hold-test-mad-dog-e2e-full:
# type: approval
# - test-mad-dog-e2e:
# context:
# - GCP2
# name: test-mad-dog-e2e-full
# mad-dog-type: test-nightly
# requires:
# - hold-test-mad-dog-e2e-full
- hold-build-logspout:
type: approval
- docker-build-and-push:
Expand Down Expand Up @@ -2046,11 +2000,11 @@ workflows:
name: test-identity-nightly
- test-solana-programs:
name: test-solana-programs-nightly
- test-mad-dog-e2e:
context:
- GCP2
name: test-mad-dog-e2e-nightly
mad-dog-type: test-nightly
# - test-mad-dog-e2e:
# context:
# - GCP2
# name: test-mad-dog-e2e-nightly
# mad-dog-type: test-nightly
- bake-gcp-dev-image:
context:
- GCP2
Expand Down
6 changes: 6 additions & 0 deletions .circleci/release-scripts/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
## Release to Production

* [Deployment Design Doc](https://www.notion.so/Deployment-Design-Doc-67f4fb5fb7574320a45dd63082ec4adf)
* [Deployment User Guide](https://www.notion.so/audiusproject/Deployment-User-Guide-708a8f64881740219d3f77707e73a097)

Called via audius-protocol's CircleCI project.
15 changes: 15 additions & 0 deletions .circleci/release-scripts/bump-hotfix.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash

set -e

script=bump-hotfix
docker build \
-t ${script} \
.circleci/release-scripts/dockerfiles/${script}

SSH_KEY=$(cat ssh_keys/id_rsa_d00ba019ac4658e46cac3499f61b31bb)
docker run \
-ti \
-e SSH_KEY="${SSH_KEY}" \
-e GH_TOKEN="${GH_TOKEN}" \
${script}
15 changes: 15 additions & 0 deletions .circleci/release-scripts/cut-release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash

set -e

script=cut-release
docker build \
-t ${script} \
.circleci/release-scripts/dockerfiles/${script}

SSH_KEY=$(cat ssh_keys/id_rsa_d00ba019ac4658e46cac3499f61b31bb)
docker run \
-ti \
-e SSH_KEY="${SSH_KEY}" \
-e GH_TOKEN="${GH_TOKEN}" \
${script}
35 changes: 35 additions & 0 deletions .circleci/release-scripts/dockerfiles/bump-hotfix/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
FROM ubuntu:22.04

ENV WORKDIR /app
WORKDIR ${WORKDIR}

RUN apt-get update \
&& apt-get install -y \
git \
gnupg \
jq \
moreutils \
wget \
&& sh -c "echo 'deb https://gitsecret.jfrog.io/artifactory/git-secret-deb git-secret main' \
>> /etc/apt/sources.list" \
&& wget -q 'https://gitsecret.jfrog.io/artifactory/api/gpg/key/public' -O /tmp/gitsecret.key \
&& md5sum /tmp/gitsecret.key | grep -q "e941935ec5dd72a60086e1b5112ade3d" \
|| exit \
&& apt-key add /tmp/gitsecret.key \
&& apt-get update \
&& apt-get install -y \
git-secret

WORKDIR /tmp
RUN wget https://github.com/tcnksm/ghr/releases/download/v0.15.0/ghr_v0.15.0_linux_amd64.tar.gz \
&& md5sum ghr_v0.15.0_linux_amd64.tar.gz | grep -q "b820693296a40bddb3873284f6051b87" \
|| exit \
&& tar xf ghr_v0.15.0_linux_amd64.tar.gz \
&& mv ghr_v0.15.0_linux_amd64/ghr ${WORKDIR}/
WORKDIR ${WORKDIR}

RUN git clone https://github.com/AudiusProject/audius-protocol.git

COPY bump.sh ./

CMD ./bump.sh
63 changes: 63 additions & 0 deletions .circleci/release-scripts/dockerfiles/bump-hotfix/bump.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env bash

set -ex

# required for ./ghr to create and tag Github Releases
if [ "${GH_TOKEN}" == "" ]; then
echo '$GH_TOKEN not set.'
exit 1
fi

# ensure ability to connect to Github
eval $(ssh-agent)
mkdir -p ~/.ssh/
echo "$SSH_KEY" > ~/.ssh/id_rsa
chmod 400 ~/.ssh/id_rsa
ssh-add ~/.ssh/id_*
ssh-keyscan -H github.com >> ~/.ssh/known_hosts
ssh -T git@github.com || true
git config --global user.email "audius-infra@audius.co"
git config --global user.name "audius-infra"
git config --global url."git@github.com:".insteadOf "https://github.com/"

# work in and always update the repo
cd ${WORKDIR}/audius-protocol
git pull

# grab latest release version and hash
RELEASE_VERSION=$(jq -r .version ${WORKDIR}/audius-protocol/discovery-provider/.version.json)
OLD_VERSION=$(echo ${RELEASE_VERSION} | awk -F. '{$NF = $NF - 1;} 1' | sed 's/ /./g')
git checkout release-v${RELEASE_VERSION}
RELEASE_HASH=$(git rev-parse HEAD)

function generate-changelog() {
(
SERVICE=${1}
START_COMMIT=$(git show-ref --hash refs/tags/@audius/${SERVICE}@${OLD_VERSION})
CHANGE_LOG=$(git log --pretty=format:'[%h] - %s' --abbrev-commit ${START_COMMIT}..${RELEASE_HASH} ${SERVICE})
echo -e "Full Changelog:\n${CHANGE_LOG}"
)
}

# recreate Github Releases (prerelease) + Git Tags
${WORKDIR}/ghr \
-token ${GH_TOKEN} \
-username AudiusProject \
-repository audius-protocol \
-commitish ${RELEASE_HASH} \
-body "$(generate-changelog discovery-provider)" \
-name "Discovery Provider ${RELEASE_VERSION}" \
-delete \
-prerelease \
@audius/discovery-provider@${RELEASE_VERSION}

${WORKDIR}/ghr \
-token ${GH_TOKEN} \
-username AudiusProject \
-repository audius-protocol \
-commitish ${RELEASE_HASH} \
-body "$(generate-changelog creator-node)" \
-name "Creator Node ${RELEASE_VERSION}" \
-delete \
-prerelease \
@audius/creator-node@${RELEASE_VERSION}
Loading

0 comments on commit dbfe4c1

Please sign in to comment.