diff --git a/package.json b/package.json index 0af115cb246..717c90d2703 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "check-secrets": "sh ./scripts/check_secrets.sh", "clean:modules": "rm -rf node_modules && rm -rf protocol-dashboard/node_modules && rm -rf packages/*/node_modules", "clean:dist": "rm -rf packages/*/dist", + "install-git-secrets": "sh ./scripts/install-git-secrets.sh", "install-hooks": "mookme init --only-hook --skip-types-selection", "lint:fix": "turbo run lint:fix", "lint": "turbo run lint", diff --git a/packages/ddex/README.md b/packages/ddex/README.md index 6759761223b..b4a22968a04 100644 --- a/packages/ddex/README.md +++ b/packages/ddex/README.md @@ -5,12 +5,11 @@ Processes and uploads DDEX releases to Audius. ## Production setup Use audius-docker-compose to run a production DDEX instance. After you've installed audius-docker-compose, set the following required environment variables in override.env (in the audius-docker-compose repository, not here). -### Glossary +### Creating a bucket in S3 * `env` refers to `dev`, `staging`, or `prod` * `provider` refers to the name of the label/distributor/provider -### Creating a bucket in S3 1. Create a new bucket in the S3 console with the name `ddex---raw`. Use all the defaults, including "ACLs disabled" 2. Do the same for a bucket named `ddex---crawled`. Use all the defaults, including "ACLs disabled" @@ -125,37 +124,71 @@ For docker compose to work: `cat packages/ddex/.env >> dev-tools/compose/.env` ### One-time setup 1. `audius-compose connect` to update your `/etc/hosts` -2. Install the AWS cli and configure it for local dev: - ```sh - pip install awscli && \ - aws configure set aws_access_key_id test && \ - aws configure set aws_secret_access_key test && \ - aws configure set region us-west-2 - ``` +2. Install the AWS cli and configure a profile for local dev: + +```bash +pip install awscli + +aws configure --profile local +# enter these details +# AWS Access Key ID [None]: test +# AWS Secret Access Key [None]: test +# Default region name [None]: us-west-2 +# Default output format [None]: json +``` + +edit `~/.aws/config` and add +``` +[profile local] +region = us-west-2 +endpoint_url = http://ingress:4566 +``` + +To use the created profile, run: +```bash +export AWS_PROFILE=local +``` +You may also pass `--profile local` to all aws commands instead. + 3. To use the DDEX webapp as an admin, add your decoded staging user ID to `extra-env.DDEX_ADMIN_ALLOWLIST` in `../../dev-tools/config.json` - Find your user ID by going to `https://discoveryprovider.staging.audius.co/v1/full/users/handle/`, searching for `id`, and then decoding it by pasting it into the "Encoded" textbox [here](https://healthz.audius.co/#/utils/id) and copying the "Integer" value - - Note that this requires a restart if the app is already running (`audius-compose down && audius-compose up -ddex-[release-by-release|batched]`) + - Note that this requires a restart if the app is already running (`audius-compose down && audius-compose up --ddex-[release-by-release|batched]`) ### Bring up the ddex stack locally Run `audius-compose up --ddex-release-by-release` (or `audius-compose up --ddex-batched` -- see "Choreography" in Glossary below), and navigate to `http://localhost:9000` to view the DDEX webapp To upload a delivery to be processed: - 1. Create buckets: `aws --endpoint=http://ingress:4566 s3 mb s3://audius-test-raw && aws --endpoint=http://ingress:4566 s3 mb s3://audius-test-crawled` - 2. Upload your file: `aws --endpoint=http://ingress:4566 s3 cp s3://audius-test-raw`. Example: `aws --endpoint=http://ingress:4566 s3 cp ./ingester/e2e_test/fixtures/release_by_release/ern381/sony1.zip s3://audius-test-raw` + 1. Create buckets +```bash +aws s3 mb s3://audius-test-raw +aws s3 mb s3://audius-test-crawled +``` + + 2. Upload your file +```bash +aws s3 cp s3://audius-test-raw +# e.g. +# aws s3 cp ./ingester/e2e_test/fixtures/release_by_release/ern381/sony1.zip s3://audius-test-raw +``` + 3. Watch the UI (localhost:9000) for the delivery to be crawled in a few seconds -To access the ddex db via the mongo shell: `docker exec -it ddex-mongo mongosh -u mongo -p mongo --authenticationDatabase admin`, and then `use ddex`. +To access the ddex db via the mongo shell: +```bash +docker exec -it ddex-mongo mongosh -u mongo -p mongo --authenticationDatabase admin +> use ddex +``` ### Develop with hot reloading Each service can be run independently as long as `ddex-mongo` is up (from `audius-compose up --ddex-[release-by-release|batched]` and then optionally stopping individual services). See the respective subdirectories' READMEs. ### Running / debugging the e2e test * Run `audius-compose test down && audius-compose test run ddex-e2e-release-by-release` to start the ddex stack and run the e2e test for the Release-By-Release choreography. Or run `audius-compose test run ddex-e2e-batched` to run the e2e test for the Batched choreography. -* To debug S3, follow the onte-time setup instructions above to update your `/etc/hosts` and install the AWS cli. Then you can run `aws --endpoint=http://localhost:4566 s3 ls` and other commands to debug the S3 state. +* To debug S3, follow the onte-time setup instructions above to update your `/etc/hosts` and install the AWS cli. Then you can run `aws s3 ls` and other commands to debug the S3 state. ## App architecture and flows -1. A distributor uploads a ZIP file to the "raw" AWS S3 bucket. +1. A distributor either uploads a ZIP file to the "raw" AWS S3 bucket or flat files directly. 2. The Crawler periodically checks this bucket for new uploads. It downloads+unzips the file and crawls it for one or more "releases" (ie, metadata and assets for a track -- or collection of tracks -- to upload to Audius). The assets are uploaded to the "crawled" AWS S3 bucket, and metadata is stored in MongoDB. 3. The Parser app watches for new releases and processes each one into a format that the Publisher app can use to upload to Audius. 4. When the release date is reached for a release, the Publisher app uploads the release to Audius. diff --git a/packages/ddex/publisher/Dockerfile b/packages/ddex/publisher/Dockerfile index f9ebd5f66ff..268d23880df 100644 --- a/packages/ddex/publisher/Dockerfile +++ b/packages/ddex/publisher/Dockerfile @@ -37,7 +37,7 @@ RUN CI=true npm i --maxsockets 1 COPY --from=turbo-builder /app/out/full/ . COPY svgr-template.js svgr-template.js COPY turbo.json turbo.json -RUN npx turbo run build --filter=@audius/ddex-publisher --filter=@audius/ddex-publisher +RUN npx turbo run build --filter=@audius/ddex-publisher # Make a smaller image by removing all src directories (except for in node_modules) RUN find packages -path '*/node_modules/*' -prune -o -name 'src' -type d -exec rm -rf {} + diff --git a/packages/ddex/webapp/Dockerfile.fast b/packages/ddex/webapp/Dockerfile.fast deleted file mode 100644 index 56ea78dd5ba..00000000000 --- a/packages/ddex/webapp/Dockerfile.fast +++ /dev/null @@ -1,55 +0,0 @@ -# Attempting to build+push locally for a Linux (stage or prod) target may hang. Use this file as a workaround: -# 1. in packages/ddex/webapp/server: npm run build -# 2. in packages/ddex/webapp/client: npm run build -# 3. in docker-compose.ddex.yml, change the value of dockerfile from Dockerfile to Dockerfile.fast -# 4. run DOCKER_DEFAULT_PLATFORM=linux/amd64 audius-compose push --prod "ddex" -# 5. ssh into a stage node and edit ~/audius-docker-compose/discovery-provider/docker-compose.yml: change the ddex image to the SHA you just pushed - -FROM node:18-alpine AS base - -ARG TURBO_TEAM -ENV TURBO_TEAM=$TURBO_TEAM - -ARG TURBO_TOKEN -ENV TURBO_TOKEN=$TURBO_TOKEN - -FROM base as builder - -RUN apk add --no-cache libc6-compat -RUN apk update - -WORKDIR /app -RUN npm install turbo --global - -COPY . . -RUN turbo prune --scope=@audius/ddex-webapp-server --docker - -# Install and build backend dist and run app -FROM base AS runner - -WORKDIR /app - -RUN apk add --no-cache python3 py3-pip make g++ curl bash libc6-compat -RUN apk update - -# First install dependencies (as they change less often) -COPY .gitignore .gitignore -COPY --from=builder /app/out/json/ . -COPY --from=builder /app/out/package-lock.json ./package-lock.json -COPY --from=builder /app/scripts ./scripts - -RUN CI=true npm i - -# Build the app and its dependencies -COPY --from=builder /app/out/full/ . -COPY svgr-template.js svgr-template.js -COPY turbo.json turbo.json -RUN npx turbo run build --filter=@audius/ddex-webapp-server - -# Copy the client dist from local filesystem instead of building it in Docker, which hangs -COPY packages/ddex/webapp/server/dist /app/packages/ddex/webapp/server/public - -WORKDIR /app/packages/ddex/webapp/server - -EXPOSE 9000 -CMD ["npm", "run", "start:prod"] diff --git a/scripts/check_secrets.sh b/scripts/check_secrets.sh index db2bb25f452..1c01a490328 100644 --- a/scripts/check_secrets.sh +++ b/scripts/check_secrets.sh @@ -1,23 +1,2 @@ -# used to exercise the `if command is not git command` logic below. change this to an invalid value to test printing out install message -COMMAND_PREFIX='secrets' - -if ! git secrets > /dev/null 2>&1; then - echo "Please install 'git-secrets' from https://github.com/awslabs/git-secrets and run 'git secrets --install' in the audius-protocol/ repo" - exit 1 -fi - -# git secrets --install # this needs to be run once right after install -git secrets --register-aws - -# these are the match rules to add. by default the aws rules only match AWS access id's and keys -# add any additional match strings here -# rds urls -git secrets --add '.*[a-z0-9]*.rds.amazonaws.com:[0-9]*\/.*' - -# match any postgres db with an IP hostname -git secrets --add 'postgres:\/\/.*\:.*@([0-9]*\.?)*:[0-9]{4}\/.*' -git secrets --add --allowed 'postgres:postgres@1\.2\.3\.4:[0-9]{4}\/.*' - # scan the staged files git secrets --scan --cached $(git diff --cached --name-only) - diff --git a/scripts/install-git-secrets.sh b/scripts/install-git-secrets.sh new file mode 100644 index 00000000000..a514025b09e --- /dev/null +++ b/scripts/install-git-secrets.sh @@ -0,0 +1,22 @@ +# used to exercise the `if command is not git command` logic below. change this to an invalid value to test printing out install message +COMMAND_PREFIX='secrets' + +if ! git secrets > /dev/null 2>&1; then + echo "Please install 'git-secrets' from https://github.com/awslabs/git-secrets and run 'git secrets --install' in the audius-protocol/ repo" + exit 1 +fi + +git secrets --add --allowed 'dev-tools/config.json' +git secrets --add --allowed 'scripts/install-git-secrets.sh' +git secrets --add --allowed 'package.json' + +git secrets --register-aws +# these are the match rules to add. by default the aws rules only match AWS access id's and keys +# add any additional match strings here +# rds urls +git secrets --add '.*[a-z0-9]*.rds.amazonaws.com:[0-9]*\/.*' +git secrets --add --allowed 'packages/ddex/.*/README.md:.*' + +# match any postgres db with an IP hostname +git secrets --add 'postgres:\/\/.*\:.*@([0-9]*\.?)*:[0-9]{4}\/.*' +git secrets --add --allowed 'postgres:postgres@1\.2\.3\.4:[0-9]{4}\/.*' diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 97b760a2516..7cb6776d914 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -6,10 +6,15 @@ NC='\033[0m' if [[ -z "${CI}" ]]; then - printf "${GREEN}Installing git hooks...\n${NC}" + printf "${GREEN}Updating git hooks...\n${NC}" npm run install-hooks > /dev/null fi +if [[ -z "${CI}" ]]; then + printf "${GREEN}Updating git secrets...\n${NC}" + npm run install-git-secrets > /dev/null +fi + printf "${GREEN}Applying patches...\n${NC}" npm run patch-package > /dev/null