Merge pull request #9936 from ipfs/release-v0.21.0

.github/pull_request_template.md

@@ -0,0 +1,4 @@
+<!--
+PR Creation Checklist
+- [ ] Update Changelog
+-->

.github/workflows/build.yml

@@ -3,17 +3,23 @@ name: Interop
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
 
 env:
-  GO_VERSION: 1.19.1
+  GO_VERSION: 1.19.x
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   interop-prep:
     if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
@@ -44,7 +50,7 @@ jobs:
           path: cmd/ipfs/ipfs
   interop:
     needs: [interop-prep]
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     defaults:
       run:
@@ -75,7 +81,7 @@ jobs:
           npm install ipfs-interop@^10.0.1
         working-directory: interop
       # Run the interop tests while ignoring the js-js interop test cases
-      - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))'
+      - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))' --parallel
         env:
           LIBP2P_TCP_REUSEPORT: false
           LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
@@ -121,42 +127,9 @@ jobs:
         working-directory: go-ipfs-api
       - run: cmd/ipfs/ipfs shutdown
         if: always()
-  go-ipfs-http-client:
-    needs: [interop-prep]
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    env:
-      TEST_DOCKER: 0
-      TEST_FUSE: 0
-      TEST_VERBOSE: 1
-      TRAVIS: 1
-      GIT_PAGER: cat
-      IPFS_CHECK_RCMGR_DEFAULTS: 1
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - uses: actions/download-artifact@v3
-        with:
-          name: kubo
-          path: cmd/ipfs
-      - run: chmod +x cmd/ipfs/ipfs
-      - uses: actions/checkout@v3
-        with:
-          repository: ipfs/go-ipfs-http-client
-          path: go-ipfs-http-client
-      - uses: protocol/cache-go-action@v1
-        with:
-          name: ${{ github.job }}
-      - run: echo '${{ github.workspace }}/cmd/ipfs' >> $GITHUB_PATH
-      - run: go test -count=1 -v ./...
-        working-directory: go-ipfs-http-client
   ipfs-webui:
     needs: [interop-prep]
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       NO_SANDBOX: true
@@ -191,14 +164,26 @@ jobs:
           key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-${{ github.job }}-
-      - run: |
-          npm ci --prefer-offline --no-audit --progress=false
-          npx playwright install
+      - env:
+          NPM_CACHE_DIR: ${{ steps.npm-cache-dir.outputs.dir }}
+        run: |
+          npm ci --prefer-offline --no-audit --progress=false --cache "$NPM_CACHE_DIR"
+          npx playwright install --with-deps
         working-directory: ipfs-webui
-      - name: Run ipfs-webui@main build and smoke-test to confirm the upstream repo is not broken
-        run: npm test
+      - id: ref
+        run: echo "ref=$(git rev-parse --short HEAD)" | tee -a $GITHUB_OUTPUT
+        working-directory: ipfs-webui
+      - id: state
+        env:
+          GITHUB_REPOSITORY: ipfs/ipfs-webui
+          GITHUB_REF: ${{ steps.ref.outputs.ref }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          echo "state=$(curl -L -H "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_REF/status" --jq '.state')" | tee -a $GITHUB_OUTPUT
+      - name: Build ipfs-webui@main (state=${{ steps.state.outputs.state }})
+        run: npm run test:build
         working-directory: ipfs-webui
-      - name: Test ipfs-webui@main E2E against the locally built Kubo binary
+      - name: Test ipfs-webui@main (state=${{ steps.state.outputs.state }}) E2E against the locally built Kubo binary
         run: npm run test:e2e
         env:
           IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs

.github/workflows/codeql-analysis.yml

@@ -8,6 +8,8 @@ on:
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ master ]
+    paths-ignore:
+      - '**/*.md'
   schedule:
     - cron: '30 12 * * 2'
 

.github/workflows/docker-build.yml

@@ -3,6 +3,8 @@ name: Docker Build
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -25,6 +27,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - uses: actions/checkout@v3
       - run: docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG .

.github/workflows/gateway-conformance.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - master
   pull_request:
+    paths-ignore:
+      - '**/*.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
@@ -21,7 +23,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.0
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.1
         with:
           output: fixtures
 
@@ -38,8 +40,8 @@ jobs:
         run: make build
         working-directory: kubo-gateway
 
-      # 3. Start the kubo-gateway
-      - name: Start kubo-gateway
+      # 3. Init the kubo-gateway
+      - name: Init kubo-gateway
         env:
           GATEWAY_PUBLIC_GATEWAYS: |
             {
@@ -56,16 +58,36 @@ jobs:
         run: |
           ./ipfs init
           ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
-          ./ipfs daemon --offline &
         working-directory: kubo-gateway/cmd/ipfs
 
       # 4. Populate the Kubo gateway with the gateway-conformance fixtures
       - name: Import fixtures
-        run: find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
+        run: |
+          # Import car files
+          find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
+
+          # Import ipns records
+          records=$(find ./fixtures -name '*.ipns-record')
+          for record in $records
+          do
+              key=$(basename -s .ipns-record "$record" | cut -d'_' -f1)
+              kubo-gateway/cmd/ipfs/ipfs routing put --allow-offline "/ipns/$key" "$record"
+          done
+
+          # Import dnslink records
+          # the IPFS_NS_MAP env will be used by the daemon
+          export IPFS_NS_MAP=$(cat ./fixtures/dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")')
+          echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV
+
+      # 5. Start the kubo-gateway
+      - name: Start kubo-gateway
+        run: |
+          ./ipfs daemon --offline &
+        working-directory: kubo-gateway/cmd/ipfs
 
-      # 5. Run the gateway-conformance tests
+      # 6. Run the gateway-conformance tests
       - name: Run gateway-conformance tests
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.0
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.1
         with:
           gateway-url: http://127.0.0.1:8080
           json: output.json
@@ -74,7 +96,7 @@ jobs:
           markdown: output.md
           args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
 
-      # 6. Upload the results
+      # 7. Upload the results
       - name: Upload MD summary
         if: failure() || success()
         run: cat output.md >> $GITHUB_STEP_SUMMARY

.github/workflows/gobuild.yml

@@ -3,6 +3,8 @@ name: Go Build
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  go-build-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   go-build:
-    needs: [go-build-runner]
-    runs-on: ${{ fromJSON(needs.go-build-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       TEST_DOCKER: 0
@@ -31,7 +30,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - uses: actions/checkout@v3
       - uses: protocol/cache-go-action@v1
         with:

.github/workflows/golang-analysis.yml

@@ -3,6 +3,8 @@ name: Go Check
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'

.github/workflows/golint.yml

@@ -3,6 +3,8 @@ name: Go Lint
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -29,7 +31,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - uses: actions/checkout@v3
       - uses: protocol/cache-go-action@v1
         with:

.github/workflows/gotest.yml

@@ -3,6 +3,8 @@ name: Go Test
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  go-test-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   go-test:
-    needs: [go-test-runner]
-    runs-on: ${{ fromJSON(needs.go-test-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       TEST_DOCKER: 0
@@ -33,16 +32,19 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.1
+          go-version: 1.19.x
       - name: Check out Kubo
         uses: actions/checkout@v3
       - name: Restore Go cache
         uses: protocol/cache-go-action@v1
         with:
           name: ${{ github.job }}
       - name: 👉️ If this step failed, go to «Summary» (top left) → inspect the «Failures/Errors» table
+        env:
+          # increasing parallelism beyond 2 doesn't speed up the tests much
+          PARALLEL: 2
         run: |
-          make -j 2 test/unit/gotest.junit.xml &&
+          make -j "$PARALLEL" test/unit/gotest.junit.xml &&
             [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0