client/rpc: bring up to speed with streaming pins

ipfs · Jun 2, 2023 · 06588e7 · 06588e7
2 parents a197125 + eb265f7
commit 06588e7
Show file tree

Hide file tree

Showing 80 changed files with 4,772 additions and 378 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -0,0 +1,4 @@
+<!--
+PR Creation Checklist
+- [ ] Update Changelog
+-->
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -3,6 +3,8 @@ name: Interop
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -14,6 +16,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   interop-prep:
     if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
@@ -44,7 +50,7 @@ jobs:
           path: cmd/ipfs/ipfs
   interop:
     needs: [interop-prep]
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     defaults:
       run:
@@ -75,7 +81,7 @@ jobs:
           npm install ipfs-interop@^10.0.1
         working-directory: interop
       # Run the interop tests while ignoring the js-js interop test cases
-      - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))'
+      - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))' --parallel
         env:
           LIBP2P_TCP_REUSEPORT: false
           LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
@@ -121,42 +127,9 @@ jobs:
         working-directory: go-ipfs-api
       - run: cmd/ipfs/ipfs shutdown
         if: always()
-  go-ipfs-http-client:
-    needs: [interop-prep]
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    env:
-      TEST_DOCKER: 0
-      TEST_FUSE: 0
-      TEST_VERBOSE: 1
-      TRAVIS: 1
-      GIT_PAGER: cat
-      IPFS_CHECK_RCMGR_DEFAULTS: 1
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - uses: actions/download-artifact@v3
-        with:
-          name: kubo
-          path: cmd/ipfs
-      - run: chmod +x cmd/ipfs/ipfs
-      - uses: actions/checkout@v3
-        with:
-          repository: ipfs/go-ipfs-http-client
-          path: go-ipfs-http-client
-      - uses: protocol/cache-go-action@v1
-        with:
-          name: ${{ github.job }}
-      - run: echo '${{ github.workspace }}/cmd/ipfs' >> $GITHUB_PATH
-      - run: go test -count=1 -v ./...
-        working-directory: go-ipfs-http-client
   ipfs-webui:
     needs: [interop-prep]
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       NO_SANDBOX: true
@@ -191,14 +164,26 @@ jobs:
           key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-${{ github.job }}-
-      - run: |
-          npm ci --prefer-offline --no-audit --progress=false
-          npx playwright install
+      - env:
+          NPM_CACHE_DIR: ${{ steps.npm-cache-dir.outputs.dir }}
+        run: |
+          npm ci --prefer-offline --no-audit --progress=false --cache "$NPM_CACHE_DIR"
+          npx playwright install --with-deps
         working-directory: ipfs-webui
-      - name: Run ipfs-webui@main build and smoke-test to confirm the upstream repo is not broken
-        run: npm test
+      - id: ref
+        run: echo "ref=$(git rev-parse --short HEAD)" | tee -a $GITHUB_OUTPUT
+        working-directory: ipfs-webui
+      - id: state
+        env:
+          GITHUB_REPOSITORY: ipfs/ipfs-webui
+          GITHUB_REF: ${{ steps.ref.outputs.ref }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          echo "state=$(curl -L -H "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_REF/status" --jq '.state')" | tee -a $GITHUB_OUTPUT
+      - name: Build ipfs-webui@main (state=${{ steps.state.outputs.state }})
+        run: npm run test:build
         working-directory: ipfs-webui
-      - name: Test ipfs-webui@main E2E against the locally built Kubo binary
+      - name: Test ipfs-webui@main (state=${{ steps.state.outputs.state }}) E2E against the locally built Kubo binary
         run: npm run test:e2e
         env:
           IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -8,6 +8,8 @@ on:
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ master ]
+    paths-ignore:
+      - '**/*.md'
   schedule:
     - cron: '30 12 * * 2'
 

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -3,6 +3,8 @@ name: Docker Build
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'

diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml
@@ -5,6 +5,8 @@ on:
     branches:
       - master
   pull_request:
+    paths-ignore:
+      - '**/*.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}

diff --git a/.github/workflows/gobuild.yml b/.github/workflows/gobuild.yml
@@ -3,6 +3,8 @@ name: Go Build
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  go-build-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   go-build:
-    needs: [go-build-runner]
-    runs-on: ${{ fromJSON(needs.go-build-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       TEST_DOCKER: 0

diff --git a/.github/workflows/golang-analysis.yml b/.github/workflows/golang-analysis.yml
@@ -3,6 +3,8 @@ name: Go Check
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'

diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml
@@ -3,6 +3,8 @@ name: Go Lint
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'

diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml
@@ -3,6 +3,8 @@ name: Go Test
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  go-test-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   go-test:
-    needs: [go-test-runner]
-    runs-on: ${{ fromJSON(needs.go-test-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     env:
       TEST_DOCKER: 0
@@ -41,8 +40,11 @@ jobs:
         with:
           name: ${{ github.job }}
       - name: 👉️ If this step failed, go to «Summary» (top left) → inspect the «Failures/Errors» table
+        env:
+          # increasing parallelism beyond 2 doesn't speed up the tests much
+          PARALLEL: 2
         run: |
-          make -j 2 test/unit/gotest.junit.xml &&
+          make -j "$PARALLEL" test/unit/gotest.junit.xml &&
             [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0

diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml
diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml
@@ -3,6 +3,8 @@ name: Sharness
 on:
   workflow_dispatch:
   pull_request:
+    paths-ignore:
+      - '**/*.md'
   push:
     branches:
       - 'master'
@@ -12,12 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  sharness-runner:
-    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
-    uses: ipfs/kubo/.github/workflows/runner.yml@master
   sharness-test:
-    needs: [sharness-runner]
-    runs-on: ${{ fromJSON(needs.sharness-runner.outputs.config).labels }}
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     defaults:
       run:
@@ -57,7 +56,8 @@ jobs:
           TEST_EXPENSIVE: 1
           IPFS_CHECK_RCMGR_DEFAULTS: 1
           CONTINUE_ON_S_FAILURE: 1
-          PARALLEL: ${{ fromJSON(needs.sharness-runner.outputs.config).parallel }}
+          # increasing parallelism beyond 10 doesn't speed up the tests much
+          PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }}
       - name: Upload coverage report
         uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
         if: failure() || success()
@@ -86,12 +86,12 @@ jobs:
       - name: Upload one-page HTML report to S3
         id: one-page
         uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main
-        if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
+        if: github.repository == 'ipfs/kubo' && (failure() || success())
         with:
           source: kubo/test/sharness/test-results/sharness.html
           destination: sharness.html
       - name: Upload one-page HTML report
-        if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success())
+        if: github.repository != 'ipfs/kubo' && (failure() || success())
         uses: actions/upload-artifact@v3
         with:
           name: sharness.html
@@ -106,18 +106,18 @@ jobs:
       - name: Upload full HTML report to S3
         id: full
         uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main
-        if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
+        if: github.repository == 'ipfs/kubo' && (failure() || success())
         with:
           source: kubo/test/sharness/test-results/sharness-html
           destination: sharness-html/
       - name: Upload full HTML report
-        if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success())
+        if: github.repository != 'ipfs/kubo' && (failure() || success())
         uses: actions/upload-artifact@v3
         with:
           name: sharness-html
           path: kubo/test/sharness/test-results/sharness-html
       - name: Add S3 links to the summary
-        if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
+        if: github.repository == 'ipfs/kubo' && (failure() || success())
         run: echo "$MD" >> $GITHUB_STEP_SUMMARY
         env:
           MD: |

diff --git a/.golangci.yml b/.golangci.yml
@@ -4,5 +4,8 @@ linters:
 
 linters-settings:
   stylecheck:
+    checks:
+    - all
+    - '-ST1003'
     dot-import-whitelist:
       - github.com/ipfs/kubo/test/cli/testutils
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
 # Kubo Changelogs
 
+- [v0.21](docs/changelogs/v0.21.md)
 - [v0.20](docs/changelogs/v0.20.md)
 - [v0.19](docs/changelogs/v0.19.md)
 - [v0.18](docs/changelogs/v0.18.md)

diff --git a/README.md b/README.md
@@ -187,10 +187,10 @@ $ ipfs get /ipns/dist.ipfs.tech/kubo/$VERSION/kubo_$VERSION_windows-amd64.zip
 With the purely functional package manager [Nix](https://nixos.org/nix/) you can install kubo (go-ipfs) like this:
 
 ```
-$ nix-env -i ipfs
+$ nix-env -i kubo
 ```
 
-You can also install the Package by using its attribute name, which is also `ipfs`.
+You can also install the Package by using its attribute name, which is also `kubo`.
 
 #### Solus
 
@@ -251,10 +251,10 @@ $ sudo port install ipfs
 In macOS you can use the purely functional package manager [Nix](https://nixos.org/nix/):
 
 ```
-$ nix-env -i ipfs
+$ nix-env -i kubo
 ```
 
-You can also install the Package by using its attribute name, which is also `ipfs`.
+You can also install the Package by using its attribute name, which is also `kubo`.
 
 #### Homebrew