Merge pull request #54 from okp4/feat/buf

⚙️ Generate proto from buf
okp4 · May 31, 2023 · 30591d2 · 30591d2
2 parents cf1a439 + 5c93185
commit 30591d2
Show file tree

Hide file tree

Showing 166 changed files with 775 additions and 12,067 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -14,52 +14,47 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-kotlin:
+  matrix:
     runs-on: ubuntu-22.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          java-version: "11"
-
-      - name: Configure gradle.properties
-        run: |
-          cat <<EOF >> kotlin/gradle.properties
-          maven.credentials.username=${{ secrets.MAVEN_REPOSITORY_USERNAME }}
-          maven.credentials.password=${{ secrets.MAVEN_REPOSITORY_PASSWORD }}
-          EOF
-
-      - name: Build kotlin
-        uses: gradle/gradle-build-action@v2
-        with:
-          arguments: build -x check
-          build-root-directory: kotlin
+      - uses: actions/checkout@v2
+      - id: set-matrix
+        run: echo "::set-output name=matrix::$(cat proto.json | jq -c .)"
 
   build-typescript:
     runs-on: ubuntu-22.04
+    needs: matrix
+    strategy:
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
 
+      - name: Configure buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ github.token }}
+
+      - name: Generate proto
+        run: |
+          buf generate --template buf.gen.ts.yaml ${{ matrix.proto.buf }} -o ${{ matrix.proto.directory.ts }}
+
       - name: Setup node environment (for building)
         uses: actions/setup-node@v3
         with:
-          node-version: 16.14.0
+          node-version: 16.19.0
 
       - name: Fetch dependencies
-        run: yarn --cwd ts
-
-      - name: Generate sources
-        run: yarn --cwd ts generate
+        run: yarn --cwd ${{ matrix.proto.directory.ts }}
 
       - name: Build
-        run: yarn --cwd ts build
+        run: yarn --cwd ${{ matrix.proto.directory.ts }} build
 
   build-rust:
     runs-on: ubuntu-22.04
+    needs: matrix
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
@@ -74,6 +69,7 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: |
             ${{ runner.os }}-cargo-
+
       - name: Setup rust
         uses: actions-rs/toolchain@v1
         with:
@@ -84,19 +80,30 @@ jobs:
       - name: Install cargo make
         uses: davidB/rust-cargo-make@v1
 
+      - name: Configure buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ github.token }}
+
+      - name: Generate proto
+        run: |
+          jq -c '.proto[]' proto.json | while read i; do
+            BUF=$(echo $i | jq -cr '.buf')
+            BUF_YAML=$(grep -q "cosmos/cosmos-sdk" <<< "$BUF" && echo "buf.gen.rust.cosmos.yaml" || echo "buf.gen.rust.yaml")
+            DIRECTORY=$(echo $i | jq -cr '.directory.rust')
+            buf generate --template $BUF_YAML $BUF -o $DIRECTORY
+          done
+
       - name: Build rust
         working-directory: ./rust
-        run: cargo make build
+        run: |
+          cargo make build
 
   build-python:
     runs-on: ubuntu-22.04
+    needs: matrix
     strategy:
-      matrix:
-        target:
-          - name: cosmos_sdk_grpc_client
-            includes: "-I=../../proto/cosmos-sdk $(find ../../proto/cosmos-sdk -type f -name '*.proto' | tr '\n' ' ')"
-          - name: okp4_grpc_client
-            includes: "-I=../../proto/cosmos-sdk -I=../../proto/okp4 $(find ../../proto -type f -name '*.proto' | tr '\n' ' ')"
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v2
@@ -108,20 +115,23 @@ jobs:
       - name: Install Python Poetry
         uses: abatilo/actions-poetry@v2.3.0
         with:
-          poetry-version: 1.1.11
+          poetry-version: 1.4.1
 
-      - name: Install project dependencies
-        working-directory: ./python/${{ matrix.target.name }}
+      - name: Configure buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ github.token }}
+
+      - name: Generate proto
         run: |
-          poetry install
+          buf generate --template buf.gen.python.yaml ${{ matrix.proto.buf }} -o ${{ matrix.proto.directory.python }}
 
       - name: Install project dependencies
-        working-directory: ./python/${{ matrix.target.name }}
+        working-directory: ./${{ matrix.proto.directory.python }}/../
         run: |
-          mkdir -p ${{ matrix.target.name }}
-          poetry run python -m grpc_tools.protoc ${{ matrix.target.includes }} --grpc_python_out=${{ matrix.target.name }}
+          poetry install
 
       - name: Build project
-        working-directory: ./python/${{ matrix.target.name }}
+        working-directory: ./${{ matrix.proto.directory.python }}/../
         run: |
           poetry build
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -10,40 +10,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  publish-maven:
+  matrix:
     runs-on: ubuntu-22.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          java-version: "11"
-
-      - name: Configure gradle.properties
-        run: |
-          cat <<EOF >> kotlin/gradle.properties
-          maven.credentials.username=${{ secrets.MAVEN_REPOSITORY_USERNAME }}
-          maven.credentials.password=${{ secrets.MAVEN_REPOSITORY_PASSWORD }}
-          EOF
-
-      - name: Publish maven snapshot
-        if: "!contains(github.ref, 'refs/tags/')"
-        uses: gradle/gradle-build-action@v2
-        with:
-          arguments: publish
-          build-root-directory: kotlin
-
-      - name: Publish maven release
-        if: contains(github.ref, 'refs/tags/')
-        uses: gradle/gradle-build-action@v2
-        with:
-          arguments: publish -Prelease
-          build-root-directory: kotlin
+      - uses: actions/checkout@v2
+      - id: set-matrix
+        run: echo "::set-output name=matrix::$(cat proto.json | jq -c .)"
 
   publish-npm-okp4:
     runs-on: ubuntu-22.04
+    needs: matrix
+    strategy:
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
@@ -52,10 +32,19 @@ jobs:
         id: project_context
         uses: FranzDiebold/github-env-vars-action@v2.7.0
 
+      - name: Configure buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ github.token }}
+
+      - name: Generate proto
+        run: |
+          buf generate --template buf.gen.ts.yaml ${{ matrix.proto.buf }} -o ${{ matrix.proto.directory.ts }}
+
       - name: Setup node environment (for publishing)
         uses: actions/setup-node@v3
         with:
-          node-version: 16.14.0
+          node-version: 16.19.0
           registry-url: "https://npm.pkg.github.com"
           scope: "@okp4"
 
@@ -65,9 +54,9 @@ jobs:
 
           DATE=$(date +%Y%m%d%H%M%S)
 
-          yarn --cwd ts && yarn --cwd ts generate && yarn --cwd ts build
+          yarn --cwd ${{ matrix.proto.directory.ts }} && yarn --cwd ${{ matrix.proto.directory.ts }} build
 
-          publish=(yarn --cwd ts publish --no-git-tag-version --non-interactive)
+          publish=(yarn --cwd ${{ matrix.proto.directory.ts }} publish --no-git-tag-version --non-interactive)
 
           if [[ $GITHUB_REF == refs/tags/v* ]]; then
             publish+=(--tag latest)
@@ -85,6 +74,9 @@ jobs:
   publish-npm-public:
     if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
     runs-on: ubuntu-22.04
+    needs: matrix
+    strategy:
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
@@ -93,20 +85,29 @@ jobs:
         id: project_context
         uses: FranzDiebold/github-env-vars-action@v2.7.0
 
+      - name: Configure buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ github.token }}
+
+      - name: Generate proto
+        run: |
+          buf generate --template buf.gen.ts.yaml ${{ matrix.proto.buf }} -o ${{ matrix.proto.directory.ts }}
+
       - name: Setup node environment (for publishing)
         uses: actions/setup-node@v3
         with:
-          node-version: 16.14.0
+          node-version: 16.19.0
           registry-url: "https://registry.npmjs.org"
           scope: "@okp4"
 
       - name: Publish package
         run: |
           set -eu
 
-          yarn --cwd ts && yarn --cwd ts generate && yarn --cwd ts build
+          yarn --cwd ${{ matrix.proto.directory.ts }} && yarn --cwd ${{ matrix.proto.directory.ts }} build
 
-          publish=(yarn --cwd ts publish --access=public --no-git-tag-version --non-interactive --tag latest)
+          publish=(yarn --cwd ${{ matrix.proto.directory.ts }} publish --access=public --no-git-tag-version --non-interactive --tag latest)
 
           echo "🚀 Publishing npm package with following command line: ${publish[@]}"
           "${publish[@]}"
@@ -130,6 +131,20 @@ jobs:
       - name: Install cargo make
         uses: davidB/rust-cargo-make@v1
 
+      - name: Configure buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ github.token }}
+
+      - name: Generate proto
+        run: |
+          jq -c '.proto[]' proto.json | while read i; do
+            BUF=$(echo $i | jq -cr '.buf')
+            BUF_YAML=$(grep -q "cosmos/cosmos-sdk" <<< "$BUF" && echo "buf.gen.rust.cosmos.yaml" || echo "buf.gen.rust.yaml")
+            DIRECTORY=$(echo $i | jq -cr '.directory.rust')
+            buf generate --template $BUF_YAML $BUF -o $DIRECTORY
+          done
+
       - name: Publish packages
         working-directory: ./rust
         run: |
@@ -139,13 +154,9 @@ jobs:
   publish-pypi:
     if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
     runs-on: ubuntu-22.04
+    needs: matrix
     strategy:
-      matrix:
-        target:
-          - name: cosmos_sdk_grpc_client
-            includes: "-I=../../proto/cosmos-sdk $(find ../../proto/cosmos-sdk -type f -name '*.proto' | tr '\n' ' ')"
-          - name: okp4_grpc_client
-            includes: "-I=../../proto/cosmos-sdk -I=../../proto/okp4 $(find ../../proto -type f -name '*.proto' | tr '\n' ' ')"
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
@@ -161,17 +172,22 @@ jobs:
       - name: Install Python Poetry
         uses: abatilo/actions-poetry@v2.3.0
         with:
-          poetry-version: 1.1.11
+          poetry-version: 1.4.1
+
+      - name: Configure buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ github.token }}
+
+      - name: Generate proto
+        run: |
+          buf generate --template buf.gen.python.yaml ${{ matrix.proto.buf }} -o ${{ matrix.proto.directory.python }}
 
       - name: Publish package
-        working-directory: ./python/${{ matrix.target.name }}
+        working-directory: ./${{ matrix.proto.directory.python }}/../
         run: |
           poetry install
-
-          mkdir -p ${{ matrix.target.name }}
-          poetry run python -m grpc_tools.protoc ${{ matrix.target.includes }} --grpc_python_out=${{ matrix.target.name }}
-
           poetry build
 
-          echo "🚀 Publishing pypi package '${{ matrix.target.name }}'"
+          echo "🚀 Publishing pypi package '${{ matrix.proto.directory.python }}'"
           poetry publish  -u __token__ -p "${{ secrets.PYPI_API_TOKEN }}"
diff --git a/.gitignore b/.gitignore
@@ -11,5 +11,4 @@ python/okp4_grpc_client/okp4_grpc_client
 python/cosmos_sdk_grpc_client/cosmos_sdk_grpc_client
 
 rust/target/
-rust/cosmos_sdk_grpc_client/src/
-rust/okp4_grpc_client/src/
+rust/**/src/gen/