Merge branch 'development' into feature-dan2-merge

* development: (85 commits) revert: merge development to feature-dan2 (tari-project#6100) (tari-project#6102) chore: merge development to feature-dan2 (tari-project#6100) feat: prevent runtime error with compact error input (tari-project#6096) chore(deps): bump h2 from 0.3.21 to 0.3.24 (tari-project#6091) chore: update monero to latest release (tari-project#6098) chore(deps): bump actions/cache from 3 to 4 (tari-project#6093) chore(fix): ci - move s3 uploads to only on release (tari-project#6094) feat: update codeowners (tari-project#6088) chore: update change log (tari-project#6086) feat: add search kernels method to nodejs client (tari-project#6082) chore: move domain hash names to separate crate (tari-project#6076) chore: new esme release (tari-project#6084) fix: tms validation correctly updating (tari-project#6079) fix: wallet coinbases not validated correctly (tari-project#6074) chore(fix): ci - ffis android - add protobuf for cross-compile (tari-project#6083) chore(fix): include protobuf during build (tari-project#6077) chore: upgrade tonic and prost (tari-project#6067) feat: add tari address as valid string for discovering a peer (tari-project#6075) docs: update disclosure policy (tari-project#6072) chore(ci): ffi extend to build for none mobile platforms (tari-project#6069) ...
sdbondi · Jan 26, 2024 · cc67bfc · cc67bfc
2 parents 9a87165 + ee8846d
commit cc67bfc
Show file tree

Hide file tree

Showing 360 changed files with 9,747 additions and 7,841 deletions.
diff --git a/.github/workflows/base_node_binaries.json b/.github/workflows/base_node_binaries.json
@@ -7,7 +7,7 @@
     "cross": false,
     "target_cpu": "x86-64",
     "target_bins": "--bin minotari_node --bin minotari_console_wallet --bin minotari_merge_mining_proxy --bin minotari_miner",
-    "features": "safe"
+    "features": "default, safe"
   },
   {
     "name": "linux-arm64",
@@ -28,7 +28,7 @@
     "cross": false,
     "target_cpu": "x86-64",
     "target_bins": "--bin minotari_node --bin minotari_console_wallet --bin minotari_merge_mining_proxy --bin minotari_miner",
-    "features": "libtor, safe"
+    "features": "default, safe"
   },
   {
     "name": "macos-arm64",
@@ -38,7 +38,7 @@
     "cross": false,
     "target_cpu": "generic",
     "target_bins": "--bin minotari_node --bin minotari_console_wallet --bin minotari_merge_mining_proxy --bin minotari_miner",
-    "features": "libtor, safe",
+    "features": "default, safe",
     "build_enabled": true
   },
   {

diff --git a/.github/workflows/base_node_binaries.yml b/.github/workflows/base_node_binaries.yml
diff --git a/.github/workflows/build_dockers_workflow.yml b/.github/workflows/build_dockers_workflow.yml
@@ -20,7 +20,7 @@ name: Build docker images - workflow_call/on-demand
         default: x86-64
       features:
         type: string
-        default: safe
+        default: 'safe,grpc'
       version:
         type: string
         description: 'build tag/version'
@@ -183,9 +183,10 @@ jobs:
         with:
           images: |
             #name/${{ matrix.builds.image_name }}
-            #ghcr.io/${{ github.repository }}
             ghcr.io/${{ github.repository_owner }}/${{ matrix.builds.image_name }}
           tags: |
+            type=raw,value=latest-${{ env.TARI_NETWORK }},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+            type=ref,event=tag
             type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
@@ -197,7 +198,6 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          #username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Login to Docker Image Provider

diff --git a/.github/workflows/build_libffis.json b/.github/workflows/build_libffis.json
@@ -9,6 +9,24 @@
     "target": "aarch64-linux-android",
     "cross": true
   },
+  {
+    "runs-on": "ubuntu-latest",
+    "target": "x86_64-unknown-linux-gnu",
+    "cross": false,
+    "build_enabled": true
+  },
+  {
+    "runs-on": "ubuntu-latest",
+    "target": "aarch64-unknown-linux-gnu",
+    "cross": false,
+    "build_enabled": false
+  },
+  {
+    "runs-on": "ubuntu-latest",
+    "target": "riscv64gc-unknown-linux-gnu",
+    "cross": false,
+    "build_enabled": false
+  },
   {
     "runs-on": "macos-latest",
     "target": "x86_64-apple-ios",
@@ -23,5 +41,29 @@
     "runs-on": "macos-latest",
     "target": "aarch64-apple-ios-sim",
     "cross": false
+  },
+  {
+    "runs-on": "macos-latest",
+    "target": "x86_64-apple-darwin",
+    "cross": false,
+    "build_enabled": true
+  },
+  {
+    "runs-on": "macos-latest",
+    "target": "aarch64-apple-darwin",
+    "cross": false,
+    "build_enabled": true
+  },
+  {
+    "runs-on": "windows-latest",
+    "target": "x86_64-pc-windows-msvc",
+    "cross": false,
+    "build_enabled": false
+  },
+  {
+    "runs-on": "windows-latest",
+    "target": "aarch64-pc-windows-msvc",
+    "cross": false,
+    "build_enabled": false
   }
 ]
diff --git a/.github/workflows/build_libffis.yml b/.github/workflows/build_libffis.yml
@@ -9,9 +9,13 @@ name: Build ffi libraries
       - 'build-ffis-*'
   schedule:
     - cron: '05 00 * * *'
+  workflow_dispatch:
 
 env:
   CARGO_UNSTABLE_SPARSE_REGISTRY: true
+  CARGO: cargo
+  # CARGO_OPTIONS: "--verbose"
+  CARGO_OPTIONS: "--release"
   TARI_NETWORK_CHANGELOG: 'development'
   TOOLCHAIN: 'stable'
   ## Must be a JSon string
@@ -30,28 +34,30 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       libffiss: ${{ steps.set-matrix.outputs.libffiss }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout source code
+        uses: actions/checkout@v4
 
       - name: Set Matrix from JSon file
         id: set-matrix
         run: |
           ## build all targets images
-          # matrix=$( jq -s -c .[] .github/workflows/build_libffis_workflow.json )
-          matrix=$( jq -s -c .[] ${{ env.matrix-json-file }} )
+          # matrix=$( jq -s -c .[] ${{ env.matrix-json-file }} )
           #
           ## build only single target image
-          # matrix_selection=$( jq -c '.[] | select( ."runs-on" == "macos-latest" )' ${{ env.matrix-json-file }} )
-          # matrix_selection=$( jq -c '.[] | select( ."runs-on" == "macos-latest" )' ${{ env.matrix-json-file }} )
+          # matrix_selection=$( jq -c '.[] | select( ."runs-on" == "ubuntu-latest" )' ${{ env.matrix-json-file }} )
+          # matrix_selection=$( jq -c '.[] | select( ."target" == "x86_64-linux-android" )' ${{ env.matrix-json-file }} )
           #
           ## buid select target images - build_enabled
-          # matrix_selection=$( jq -c '.[] | select( ."build_enabled" != false )' ${{ env.matrix-json-file }} )
+          matrix_selection=$( jq -c '.[] | select( ."build_enabled" != false )' ${{ env.matrix-json-file }} )
           #
           ## Setup json array
-          # matrix=$(echo ${matrix_selection} | jq -s -c)
+          matrix=$(echo ${matrix_selection} | jq -s -c)
           #
           ## Setup the json build matrix
           # matrix=$(echo ${matrix_selection} | jq -s -c '{"builds": .}')
           #
+          echo $matrix
+          echo $matrix | jq .
           echo "matrix=${matrix}" >> $GITHUB_OUTPUT
           echo "libffiss=${BUILD_LIBFFIS}" >> $GITHUB_OUTPUT
 
@@ -106,11 +112,15 @@ jobs:
           if [[ "${{ matrix.builds.target }}" =~ "android" ]]; then
             echo "CFLAGS=-DMDB_USE_ROBUST=0" >> $GITHUB_ENV
             echo "TARGET_PLATFORM=android" >> $GITHUB_ENV
-          else
+          elif [[ "${{ matrix.builds.target }}" =~ "ios" ]]; then
             echo "TARGET_PLATFORM=ios" >> $GITHUB_ENV
             if [[ "${{ matrix.builds.target }}" =~ "-sim" ]]; then
               echo "TARGET_SIM=-sim" >> $GITHUB_ENV
             fi
+          elif [[ "${{ matrix.builds.target }}" =~ "linux-gnu" ]]; then
+            echo "TARGET_PLATFORM=linux" >> $GITHUB_ENV
+          elif [[ "${{ matrix.builds.target }}" =~ "apple-darwin" ]]; then
+            echo "TARGET_PLATFORM=macos" >> $GITHUB_ENV
           fi
           # Strip begining
           tempEnv="${{ matrix.builds.target }}"
@@ -120,36 +130,63 @@ jobs:
           tempEnv=${tempEnv#*_}
           echo "TARGET_NAME=${tempEnv%_*}" >> $GITHUB_ENV
 
+      - name: Install Linux dependencies - Ubuntu
+        if: ${{ startsWith(runner.os,'Linux') && ( ! matrix.builds.cross ) }}
+        run: |
+          sudo apt-get update
+          sudo bash scripts/install_ubuntu_dependencies.sh
+
+      - name: Install macOS dependencies
+        if: startsWith(runner.os,'macOS')
+        run: |
+          brew install openssl cmake coreutils automake autoconf protobuf
+          rustup target add ${{ matrix.builds.target }}
+
       - name: Setup Rust toolchain
+        if: ${{ ! matrix.builds.cross }}
         uses: dtolnay/rust-toolchain@master
         with:
           components: rustfmt, clippy
           toolchain: ${{ env.TOOLCHAIN }}
           targets: ${{ matrix.builds.target }}
 
-      # Don't use caches for binary builds when releasing. Start from a clean slate.
       - name: Cache cargo files and outputs
-        if: ${{ ! startsWith(github.ref, 'refs/tags/v') }}
+        if: ${{ ( ! startsWith(github.ref, 'refs/tags/v') ) && ( ! matrix.builds.cross ) }}
         uses: Swatinem/rust-cache@v2
+        with:
+          key: ${{ matrix.builds.target }}
 
-      - name: Build ${{ matrix.libffis }} libraries
+      - name: Install and setup cargo cross
+        if: ${{ matrix.builds.cross }}
         shell: bash
         run: |
-          # rustup target list
-          # rustup toolchain list
-          # rustup show
-          if [ "${{ matrix.builds.cross }}" == "true" ]; then
-            cargo install cross
-            cross build --lib --release --locked \
-              --package ${{ matrix.libffis }} \
-              --target=${{ matrix.builds.target }}
-          else
-            rustup target add ${{ matrix.builds.target }}
-            # rustup toolchain install stable-aarch64-unknown-linux-gnu --force-non-host
-            cargo build --lib --release --locked \
-              --package ${{ matrix.libffis }} \
-              --target=${{ matrix.builds.target }}
-          fi
+          cargo install cross
+          echo "CARGO=cross" >> $GITHUB_ENV
+
+      - name: Install rust target/toolchain for native/local cross-compile builds
+        if: ${{ ! matrix.builds.cross  }}
+        # if: ${{ ( ! matrix.builds.cross ) && env.TARGET_PLATFORM != 'ios' }}
+        run: |
+          rustup target add ${{ matrix.builds.target }}
+          # rustup toolchain install stable-${{ matrix.builds.target }} --force-non-host
+          rustup target list
+          rustup toolchain list
+          rustup show
+
+      - name: Show command used for Cargo
+        run: |
+          echo "cargo command is: ${{ env.CARGO }}"
+          echo "cargo options is: ${{ env.CARGO_OPTIONS }}"
+          echo "cross flag: ${{ matrix.builds.cross }}"
+
+      - name: Build ${{ matrix.libffis }} libraries
+        # can't use bash for windows build, perl not supported (openssl)
+        # don't mess with indentation - breaks inline multiline (readiblity)
+        run: >
+          ${{ env.CARGO }} build --lib ${{ env.CARGO_OPTIONS }}
+          --package ${{ matrix.libffis }}
+          --target ${{ matrix.builds.target }}
+          --locked
 
       - name: Prepare ${{ matrix.libffis }} libraries
         shell: bash
@@ -172,7 +209,7 @@ jobs:
           ls -alhtR "${{ runner.temp }}/lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}"
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}
           path: ${{ runner.temp }}/lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}
@@ -194,7 +231,7 @@ jobs:
         run: brew install coreutils
 
       - name: Download iOS libffiss for ${{ matrix.libffis }}
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           # wildcard downloads not supported yet ( minotari_*_ffi-ios-* )
           # name: ${{ matrix.libffis }}-ios
@@ -247,7 +284,7 @@ jobs:
           ls -alhtR
 
       - name: Upload iOS universal libffis artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: lib${{ matrix.libffis }}-ios-universal
           path: libffiss/lib${{ matrix.libffis }}-ios-universal
@@ -299,7 +336,7 @@ jobs:
           ls -alhtR
 
       - name: Upload iOS xcframework libffis artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: lib${{ matrix.libffis }}-ios-xcframework
           path: libffiss/lib${{ matrix.libffis }}-ios-xcframework
@@ -311,7 +348,7 @@ jobs:
     if: ${{ startsWith(github.ref, 'refs/tags/v') }}
     steps:
       - name: Download all ffi libraries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: libffiss
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -45,7 +45,7 @@ jobs:
         # Without restore keys, we lose the ability to get partial matches on caches, and end
         # up with too many cache misses.
         # Use a "small" suffix to use the build caches where possible, but build caches won't use this
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry/index
@@ -85,7 +85,7 @@ jobs:
         # Rust-cache disables a key feature of actions/cache: restoreKeys.
         # Without restore keys, we lose the ability to get partial matches on caches, and end
         # up with too many cache misses.
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry/index
@@ -144,8 +144,10 @@ jobs:
         run: rustup show
       - name: cargo check
         run: cargo check --release --all-targets --workspace --exclude tari_integration_tests --locked
-      - name: cargo check ffi separately
+      - name: cargo check wallet ffi separately
         run: cargo check --release --package minotari_wallet_ffi --locked
+      - name: cargo check chat ffi separately
+        run: cargo check --release --package minotari_chat_ffi --locked
   licenses:
     name: file licenses
     runs-on: [ubuntu-20.04]
@@ -201,7 +203,7 @@ jobs:
       - name: cargo test
         run: cargo nextest run --all-features --release -E "not package(tari_integration_tests)" --profile ci
       - name: upload artifact
-        uses: actions/upload-artifact@v3  # upload test results as artifact
+        uses: actions/upload-artifact@v4  # upload test results as artifact
         if: always()
         with:
           name: test-results
@@ -219,7 +221,7 @@ jobs:
         run: echo $PR_NUM > pr_num.txt
 
       - name: Upload the PR number
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pr_num
           path: ./pr_num.txt
@@ -230,7 +232,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -109,7 +109,7 @@ jobs:
             --retry 2
 
       - name: upload artifact
-        uses: actions/upload-artifact@v3  # upload test results as artifact
+        uses: actions/upload-artifact@v4  # upload test results as artifact
         if: always()
         with:
           name: junit-cucumber
@@ -191,7 +191,7 @@ jobs:
             --retry 2
 
       - name: upload artifact
-        uses: actions/upload-artifact@v3  # upload test results as artifact
+        uses: actions/upload-artifact@v4  # upload test results as artifact
         if: always()
         with:
           name: junit-ffi-cucumber
@@ -203,7 +203,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}