ci: upgrade ci to support macos and automated releases

.github/workflows/ci.yml

@@ -20,55 +20,120 @@ on:
 
 jobs:
   build:
-    name: Test CLN=${{ inputs.cln-version }} PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
+    name: Test CLN=${{ inputs.cln-version }}, OS=${{ matrix.os }}, PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
     strategy:
         fail-fast: false
         matrix:
-            bitcoind-version: ["26.0"]
+            bitcoind-version: ["26.1"]
             experimental: [1]
             deprecated: [0]
             python-version: ["3.8", "3.12"]
-
-    runs-on: ubuntu-latest
-
+            os: ["ubuntu-latest", "macos-12"]
+
+    runs-on: ${{ matrix.os }}
+
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
 
+    - name: Create cache paths
+      run: |
+        sudo mkdir /usr/local/libexec
+        sudo mkdir /usr/local/libexec/c-lightning
+        sudo mkdir /usr/local/libexec/c-lightning/plugins
+        sudo chown -R $USER /usr/local/libexec
+
+    - name: Cache CLN
+      id: cache-cln
+      uses: actions/cache@v4
+      with:
+        path: |
+          /usr/local/bin/lightning*
+          /usr/local/libexec/c-lightning
+        key: cache-cln-${{ inputs.cln-version }}-${{ runner.os }}
+
+    - name: Cache bitcoind
+      id: cache-bitcoind
+      uses: actions/cache@v4
+      with:
+        path: /usr/local/bin/bitcoin*
+        key: cache-bitcoind-${{ matrix.bitcoind-version }}-${{ runner.os }}
+
     - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries
+      if: ${{ steps.cache-bitcoind.outputs.cache-hit != 'true' }}
       run: |
         export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
-        wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
-        tar -xzf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
+        if [[ "${{ matrix.os }}" =~ "ubuntu" ]]; then
+          export TARGET_ARCH="x86_64-linux-gnu"
+        fi
+        if [[ "${{ matrix.os }}" =~ "macos" ]]; then
+          export TARGET_ARCH="x86_64-apple-darwin"
+        fi
+        wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
+        tar -xzf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
         sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
-        rm -rf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz bitcoin-${BITCOIND_VERSION}
-      
+        rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION}
+
     - name: Download Core Lightning ${{ inputs.cln-version }} & install binaries
+      if: ${{ contains(matrix.os, 'ubuntu') &&  steps.cache-cln.outputs.cache-hit != 'true' }}
       run: |
           url=$(curl -s https://api.github.com/repos/ElementsProject/lightning/releases/tags/${{ inputs.cln-version }} \
             | jq '.assets[] | select(.name | contains("22.04")) | .browser_download_url' \
             | tr -d '\"')
           wget $url
           sudo tar -xvf ${url##*/} -C /usr/local --strip-components=2
           echo "CLN_VERSION=$(lightningd --version)" >> "$GITHUB_OUTPUT"
-        
+
+    - name: Set up Rust
+      if:  ${{ inputs.tagged-release == false}}
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Set up protoc
+      if: ${{ inputs.tagged-release == false}} || contains(matrix.os, 'macos') && ${{ steps.cache-cln.outputs.cache-hit != 'true' }}
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install Python dependencies
+    - name: Checkout Core Lightning ${{ inputs.cln-version }}
+      if: ${{ contains(matrix.os, 'macos') && steps.cache-cln.outputs.cache-hit != 'true' }}
+      uses: actions/checkout@v4
+      with:
+        repository: 'ElementsProject/lightning'
+        path: 'lightning'
+        ref: ${{ inputs.cln-version }}
+        submodules: 'recursive'
+
+    - name: Install Python and System dependencies
       run: |
-        pip3 install --user "pyln-proto<=${{ inputs.pyln-version }}" "pyln-client<=${{ inputs.pyln-version }}" "pyln-testing<=${{ inputs.pyln-version }}"
-        pip3 install --user pytest-xdist
+        if [[ "${{ matrix.os }}" =~ "macos" ]]; then
+          brew install autoconf automake libtool gnu-sed gettext libsodium sqlite
+        fi
+        python -m venv venv
+        source venv/bin/activate
+        python -m pip install -U pip poetry wheel
+        pip3 install "pyln-proto<=${{ inputs.pyln-version }}" "pyln-client<=${{ inputs.pyln-version }}" "pyln-testing<=${{ inputs.pyln-version }}"
+        pip3 install pytest-xdist pytest-test-groups pytest-timeout
 
-    - name: Set up Rust
-      if:  ${{ inputs.tagged-release == false}}
-      uses: dtolnay/rust-toolchain@stable
+    - name: Compile Core Lightning ${{ inputs.cln-version }} & install binaries
+      if: ${{ contains(matrix.os, 'macos') && steps.cache-cln.outputs.cache-hit != 'true' }}
+      run: |
+        export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
+        export COMPAT=${{ matrix.deprecated }}
+        export VALGRIND=0
+        source venv/bin/activate
 
-    - name: Set up protoc
-      if: ${{ inputs.tagged-release == false}}
-      uses: arduino/setup-protoc@v3
+        cd lightning
+
+        poetry lock
+        poetry install
+        ./configure --disable-valgrind
+        poetry run make
+        sudo make install
 
     - name: Get plugin binary
       run: |
@@ -89,4 +154,6 @@ jobs:
         export TEST_DEBUG=1
         export TRAVIS=1
         export VALGRIND=0
+        export PYTEST_TIMEOUT=600
+        source venv/bin/activate
         pytest -n=5 tests/test_*.py

.github/workflows/main_v23.08.yml

@@ -9,6 +9,7 @@ on:
             - 'LICENSE'
             - '.gitignore'
             - 'coffee.yml'
+            - 'tests/setup.sh'
     pull_request:
     workflow_dispatch:
 

.github/workflows/main_v23.11.yml

@@ -9,6 +9,7 @@ on:
             - 'LICENSE'
             - '.gitignore'
             - 'coffee.yml'
+            - 'tests/setup.sh'
     pull_request:
     workflow_dispatch:
 

.github/workflows/main_v24.02.yml

@@ -9,6 +9,7 @@ on:
             - 'LICENSE'
             - '.gitignore'
             - 'coffee.yml'
+            - 'tests/setup.sh'
     pull_request:
     workflow_dispatch:
 

.github/workflows/release.yml

@@ -0,0 +1,104 @@
+name: Build and release
+on:
+    push:
+      tags:
+      - 'v*'
+
+jobs:
+    build:
+        name: build release binaries on ${{ matrix.os }}
+        strategy:
+            fail-fast: false
+            matrix:
+                os: ["ubuntu-latest", "macos-12"]
+        runs-on: ${{ matrix.os }}
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v4
+            - name: Install rust
+              id: rust
+              uses: dtolnay/rust-toolchain@stable
+            - name: Install cross
+              if: contains(matrix.os, 'ubuntu')
+              run: |
+                cargo install cross --git https://github.com/cross-rs/cross
+            - name: Build unix
+              id: unix_build
+              if: contains(matrix.os, 'ubuntu')
+              run: |
+                cross build --profile optimized --locked --target x86_64-unknown-linux-gnu
+                cross build --profile optimized --locked --target armv7-unknown-linux-gnueabihf
+                cross build --profile optimized --locked --target aarch64-unknown-linux-gnu
+                tar -czf "${{ github.event.repository.name }}-${{github.ref_name}}-aarch64-linux-gnu.tar.gz" --transform 's|.*/||' "target/aarch64-unknown-linux-gnu/optimized/${{ github.event.repository.name }}"
+                tar -czf "${{ github.event.repository.name }}-${{github.ref_name}}-armv7-linux-gnueabihf.tar.gz" --transform 's|.*/||' "target/armv7-unknown-linux-gnueabihf/optimized/${{ github.event.repository.name }}"
+                tar -czf "${{ github.event.repository.name }}-${{github.ref_name}}-x86_64-linux-gnu.tar.gz" --transform 's|.*/||' "target/x86_64-unknown-linux-gnu/optimized/${{ github.event.repository.name }}"
+                ls -alh
+            - name: Build macos
+              id: macos_build
+              if: contains(matrix.os, 'macos')
+              run: |
+                rustup target add aarch64-apple-darwin
+                export CROSSBUILD_MACOS_SDK="macosx13.1"
+                export SDKROOT=$(xcrun -sdk $CROSSBUILD_MACOS_SDK --show-sdk-path)
+                export MACOSX_DEPLOYMENT_TARGET=12.0
+                cargo build --profile optimized --locked --target=x86_64-apple-darwin
+                cargo build --profile optimized --locked --target=aarch64-apple-darwin
+                lipo -create -output target/${{ github.event.repository.name }} target/aarch64-apple-darwin/optimized/${{ github.event.repository.name }} target/x86_64-apple-darwin/optimized/${{ github.event.repository.name }}
+                ditto -c -k --sequesterRsrc target/${{ github.event.repository.name }} ${{ github.event.repository.name }}-${{github.ref_name}}-universal-apple-darwin.zip
+                otool -l target/aarch64-apple-darwin/optimized/${{ github.event.repository.name }} | grep -A 5 LC_BUILD_VERSION
+                otool -l target/x86_64-apple-darwin/optimized/${{ github.event.repository.name }} | grep -A 5 LC_BUILD_VERSION
+                echo "macos_version=$MACOSX_DEPLOYMENT_TARGET" >> "$GITHUB_OUTPUT"
+                echo $(xcodebuild -showsdks)
+                ls -alh
+            - name: Upload unix artifacts
+              uses: actions/upload-artifact@v4
+              if: contains(matrix.os, 'ubuntu')
+              with:
+                name: unix-binaries
+                path: |
+                    ${{ github.event.repository.name }}-${{github.ref_name}}-*.tar.gz
+            - name: Upload macos artifacts
+              uses: actions/upload-artifact@v4
+              if: contains(matrix.os, 'macos')
+              with:
+                name: macos-binaries
+                path: |
+                    ${{ github.event.repository.name }}-${{github.ref_name}}-universal-apple-darwin.zip
+            - name: Get rust version
+              id: rversion
+              run: |
+                echo "rust_version=$(rustc --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"
+        outputs:
+            rust-version: ${{ steps.rversion.outputs.rust_version }}
+            macos-version: ${{ steps.macos_build.outputs.macos_version }}
+
+    release:
+        name: Github Release
+        needs: [build]
+        runs-on: "ubuntu-latest"
+        permissions:
+            contents: write
+        steps:
+            - name: Get semver version from tag
+              id: tag_name
+              run: echo "current_version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT"
+            - name: Checkout code
+              uses: actions/checkout@v4
+            - name: Get Changelog Entry
+              id: changelog_reader
+              uses: mindsers/changelog-reader-action@v2
+              with:
+                validation_level: warn
+                version: ${{ steps.tag_name.outputs.current_version }}
+                path: ./CHANGELOG.md
+            - name: Download Artifacts
+              uses: actions/download-artifact@v4
+              with:
+                merge-multiple: true
+            - name: Release
+              uses: ncipollo/release-action@v1
+              with:
+                allowUpdates: false
+                artifactErrorsFailBuild: true
+                body: "${{ steps.changelog_reader.outputs.changes }} \n\n### Release binaries info\n\n- Release binaries were built using rust ${{ needs.build.outputs.rust-version }}\n- Linux release binaries require glibc>=2.31\n- MacOS release binary requires MacOS>=${{ needs.build.outputs.macos-version }}"
+                artifacts: "${{ github.event.repository.name }}-${{github.ref_name}}-*.tar.gz,${{ github.event.repository.name }}-${{github.ref_name}}-*.zip"

tests/setup.sh

@@ -5,41 +5,64 @@ script_dir=$(dirname -- "$(readlink -f -- "$0")")
 
 cargo_toml_path="$script_dir/../Cargo.toml"
 
-# Use grep and awk to extract the version
+# Use grep and awk to extract the name and version
+name=$(awk -F'=' '/^\[package\]/ { in_package = 1 } in_package && /name/ { gsub(/[" ]/, "", $2); print $2; exit }' "$cargo_toml_path")
 version=$(awk -F'=' '/^\[package\]/ { in_package = 1 } in_package && /version/ { gsub(/[" ]/, "", $2); print $2; exit }' "$cargo_toml_path")
 
-get_architecture() {
+get_platform_file_end() {
     machine=$(uname -m)
+    kernel=$(uname -s)
 
-    case $machine in
-        x86_64)
-            echo 'x86_64-linux-gnu'
+    case $kernel in
+        Darwin)
+            echo 'universal-apple-darwin.zip'
             ;;
-        armv7l)
-            echo 'armv7-linux-gnueabihf'
-            ;;
-        aarch64)
-            echo 'aarch64-linux-gnu'
+        Linux)
+            case $machine in
+                x86_64)
+                    echo 'x86_64-linux-gnu.tar.gz'
+                    ;;
+                armv7l)
+                    echo 'armv7-linux-gnueabihf.tar.gz'
+                    ;;
+                aarch64)
+                    echo 'aarch64-linux-gnu.tar.gz'
+                    ;;
+                *)
+                    echo "No self-compiled binary found and unsupported release-architecture: $machine" >&2
+                    exit 1
+                    ;;
+            esac
             ;;
         *)
-            echo "No self-compiled binary found and unsupported release-architecture: $machine" >&2
+            echo "No self-compiled binary found and unsupported OS: $kernel" >&2
             exit 1
             ;;
     esac
 }
-architecture=$(get_architecture)
+platform_file_end=$(get_platform_file_end)
 
-github_url="https://github.com/daywalker90/sling/releases/download/v$version/sling-v$version-$architecture.tar.gz"
+github_url="https://github.com/daywalker90/$name/releases/download/v$version/$name-v$version-$platform_file_end"
 
 
-# Download the file using curl
-if ! curl -L "$github_url" -o "$script_dir/sling-v$version-$architecture.tar.gz"; then
+# Download the archive using curl
+if ! curl -L "$github_url" -o "$script_dir/$name-v$version-$platform_file_end"; then
     echo "Error downloading the file from $github_url" >&2
     exit 1
 fi
 
-# Extract the contents using tar
-if ! tar -xzvf "$script_dir/sling-v$version-$architecture.tar.gz" -C "$script_dir"; then
-    echo "Error extracting the contents of sling-v$version-$architecture.tar.gz" >&2
+# Extract the contents
+if [[ $archive_file == *.tar.gz ]]; then
+    if ! tar -xzvf "$script_dir/$name-v$version-$platform_file_end" -C "$script_dir"; then
+        echo "Error extracting the contents of $name-v$version-$platform_file_end" >&2
+        exit 1
+    fi
+elif [[ $archive_file == *.zip ]]; then
+    if ! unzip "$script_dir/$name-v$version-$platform_file_end" -C "$script_dir"; then
+        echo "Error extracting the contents of $name-v$version-$platform_file_end" >&2
+        exit 1
+    fi
+else
+    echo "Unknown archive format or unsupported file extension: $archive_file" >&2
     exit 1
-fi
+fi