Tmp/slow tests and pact 5

.github/workflows/applications.yml

@@ -264,7 +264,7 @@ jobs:
         cabal --version
     - name: Install non-Haskell dependencies (ubuntu)
       if: contains(matrix.os, 'ubuntu')
-      run: sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libsnappy-dev libbz2-dev
+      run: sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libsnappy-dev libbz2-dev libmpfr-dev
     # Project Configuration
     - name: Create cabal.project.local
       run: |
@@ -358,7 +358,15 @@ jobs:
     - name: Build chainweb library
       run: cabal build --ghc-options=-j2 lib:chainweb
     - name: Build chainweb applications
-      run: cabal build -j2 --ghc-options=-j2 exe:compact test:chainweb-tests exe:cwtool chainweb:bench:bench
+      run: | 
+        cabal build -j2 --ghc-options=-j2 \
+          exe:compact \
+          test:chainweb-tests \
+          test:multi-node-network-tests \
+          test:compaction-tests \
+          test:remote-tests \
+          exe:cwtool \
+          chainweb:bench:bench
     - name: Build chainweb-node application
       run: cabal build -j2 --ghc-options=-j2 chainweb-node:exe:chainweb-node
 
@@ -384,6 +392,9 @@ jobs:
         cp $(cabal list-bin chainweb-node) artifacts/chainweb
         cp $(cabal list-bin compact) artifacts/chainweb
         cp $(cabal list-bin chainweb-tests) artifacts/chainweb
+        cp $(cabal list-bin multi-node-network-tests) artifacts/chainweb
+        cp $(cabal list-bin remote-tests) artifacts/chainweb
+        cp $(cabal list-bin compaction-tests) artifacts/chainweb
         cp $(cabal list-bin cwtool) artifacts/chainweb
         cp $(cabal list-bin bench) artifacts/chainweb
         cp README.md artifacts/chainweb
@@ -406,10 +417,10 @@ jobs:
         if-no-files-found: error
 
   # ########################################################################## #
-  # Run Slow tests (Multinode test)
+  # Run misc test suites
 
-  slow-tests:
-    name: Slow tests
+  remote-tests:
+    name: Remote tests
     needs: [config, build]
     runs-on: ${{ matrix.os }}
     env:
@@ -430,11 +441,65 @@ jobs:
         os: ${{ matrix.os }}
         ghc: ${{ matrix.ghc }}
         use-freeze-file: ${{ matrix.use-freeze-file }}
-    - name: Run slow tests
-      timeout-minutes: 20
+    - name: Run remote-tests
+      timeout-minutes: 2
+      run: remote-tests
+
+  compaction-tests:
+    name: Compaction tests
+    needs: [config, build]
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Download build artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.ARTIFACTS_NAME }}
+        path: .
+    - name: Install chainweb from artifacts
+      uses: ./.github/actions/install-chainweb-from-artifacts
+      with:
+        os: ${{ matrix.os }}
+        ghc: ${{ matrix.ghc }}
+        use-freeze-file: ${{ matrix.use-freeze-file }}
+    - name: Run compaction-tests
+      timeout-minutes: 7
+      run: |
+          ulimit -n 10000
+          compaction-tests
+
+  multi-node-network-tests:
+    name: Multi-node network tests
+    needs: [config, build]
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Download build artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ env.ARTIFACTS_NAME }}
+        path: .
+    - name: Install chainweb from artifacts
+      uses: ./.github/actions/install-chainweb-from-artifacts
+      with:
+        os: ${{ matrix.os }}
+        ghc: ${{ matrix.ghc }}
+        use-freeze-file: ${{ matrix.use-freeze-file }}
+    - name: Run multi-node-network-tests
+      timeout-minutes: 7
       run: |
           ulimit -n 10000
-          cwtool slow-tests
+          multi-node-network-tests
 
   # ########################################################################## #
   # Run Benchmarks
@@ -697,6 +762,7 @@ jobs:
           libbz2-1.0 \
           libgflags2.2 \
           zstd \
+          libmpfr6 \
           locales &&
           rm -rf /var/lib/apt/lists/* &&
           locale-gen en_US.UTF-8 &&

.github/workflows/macos.yaml

@@ -91,7 +91,7 @@ jobs:
         EOF
 
     # Build
-    - name: Delete Freeze file if it exists
+    - name: Delete freeze file if it exists
       run: rm -f cabal.project.freeze
     - name: Update package database
       run: cabal update
@@ -116,10 +116,17 @@ jobs:
     - name: Build chainweb library
       run: cabal build lib:chainweb
     - name: Build chainweb applications
-      run: cabal build test:chainweb-tests exe:cwtool chainweb:bench:bench
+      run: | 
+        cabal build -j \
+          exe:compact \
+          test:chainweb-tests \
+          test:multi-node-network-tests \
+          test:compaction-tests \
+          test:remote-tests \
+          exe:cwtool \
+          chainweb:bench:bench
     - name: Build chainweb-node application
-      run: cabal build chainweb-node:exe:chainweb-node
-
+      run: cabal build -j chainweb-node:exe:chainweb-node
     - uses: actions/cache/save@v4
       name: Save dist-newstyle cache
       if: steps.cabal-dist-cache.outputs.cache-hit != 'true'
@@ -130,10 +137,10 @@ jobs:
           dist-newstyle
         key: ${{ steps.cabal-dist-cache.outputs.cache-primary-key }}
 
-    - name: Run Tests
+    - name: Run unit tests
       run: |
         ulimit -n 10000
-        cabal run tests -- --hide-successes -p '!/chainweb216Test/'
+        $(cabal list-bin chainweb:test:chainweb-tests) --hide-successes -p '!/chainweb216Test/'
 
     # Checks
     - name: Check that working directory tree is clean
@@ -155,3 +162,4 @@ jobs:
           exit 1
         fi
         mv cabal.project.freeze.backup cabal.project.freeze
+

.github/workflows/release.yml

@@ -229,12 +229,9 @@ jobs:
 
             *   ubuntu-22.04:
                 ```sh
-                apt-get install ca-certificates libgmp10 libssl3 libsnappy1v5 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd
-                ```
-            *   ubuntu-20.04:
-                ```sh
-                apt-get install ca-certificates libgmp10 libssl1.1 libsnappy1v5 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd
+                apt install ca-certificates libgmp10 libssl3 libsnappy1v5 zlib1g liblz4-1 libbz2-1.0 libgflags2.2 zstd libmpfr6 locales
                 ```
+
           ' >> CHANGELOG.md
           cat CHANGELOG.md
 

bench/Chainweb/Pact/Backend/Bench.hs

@@ -53,13 +53,15 @@ import Chainweb.Graph
 import Chainweb.Logger
 import Chainweb.MerkleLogHash
 import Chainweb.Pact.Backend.RelationalCheckpointer
-import Chainweb.Pact.Backend.Types
+
 import Chainweb.Pact.Backend.Utils
 import Chainweb.Pact.Types
 import Chainweb.Test.TestVersions
 import Chainweb.Utils.Bench
 import Chainweb.Utils (sshow)
 import Chainweb.Version
+import qualified Chainweb.Pact4.Backend.ChainwebPactDb as Pact4
+import qualified Pact.Types.Command as Pact
 
 testVer :: ChainwebVersion
 testVer = instantCpmTestVersion petersonChainGraph
@@ -73,10 +75,10 @@ cpRestoreAndSave
   :: (Monoid q)
   => Checkpointer logger
   -> Maybe BlockHeader
-  -> [(BlockHeader, ChainwebPactDbEnv logger -> IO q)]
+  -> [(BlockHeader, PactDbEnv (Pact4.BlockEnv logger) -> IO q)]
   -> IO q
 cpRestoreAndSave cp pc blks = snd <$> _cpRestoreAndSave cp (ParentHeader <$> pc)
-  (traverse Stream.yield [RunnableBlock $ \dbEnv _ -> (,bh) <$> fun (_cpPactDbEnv dbEnv) | (bh, fun) <- blks])
+  (traverse Stream.yield [Pact4RunnableBlock $ \dbEnv _ -> (,bh) <$> fun (Pact4._cpPactDbEnv dbEnv) | (bh, fun) <- blks])
 
 -- | fabricate a `BlockHeader` for a block given its hash and its parent.
 childOf :: Maybe BlockHeader -> BlockHash -> BlockHeader
@@ -442,5 +444,5 @@ cpBenchLookupProcessedTx transactionCount cp = C.env setup' $ \ ~(ut) ->
     pc02 = childOf (Just pc01) hash02
 
     go (NoopNFData _) = do
-        _cpReadFrom (_cpReadCp cp) (Just (ParentHeader pc02)) $ \dbEnv ->
-          _cpLookupProcessedTx dbEnv (V.fromList [Pact.TypedHash "" | _ <- [1..transactionCount]])
+        _cpReadFrom (_cpReadCp cp) (Just (ParentHeader pc02)) Pact4T $ \dbEnv _ ->
+          Pact4._cpLookupProcessedTx dbEnv (V.fromList [Pact.RequestKey (Pact.toUntypedHash $ Pact.TypedHash "") | _ <- [1..transactionCount]])

bench/Chainweb/Pact/Backend/ForkingBench.hs

@@ -15,6 +15,8 @@
 {-# LANGUAGE TupleSections #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE TypeFamilies #-}
+{-# LANGUAGE PartialTypeSignatures #-}
+{-# OPTIONS_GHC -Wno-partial-type-signatures #-}
 
 module Chainweb.Pact.Backend.ForkingBench ( bench ) where
 
@@ -31,6 +33,7 @@ import Data.Aeson hiding (Error)
 import Data.ByteString (ByteString)
 import Data.Char
 import Data.Decimal
+import Data.Either
 import Data.FileEmbed
 import Data.Foldable (toList)
 import Data.IORef
@@ -81,23 +84,22 @@ import Chainweb.BlockHeight (BlockHeight(..))
 import Chainweb.ChainId
 import Chainweb.Graph
 import Chainweb.Logger
-import Chainweb.Mempool.Mempool (BlockFill(..))
+import Chainweb.Mempool.Mempool
 import Chainweb.Miner.Pact
 import Chainweb.Pact.Backend.Compaction qualified as C
-import Chainweb.Pact.Backend.Types
+
 import Chainweb.Pact.Backend.Utils
 import Chainweb.Pact.PactService
 import Chainweb.Pact.Service.BlockValidation
 import Chainweb.Pact.Service.PactQueue
-import Chainweb.Pact.Service.Types
 import Chainweb.Pact.Types
 import Chainweb.Pact.Utils (toTxCreationTime)
 import Chainweb.Payload
 import Chainweb.Payload.PayloadStore
 import Chainweb.Payload.PayloadStore.InMemory
 import Chainweb.Test.TestVersions (slowForkingCpmTestVersion)
 import Chainweb.Time
-import Chainweb.Transaction
+import qualified Chainweb.Pact4.Transaction as Pact4
 import Chainweb.Utils
 import Chainweb.Utils.Bench
 import Chainweb.Version
@@ -242,7 +244,7 @@ createBlock validate parent nonce pact = do
      -- assemble block without nonce and timestamp
 
      bip <- throwIfNoHistory =<< newBlock noMiner NewBlockFill parent pact
-     let payload = blockInProgressToPayloadWithOutputs bip
+     let payload = forAnyPactVersion finalizeBlock bip
 
      let creationTime = add second $ view blockCreationTime $ _parentHeader parent
      let bh = newBlockHeader
@@ -383,15 +385,16 @@ withResources rdb trunkLength logLevel compact p f = C.envWithCleanup create des
 testMemPoolAccess :: IORef Int -> MVar (Map Account (NonEmpty (DynKeyPair, [SigCapability]))) -> IO MemPoolAccess
 testMemPoolAccess txsPerBlock accounts = do
   return $ mempty
-    { mpaGetBlock = \bf validate bh hash header -> do
+    { mpaGetBlock = \bf validate bh hash bct -> do
         if _bfCount bf /= 0 then pure mempty else do
-          testBlock <- getTestBlock accounts (_bct $ view blockCreationTime header) validate bh hash
+          testBlock <- getTestBlock accounts (_bct bct) validate bh hash
           pure testBlock
     }
   where
 
     setTime time pb = pb { _pmCreationTime = toTxCreationTime time }
 
+    getTestBlock :: _ -> _ -> MempoolPreBlockCheck Pact4.UnparsedTransaction to -> _ -> _ -> IO (V.Vector to)
     getTestBlock mVarAccounts txOrigTime validate bHeight hash
         | bHeight == 1 = do
             meta <- setTime txOrigTime <$> makeMeta cid
@@ -402,10 +405,10 @@ testMemPoolAccess txsPerBlock accounts = do
                   modifyMVar' mVarAccounts
                     (const $ M.fromList $ zip as kss)
 
-                  vs <- validate bHeight hash (V.fromList $ toList r)
+                  vs <- validate bHeight hash (V.fromList $ toList $ Pact4.unparseTransaction <$> r)
                   -- TODO: something better should go here
-                  unless (and vs) $ throwM $ userError $ "at blockheight 1: tx validation failed " <> sshow vs
-                  return $! V.fromList $ toList r
+                  unless (all isRight vs) $ throwM $ userError $ "at blockheight 1: tx validation failed " <> sshow r
+                  return $! V.fromList [v | Right v <- toList vs]
 
         | otherwise = do
           withMVar mVarAccounts $ \accs -> do
@@ -419,7 +422,9 @@ testMemPoolAccess txsPerBlock accounts = do
                 case eCmd of
                   Left e -> throwM $ userError e
                   Right tx -> return tx
-            return $! txs
+            vs <- validate bHeight hash (V.fromList $ toList $ Pact4.unparseTransaction <$> txs)
+            unless (all isRight vs) $ throwM $ userError $ "tx validation failed " <> sshow txs
+            return $! V.fromList [v | Right v <- toList vs]
 
     mkTransferCaps :: ReceiverName -> Amount -> (Account, NonEmpty (DynKeyPair, [SigCapability])) -> (Account, NonEmpty (DynKeyPair, [SigCapability]))
     mkTransferCaps (ReceiverName (Account r)) (Amount m) (s@(Account ss),ks) = (s, (caps <$) <$> ks)
@@ -509,10 +514,10 @@ safeCapitalize :: String -> String
 safeCapitalize = maybe [] (uncurry (:) . bimap toUpper (Prelude.map toLower)) . Data.List.uncons
 
 
--- TODO: Use the new `assertCommand` function.
-validateCommand :: Command Text -> Either String ChainwebTransaction
+-- TODO: Use the new `assertPact4Command` function.
+validateCommand :: Command Text -> Either String Pact4.Transaction
 validateCommand cmdText = case verifyCommand cmdBS of
-    ProcSucc cmd -> Right (mkPayloadWithTextOld <$> cmd)
+    ProcSucc cmd -> Right (Pact4.mkPayloadWithTextOld <$> cmd)
     ProcFail err -> Left err
   where
     cmdBS :: Command ByteString