Merge branch 'master' of github.com:AcalaNetwork/boka

* 'master' of github.com:AcalaNetwork/boka: add pvm instructions 5.10, 5.11, 5.12 (#84) recent history tests are fixed now (#88) support offenders in safrole (#87) Recent history test (#85)
open-web3-stack · Aug 29, 2024 · d2ebd09 · d2ebd09
2 parents 76984e2 + 624316d
commit d2ebd09
Show file tree

Hide file tree

Showing 19 changed files with 876 additions and 106 deletions.
diff --git a/Blockchain/Package.swift b/Blockchain/Package.swift
@@ -19,6 +19,7 @@ let package = Package(
         .package(path: "../Codec"),
         .package(path: "../Utils"),
         .package(path: "../TracingUtils"),
+        .package(path: "../PolkaVM"),
         .package(url: "https://github.com/apple/swift-testing.git", branch: "0.10.0"),
     ],
     targets: [
@@ -29,6 +30,7 @@ let package = Package(
             dependencies: [
                 "Codec",
                 "Utils",
+                "PolkaVM",
                 "TracingUtils",
             ]
         ),

diff --git a/Blockchain/Sources/Blockchain/Config/ProtocolConfig+Preset.swift b/Blockchain/Sources/Blockchain/Config/ProtocolConfig+Preset.swift
@@ -33,7 +33,7 @@ extension Ref where T == ProtocolConfig {
         maxEncodedWorkReportSize: 96 * 1 << 10,
         erasureCodedSegmentSize: 6,
         ticketSubmissionEndSlot: 500,
-        pvmDynamicAddressAlignmentFactor: 4,
+        pvmDynamicAddressAlignmentFactor: 2,
         pvmProgramInitInputDataSize: 1 << 24,
         pvmProgramInitPageSize: 1 << 14,
         pvmProgramInitSegmentSize: 1 << 16
@@ -70,7 +70,7 @@ extension Ref where T == ProtocolConfig {
         maxEncodedWorkReportSize: 96 * 1 << 10,
         erasureCodedSegmentSize: 6,
         ticketSubmissionEndSlot: 500,
-        pvmDynamicAddressAlignmentFactor: 4,
+        pvmDynamicAddressAlignmentFactor: 2,
         pvmProgramInitInputDataSize: 1 << 24,
         pvmProgramInitPageSize: 1 << 14,
         pvmProgramInitSegmentSize: 1 << 16

diff --git a/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift b/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift
@@ -1,3 +1,4 @@
+import PolkaVM
 import Utils
 
 // constants defined in the graypaper
@@ -94,7 +95,7 @@ public struct ProtocolConfig: Sendable {
     // Y = 500: The number of slots into an epoch at which ticket-submission ends.
     public var ticketSubmissionEndSlot: Int
 
-    // ZA = 4: The pvm dynamic address alignment factor.
+    // ZA = 2: The pvm dynamic address alignment factor.
     public var pvmDynamicAddressAlignmentFactor: Int
 
     // ZI = 2^24: The standard pvm program initialization input data size.
@@ -181,6 +182,8 @@ public struct ProtocolConfig: Sendable {
 
 public typealias ProtocolConfigRef = Ref<ProtocolConfig>
 
+extension ProtocolConfig: PvmConfig {}
+
 extension ProtocolConfig {
     public enum AuditTranchePeriod: ReadInt {
         public typealias TConfig = ProtocolConfigRef

diff --git a/Blockchain/Sources/Blockchain/Runtime.swift b/Blockchain/Sources/Blockchain/Runtime.swift
@@ -73,20 +73,9 @@ public final class Runtime {
         var newState = prevState.value
 
         do {
-            newState.recentHistory = try updateRecentHistory(block: block, state: prevState)
+            try updateRecentHistory(block: block, state: &newState)
 
-            let safroleResult = try newState.updateSafrole(
-                config: config, slot: block.header.timeslot, entropy: newState.entropyPool.t0, extrinsics: block.extrinsic.tickets
-            )
-            newState.mergeWith(postState: safroleResult.state)
-
-            guard safroleResult.epochMark == block.header.epoch else {
-                throw Error.invalidHeaderEpochMarker
-            }
-
-            guard safroleResult.ticketsMark == block.header.winningTickets else {
-                throw Error.invalidHeaderWinningTickets
-            }
+            try updateSafrole(block: block, state: &newState)
 
             newState.coreAuthorizationPool = try updateAuthorizationPool(
                 block: block, state: prevState
@@ -106,30 +95,33 @@ public final class Runtime {
         return StateRef(newState)
     }
 
-    public func updateRecentHistory(block: BlockRef, state: StateRef) throws -> RecentHistory {
-        var history = state.value.recentHistory
-        if history.items.count >= 0 { // if this is not block #0
-            // write the state root of last block
-            history.items[history.items.endIndex - 1].stateRoot = state.stateRoot
-        }
-
-        let workReportHashes = block.extrinsic.reports.guarantees.map(\.workReport.packageSpecification.workPackageHash)
+    public func updateSafrole(block: BlockRef, state newState: inout State) throws {
+        let safroleResult = try newState.updateSafrole(
+            config: config,
+            slot: block.header.timeslot,
+            entropy: newState.entropyPool.t0,
+            offenders: newState.judgements.punishSet,
+            extrinsics: block.extrinsic.tickets
+        )
+        newState.mergeWith(postState: safroleResult.state)
 
-        let accumulationResult = Data32() // TODO: calculate accumulation result
+        guard safroleResult.epochMark == block.header.epoch else {
+            throw Error.invalidHeaderEpochMarker
+        }
 
-        var mmr = history.items.last?.mmr ?? .init([])
-        mmr.append(accumulationResult, hasher: Keccak.self)
+        guard safroleResult.ticketsMark == block.header.winningTickets else {
+            throw Error.invalidHeaderWinningTickets
+        }
+    }
 
-        let newItem = try RecentHistory.HistoryItem(
+    public func updateRecentHistory(block: BlockRef, state newState: inout State) throws {
+        let workReportHashes = block.extrinsic.reports.guarantees.map(\.workReport.packageSpecification.workPackageHash)
+        try newState.recentHistory.update(
             headerHash: block.header.parentHash,
-            mmr: mmr,
-            stateRoot: Data32(), // empty and will be updated upon next block
+            parentStateRoot: block.header.priorStateRoot,
+            accumulateRoot: Data32(), // TODO: calculate accumulation result
             workReportHashes: ConfigLimitedSizeArray(config: config, array: workReportHashes)
         )
-
-        history.items.safeAppend(newItem)
-
-        return history
     }
 
     // TODO: add tests

diff --git a/Blockchain/Sources/Blockchain/Safrole.swift b/Blockchain/Sources/Blockchain/Safrole.swift
@@ -151,7 +151,13 @@ public protocol Safrole {
     > { get }
     var ticketsVerifier: BandersnatchRingVRFRoot { get }
 
-    func updateSafrole(config: ProtocolConfigRef, slot: TimeslotIndex, entropy: Data32, extrinsics: ExtrinsicTickets) throws(SafroleError)
+    func updateSafrole(
+        config: ProtocolConfigRef,
+        slot: TimeslotIndex,
+        entropy: Data32,
+        offenders: Set<Ed25519PublicKey>,
+        extrinsics: ExtrinsicTickets
+    ) throws(SafroleError)
         -> (
             state: SafrolePostState,
             epochMark: EpochMarker?,
@@ -207,9 +213,30 @@ func pickFallbackValidators(
     return indices.map { validators[$0].bandersnatch }
 }
 
+func withoutOffenders(
+    offenders: Set<Ed25519PublicKey>,
+    validators: ConfigFixedSizeArray<ValidatorKey, ProtocolConfig.TotalNumberOfValidators>
+) -> ConfigFixedSizeArray<ValidatorKey, ProtocolConfig.TotalNumberOfValidators> {
+    var validators = validators
+
+    for i in validators.indices {
+        let validator = validators[i]
+        if offenders.contains(validator.ed25519) {
+            validators[i] = ValidatorKey() // replace to empty key
+        }
+    }
+
+    return validators
+}
+
 extension Safrole {
-    public func updateSafrole(config: ProtocolConfigRef, slot: TimeslotIndex, entropy: Data32,
-                              extrinsics: ExtrinsicTickets) throws(SafroleError)
+    public func updateSafrole(
+        config: ProtocolConfigRef,
+        slot: TimeslotIndex,
+        entropy: Data32,
+        offenders: Set<Ed25519PublicKey>,
+        extrinsics: ExtrinsicTickets
+    ) throws(SafroleError)
         -> (
             state: SafrolePostState,
             epochMark: EpochMarker?,
@@ -250,7 +277,7 @@ extension Safrole {
 
             let (newNextValidators, newCurrentValidators, newPreviousValidators, newTicketsVerifier) = isEpochChange
                 ? (
-                    validatorQueue, // TODO: Φ filter out the one in the punishment set
+                    withoutOffenders(offenders: offenders, validators: validatorQueue),
                     nextValidators,
                     currentValidators,
                     newVerifier.ringRoot

diff --git a/Blockchain/Sources/Blockchain/Types/RecentHistory.swift b/Blockchain/Sources/Blockchain/Types/RecentHistory.swift
@@ -37,3 +37,29 @@ extension RecentHistory: Dummy {
         RecentHistory(items: try! ConfigLimitedSizeArray(config: config))
     }
 }
+
+extension RecentHistory {
+    public mutating func update(
+        headerHash: Data32,
+        parentStateRoot: Data32,
+        accumulateRoot: Data32,
+        workReportHashes: ConfigLimitedSizeArray<Data32, ProtocolConfig.Int0, ProtocolConfig.TotalNumberOfCores>
+    ) {
+        if items.count > 0 { // if this is not block #0
+            // write the state root of last block
+            items[items.endIndex - 1].stateRoot = parentStateRoot
+        }
+
+        var mmr = items.last?.mmr ?? .init([])
+        mmr.append(accumulateRoot, hasher: Keccak.self)
+
+        let newItem = RecentHistory.HistoryItem(
+            headerHash: headerHash,
+            mmr: mmr,
+            stateRoot: Data32(), // empty and will be updated upon next block
+            workReportHashes: workReportHashes
+        )
+
+        items.safeAppend(newItem)
+    }
+}
diff --git a/Blockchain/Sources/Blockchain/Types/ValidatorKey.swift b/Blockchain/Sources/Blockchain/Types/ValidatorKey.swift
@@ -17,16 +17,18 @@ public struct ValidatorKey: Sendable, Equatable, Codable {
         self.bls = bls
         self.metadata = metadata
     }
+
+    public init() {
+        bandersnatch = BandersnatchPublicKey()
+        ed25519 = Ed25519PublicKey()
+        bls = BLSKey()
+        metadata = Data128()
+    }
 }
 
 extension ValidatorKey: Dummy {
     public typealias Config = ProtocolConfigRef
     public static func dummy(config _: Config) -> ValidatorKey {
-        ValidatorKey(
-            bandersnatch: BandersnatchPublicKey(),
-            ed25519: Ed25519PublicKey(),
-            bls: BLSKey(),
-            metadata: Data128()
-        )
+        ValidatorKey()
     }
 }
diff --git a/Codec/Sources/Codec/JamDecoder.swift b/Codec/Sources/Codec/JamDecoder.swift
@@ -324,8 +324,11 @@ private struct JamUnkeyedDecodingContainer: UnkeyedDecodingContainer {
     let decoder: DecodeContext
 
     mutating func decodeNil() throws -> Bool {
+        guard let byte = decoder.data.next() else {
+            throw DecodingError.dataCorruptedError(in: self, debugDescription: "Unexpected end of data")
+        }
         currentIndex += 1
-        return decoder.data.count > 0
+        return byte == 0
     }
 
     mutating func decode(_: Bool.Type) throws -> Bool {
@@ -439,7 +442,7 @@ private struct JamSingleValueDecodingContainer: SingleValueDecodingContainer {
     let decoder: DecodeContext
 
     func decodeNil() -> Bool {
-        decoder.data.count > 0
+        decoder.data.next() == 0
     }
 
     func decode(_: Bool.Type) throws -> Bool {

diff --git a/JAMTests/Sources/JAMTests/TestLoader.swift b/JAMTests/Sources/JAMTests/TestLoader.swift
@@ -3,12 +3,20 @@ import Foundation
 // somehow without this the GH Actions CI fails
 extension Foundation.Bundle: @unchecked @retroactive Sendable {}
 
+struct Testcase: CustomStringConvertible {
+    var description: String
+    var data: Data
+}
+
 enum TestLoader {
-    static func getTestFiles(path: String, extension ext: String) throws -> [(path: String, description: String)] {
+    static func getTestcases(path: String, extension ext: String) throws -> [Testcase] {
         let prefix = Bundle.module.resourcePath! + "/jamtestvectors/\(path)"
         let files = try FileManager.default.contentsOfDirectory(atPath: prefix)
         var filtered = files.filter { $0.hasSuffix(".\(ext)") }
         filtered.sort()
-        return filtered.map { (path: prefix + "/" + $0, description: $0) }
+        return try filtered.map {
+            let data = try Data(contentsOf: URL(fileURLWithPath: prefix + "/" + $0))
+            return Testcase(description: $0, data: data)
+        }
     }
 }
diff --git a/JAMTests/Tests/JAMTests/PVMTests.swift b/JAMTests/Tests/JAMTests/PVMTests.swift
@@ -62,17 +62,14 @@ struct PolkaVMTestcase: Codable, CustomStringConvertible {
 }
 
 struct PVMTests {
-    static func loadTests() throws -> [PolkaVMTestcase] {
-        let tests = try TestLoader.getTestFiles(path: "pvm/programs", extension: "json")
-        return try tests.map {
-            let data = try Data(contentsOf: URL(fileURLWithPath: $0.path))
-            let decoder = JSONDecoder()
-            return try decoder.decode(PolkaVMTestcase.self, from: data)
-        }
+    static func loadTests() throws -> [Testcase] {
+        try TestLoader.getTestcases(path: "pvm/programs", extension: "json")
     }
 
     @Test(arguments: try loadTests())
-    func testPVM(testCase: PolkaVMTestcase) throws {
+    func testPVM(testCase: Testcase) throws {
+        let decoder = JSONDecoder()
+        let testCase = try decoder.decode(PolkaVMTestcase.self, from: testCase.data)
         let program = try ProgramCode(Data(testCase.program))
         let memory = Memory(
             pageMap: testCase.initialPageMap.map { (address: $0.address, length: $0.length, writable: $0.isWritable) },

diff --git a/JAMTests/Tests/JAMTests/RecentHistoryTests.swift b/JAMTests/Tests/JAMTests/RecentHistoryTests.swift
@@ -0,0 +1,42 @@
+import Blockchain
+import Codec
+import Foundation
+import Testing
+import Utils
+
+@testable import JAMTests
+
+struct RecentHistoryInput: Codable {
+    var headerHash: Data32
+    var parentStateRoot: Data32
+    var accumulateRoot: Data32
+    var workPackages: [Data32]
+}
+
+struct RecentHisoryTestcase: Codable {
+    var input: RecentHistoryInput
+    var preState: RecentHistory
+    var postState: RecentHistory
+}
+
+struct RecentHistoryTests {
+    static func loadTests() throws -> [Testcase] {
+        try TestLoader.getTestcases(path: "history/data", extension: "scale")
+    }
+
+    @Test(arguments: try loadTests())
+    func recentHistory(_ testcase: Testcase) throws {
+        let config = ProtocolConfigRef.mainnet
+        let testcase = try JamDecoder.decode(RecentHisoryTestcase.self, from: testcase.data, withConfig: config)
+
+        var state = testcase.preState
+        try state.update(
+            headerHash: testcase.input.headerHash,
+            parentStateRoot: testcase.input.parentStateRoot,
+            accumulateRoot: testcase.input.accumulateRoot,
+            workReportHashes: ConfigLimitedSizeArray(config: config, array: testcase.input.workPackages)
+        )
+
+        #expect(state == testcase.postState)
+    }
+}
diff --git a/JAMTests/Tests/JAMTests/SafroleTests.swift b/JAMTests/Tests/JAMTests/SafroleTests.swift
@@ -9,6 +9,7 @@ import Utils
 struct SafroleInput: Codable {
     var slot: UInt32
     var entropy: Data32
+    var offenders: [Ed25519PublicKey]
     var extrinsics: ExtrinsicTickets
 }
 
@@ -93,11 +94,6 @@ struct SafroleTestcase: Codable {
     var postState: SafroleState
 }
 
-struct Testcase: CustomStringConvertible {
-    var description: String
-    var data: Data
-}
-
 enum SafroleTestVariants: String, CaseIterable {
     case tiny
     case full
@@ -121,13 +117,9 @@ enum SafroleTestVariants: String, CaseIterable {
     }
 }
 
-final class SafroleTests {
+struct SafroleTests {
     static func loadTests(variant: SafroleTestVariants) throws -> [Testcase] {
-        let tests = try TestLoader.getTestFiles(path: "safrole/\(variant)", extension: "scale")
-        return try tests.map { path, description in
-            let data = try Data(contentsOf: URL(fileURLWithPath: path))
-            return Testcase(description: description, data: data)
-        }
+        try TestLoader.getTestcases(path: "safrole/\(variant)", extension: "scale")
     }
 
     func safroleTests(_ input: Testcase, variant: SafroleTestVariants) throws {
@@ -139,6 +131,7 @@ final class SafroleTests {
                 config: config,
                 slot: testcase.input.slot,
                 entropy: testcase.input.entropy,
+                offenders: Set(testcase.input.offenders),
                 extrinsics: testcase.input.extrinsics
             )
         }