From 993ce1333fd92dd446b3db8d81fcc90cc3281457 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Mon, 9 Sep 2024 10:39:31 +0800 Subject: [PATCH 1/2] standard program initialization (#101) * wip * rename * init registers * refactor * check total size * standard program memory init * fix * fix memory * pass some pvm tests * fix --- Codec/Sources/Codec/IntegerCodec.swift | 10 +-- JAMTests/Tests/JAMTests/PVMTests.swift | 3 - PolkaVM/Sources/PolkaVM/Instruction.swift | 2 + PolkaVM/Sources/PolkaVM/Memory.swift | 87 ++++++++++++++++--- PolkaVM/Sources/PolkaVM/PvmConfig.swift | 8 ++ PolkaVM/Sources/PolkaVM/Registers.swift | 10 +++ PolkaVM/Sources/PolkaVM/StandardProgram.swift | 80 +++++++++++++++++ PolkaVM/Sources/PolkaVM/VMState.swift | 10 +++ 8 files changed, 189 insertions(+), 21 deletions(-) create mode 100644 PolkaVM/Sources/PolkaVM/StandardProgram.swift diff --git a/Codec/Sources/Codec/IntegerCodec.swift b/Codec/Sources/Codec/IntegerCodec.swift index d2885fd2..73f3b80e 100644 --- a/Codec/Sources/Codec/IntegerCodec.swift +++ b/Codec/Sources/Codec/IntegerCodec.swift @@ -42,18 +42,18 @@ public enum IntegerCodec { return 0 } - let byteLengh = (~firstByte).leadingZeroBitCount + let byteLength = (~firstByte).leadingZeroBitCount var res: UInt64 = 0 - if byteLengh > 0 { - guard let rest: UInt64 = try decode(length: byteLengh, next: next) else { + if byteLength > 0 { + guard let rest: UInt64 = try decode(length: byteLength, next: next) else { return nil } res = rest } - let mask = UInt8(UInt(1) << (8 - byteLengh) - 1) + let mask = UInt8(UInt(1) << (8 - byteLength) - 1) let topBits = firstByte & mask - return res + UInt64(topBits) << (8 * byteLengh) + return res + UInt64(topBits) << (8 * byteLength) } } diff --git a/JAMTests/Tests/JAMTests/PVMTests.swift b/JAMTests/Tests/JAMTests/PVMTests.swift index d6a2c032..95147ff3 100644 --- a/JAMTests/Tests/JAMTests/PVMTests.swift +++ b/JAMTests/Tests/JAMTests/PVMTests.swift @@ -66,9 +66,6 @@ private let logger = Logger(label: "PVMTests") // TODO: pass these let knownFailedTestCases = [ - "inst_load_u8_trap", - "inst_store_u8_trap_read_only", - "inst_store_u8_trap_inaccessible", "inst_ret_halt", "inst_ret_invalid", ] diff --git a/PolkaVM/Sources/PolkaVM/Instruction.swift b/PolkaVM/Sources/PolkaVM/Instruction.swift index e32c7a21..3761f88c 100644 --- a/PolkaVM/Sources/PolkaVM/Instruction.swift +++ b/PolkaVM/Sources/PolkaVM/Instruction.swift @@ -37,6 +37,8 @@ extension Instruction { logger.debug("execution success! updating pc...") return updatePC(context: context, skip: skip) } catch let e as Memory.Error { + // this passes test vector + context.state.consumeGas(gasCost()) return .exit(.pageFault(e.address)) } catch let e { // other unknown errors diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index 1871ef31..d9056e0d 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -20,10 +20,67 @@ public class Memory { private let pageMap: [(address: UInt32, length: UInt32, writable: Bool)] private var chunks: [(address: UInt32, data: Data)] + private let heapStart: UInt32 + private var heapEnd: UInt32 // start idx of unallocated heap + private let heapLimit: UInt32 public init(pageMap: [(address: UInt32, length: UInt32, writable: Bool)], chunks: [(address: UInt32, data: Data)]) { self.pageMap = pageMap self.chunks = chunks + heapStart = pageMap.first(where: { $0.writable })?.address ?? 0 + heapLimit = UInt32.max + heapEnd = chunks.reduce(0) { max($0, $1.address + UInt32($1.data.count)) } + } + + /// Standard Program init + /// + /// Init with some read only data, writable data and argument data + public init(readOnlyData: Data, readWriteData: Data, argumentData: Data, heapEmptyPagesSize: UInt32, stackSize: UInt32) { + let config = DefaultPvmConfig() + let P = StandardProgram.alignToPageSize + let Q = StandardProgram.alignToSegmentSize + let ZQ = UInt32(config.pvmProgramInitSegmentSize) + let readOnlyLen = UInt32(readOnlyData.count) + let readWriteLen = UInt32(readWriteData.count) + let argumentDataLen = UInt32(argumentData.count) + + let heapStart = 2 * ZQ + Q(readOnlyLen, config) + + pageMap = [ + (ZQ, readOnlyLen, false), + (ZQ + readOnlyLen, P(readOnlyLen, config) - readOnlyLen, false), + (heapStart, readWriteLen, true), // heap + (heapStart + readWriteLen, P(readWriteLen, config) + heapEmptyPagesSize - readWriteLen, true), // heap + (UInt32(config.pvmProgramInitStackBaseAddress) - P(stackSize, config), stackSize, true), // stack + (UInt32(config.pvmProgramInitInputStartAddress), argumentDataLen, false), // argument + (UInt32(config.pvmProgramInitInputStartAddress) + argumentDataLen, P(argumentDataLen, config) - argumentDataLen, false), + ] + + chunks = [ + (ZQ, readOnlyData), + (heapStart, readWriteData), + (UInt32(config.pvmProgramInitInputStartAddress), argumentData), + ] + + self.heapStart = heapStart + heapLimit = heapStart + P(readWriteLen, config) + heapEmptyPagesSize + heapEnd = heapStart + readWriteLen + } + + public func isWritable(address: UInt32) -> Bool { + // check heap range + guard heapStart <= address, address < heapLimit else { + return false + } + + // TODO: optimize + for page in pageMap { + if page.address <= address, address < page.address + page.length { + return page.writable + } + } + + return false } public func read(address: UInt32) throws(Error) -> UInt8 { @@ -72,6 +129,10 @@ public class Memory { } public func write(address: UInt32, value: UInt8) throws(Error) { + guard isWritable(address: address) else { + throw Error.notWritable(address) + } + // TODO: optimize this // check for chunks for i in 0 ..< chunks.count { @@ -88,6 +149,7 @@ public class Memory { var newChunk = (address: address, data: Data(repeating: 0, count: Int(page.length))) newChunk.data[Int(address - page.address)] = value chunks.append(newChunk) + heapEnd = max(heapEnd, address + 1) return } } @@ -95,6 +157,10 @@ public class Memory { } public func write(address: UInt32, values: some Sequence) throws(Error) { + guard isWritable(address: address) else { + throw Error.notWritable(address) + } + // TODO: optimize this // check for chunks for i in 0 ..< chunks.count { @@ -120,13 +186,14 @@ public class Memory { var idx = Int(address - page.address) for v in values { if idx == newChunk.data.endIndex { - newChunk.data.append(v) + throw Error.notWritable(address) } else { newChunk.data[idx] = v } idx += 1 } chunks.append(newChunk) + heapEnd = max(heapEnd, UInt32(idx)) return } } @@ -134,24 +201,18 @@ public class Memory { } public func sbrk(_ increment: UInt32) throws -> UInt32 { - // TODO: update after memory layout details are implemented - var curHeapEnd: UInt32 = 0 - - if let lastChunk = chunks.last { - curHeapEnd = lastChunk.address + UInt32(lastChunk.data.count) - } - + // TODO: optimize for page in pageMap { let pageEnd = page.address + page.length - if page.writable, curHeapEnd >= page.address, curHeapEnd + increment < pageEnd { - let newChunk = (address: curHeapEnd, data: Data(repeating: 0, count: Int(increment))) + if page.writable, heapEnd >= page.address, heapEnd + increment < pageEnd { + let newChunk = (address: heapEnd, data: Data(repeating: 0, count: Int(increment))) chunks.append(newChunk) - - return curHeapEnd + heapEnd += increment + return heapEnd } } - throw Error.outOfMemory(curHeapEnd) + throw Error.outOfMemory(heapEnd) } } diff --git a/PolkaVM/Sources/PolkaVM/PvmConfig.swift b/PolkaVM/Sources/PolkaVM/PvmConfig.swift index 2c8e120c..7480ec72 100644 --- a/PolkaVM/Sources/PolkaVM/PvmConfig.swift +++ b/PolkaVM/Sources/PolkaVM/PvmConfig.swift @@ -18,10 +18,18 @@ public struct DefaultPvmConfig: PvmConfig { public let pvmProgramInitPageSize: Int public let pvmProgramInitSegmentSize: Int + public let pvmProgramInitRegister1Value: Int + public let pvmProgramInitStackBaseAddress: Int + public let pvmProgramInitInputStartAddress: Int + public init() { pvmDynamicAddressAlignmentFactor = 2 pvmProgramInitInputDataSize = 1 << 24 pvmProgramInitPageSize = 1 << 14 pvmProgramInitSegmentSize = 1 << 16 + + pvmProgramInitRegister1Value = (1 << 32) - (1 << 16) + pvmProgramInitStackBaseAddress = (1 << 32) - (2 * pvmProgramInitSegmentSize) - pvmProgramInitInputDataSize + pvmProgramInitInputStartAddress = pvmProgramInitStackBaseAddress + pvmProgramInitSegmentSize } } diff --git a/PolkaVM/Sources/PolkaVM/Registers.swift b/PolkaVM/Sources/PolkaVM/Registers.swift index 91180624..b290a978 100644 --- a/PolkaVM/Sources/PolkaVM/Registers.swift +++ b/PolkaVM/Sources/PolkaVM/Registers.swift @@ -1,3 +1,5 @@ +import Foundation + public struct Registers: Equatable { public struct Index { public let value: UInt8 @@ -47,6 +49,14 @@ public struct Registers: Equatable { reg13 = values[12] } + /// standard program init registers + public init(config: DefaultPvmConfig, argumentData: Data?) { + reg1 = UInt32(config.pvmProgramInitRegister1Value) + reg2 = UInt32(config.pvmProgramInitStackBaseAddress) + reg10 = UInt32(config.pvmProgramInitInputStartAddress) + reg11 = UInt32(argumentData?.count ?? 0) + } + public subscript(index: Index) -> UInt32 { get { switch index.value { diff --git a/PolkaVM/Sources/PolkaVM/StandardProgram.swift b/PolkaVM/Sources/PolkaVM/StandardProgram.swift new file mode 100644 index 00000000..004e2395 --- /dev/null +++ b/PolkaVM/Sources/PolkaVM/StandardProgram.swift @@ -0,0 +1,80 @@ +import Foundation + +/// Standard Program defined in GP. +/// +/// It includes some metadata for memory and registers initialization +/// other than the program code +public class StandardProgram { + public enum Error: Swift.Error { + case invalidReadOnlyLength + case invalidReadWriteLength + case invalidHeapPages + case invalidStackSize + case invalidDataLength + case invalidCodeLength + case invalidTotalMemorySize + } + + public let code: ProgramCode + public let initialMemory: Memory + public let initialRegisters: Registers + + public init(blob: Data, argumentData: Data?) throws { + var slice = Slice(base: blob, bounds: blob.startIndex ..< blob.endIndex) + guard let readOnlyLen: UInt32 = slice.decode(length: 3) else { throw Error.invalidReadOnlyLength } + guard let readWriteLen: UInt32 = slice.decode(length: 3) else { throw Error.invalidReadWriteLength } + guard let heapPages: UInt16 = slice.decode(length: 2) else { throw Error.invalidHeapPages } + guard let stackSize: UInt32 = slice.decode(length: 3) else { throw Error.invalidStackSize } + + let readOnlyEndIdx = slice.startIndex + Int(readOnlyLen) + guard readOnlyEndIdx <= slice.endIndex else { throw Error.invalidDataLength } + let readOnlyData = blob[slice.startIndex ..< readOnlyEndIdx] + + let readWriteEndIdx = readOnlyEndIdx + Int(readWriteLen) + guard readWriteEndIdx <= slice.endIndex else { throw Error.invalidDataLength } + let readWriteData = blob[readOnlyEndIdx ..< readWriteEndIdx] + + slice = slice.dropFirst(Int(readOnlyLen + readWriteLen)) + guard let codeLength: UInt32 = slice.decode(length: 4), slice.startIndex + Int(codeLength) <= slice.endIndex else { + throw Error.invalidCodeLength + } + + let config = DefaultPvmConfig() + + let Q = StandardProgram.alignToSegmentSize + let ZP = config.pvmProgramInitPageSize + let ZQ = config.pvmProgramInitSegmentSize + let ZI = config.pvmProgramInitInputDataSize + let readOnlyAlignedSize = Int(Q(readOnlyLen, config)) + let heapEmptyPagesSize = Int(heapPages) * ZP + let readWriteAlignedSize = Int(Q(readWriteLen + UInt32(heapEmptyPagesSize), config)) + let stackAlignedSize = Int(Q(stackSize, config)) + + let totalSize = 5 * ZQ + readOnlyAlignedSize + readWriteAlignedSize + stackAlignedSize + ZI + guard totalSize <= Int32.max else { + throw Error.invalidTotalMemorySize + } + + code = try ProgramCode(blob[relative: slice.startIndex ..< slice.startIndex + Int(codeLength)]) + + initialRegisters = Registers(config: config, argumentData: argumentData) + + initialMemory = Memory( + readOnlyData: readOnlyData, + readWriteData: readWriteData, + argumentData: argumentData ?? Data(), + heapEmptyPagesSize: UInt32(heapEmptyPagesSize), + stackSize: UInt32(stackSize) + ) + } + + static func alignToPageSize(size: UInt32, config: PvmConfig) -> UInt32 { + let pageSize = UInt32(config.pvmProgramInitPageSize) + return (size + pageSize - 1) / pageSize * pageSize + } + + static func alignToSegmentSize(size: UInt32, config: PvmConfig) -> UInt32 { + let segmentSize = UInt32(config.pvmProgramInitSegmentSize) + return (size + segmentSize - 1) / segmentSize * segmentSize + } +} diff --git a/PolkaVM/Sources/PolkaVM/VMState.swift b/PolkaVM/Sources/PolkaVM/VMState.swift index c06c7312..3fc53db9 100644 --- a/PolkaVM/Sources/PolkaVM/VMState.swift +++ b/PolkaVM/Sources/PolkaVM/VMState.swift @@ -17,6 +17,16 @@ public class VMState { self.memory = memory } + /// Initialize from a standard program blob + public init(standardProgramBlob blob: Data, pc: UInt32, gas: UInt64, argumentData: Data?) throws { + let program = try StandardProgram(blob: blob, argumentData: argumentData) + self.program = program.code + registers = program.initialRegisters + memory = program.initialMemory + self.pc = pc + self.gas = Int64(gas) + } + public func getRegisters() -> Registers { registers } From 53683051a9fde3344e898355120c97832072c608 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Mon, 9 Sep 2024 11:28:42 +0800 Subject: [PATCH 2/2] fix lint (#106) --- Codec/Sources/Codec/JamDecoder.swift | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Codec/Sources/Codec/JamDecoder.swift b/Codec/Sources/Codec/JamDecoder.swift index ae8b6ddf..9132d885 100644 --- a/Codec/Sources/Codec/JamDecoder.swift +++ b/Codec/Sources/Codec/JamDecoder.swift @@ -237,7 +237,10 @@ private struct JamKeyedDecodingContainer: KeyedDecodingContainerPr func decode(_: String.Type, forKey key: K) throws -> String { let data: Data = try decoder.decodeData(codingPath: codingPath + [key]) - return String(decoding: data, as: UTF8.self) + guard let string = String(data: data, encoding: .utf8) else { + throw DecodingError.dataCorruptedError(forKey: key, in: self, debugDescription: "Invalid UTF8 string") + } + return string } func decode(_: Double.Type, forKey key: K) throws -> Double { @@ -352,7 +355,9 @@ private struct JamUnkeyedDecodingContainer: UnkeyedDecodingContainer { mutating func decode(_: String.Type) throws -> String { let data: Data = try decoder.decodeData(codingPath: codingPath) - let value = String(decoding: data, as: UTF8.self) + guard let value = String(data: data, encoding: .utf8) else { + throw DecodingError.dataCorruptedError(in: self, debugDescription: "Invalid UTF8 string") + } currentIndex += 1 return value } @@ -465,7 +470,10 @@ private struct JamSingleValueDecodingContainer: SingleValueDecodingContainer { func decode(_: String.Type) throws -> String { let data: Data = try decoder.decodeData(codingPath: codingPath) - return String(decoding: data, as: UTF8.self) + guard let string = String(data: data, encoding: .utf8) else { + throw DecodingError.dataCorruptedError(in: self, debugDescription: "Invalid UTF8 string") + } + return string } func decode(_: Double.Type) throws -> Double {