Skip to content

Commit

Permalink
Merge branch 'master' into dev_actor
Browse files Browse the repository at this point in the history
* master:
  update and fix testvectors (#146)
  validator service tests (#145)
  safrole service tests (#143)
  fix log filtering (#142)
  improve logging (#141)
  refactor and more tests (#140)
  Block author tickets (#139)
  block author tests (#138)
  Fix accessor (#137)
  tests for DispatchQueueScheduler (#136)
  update readme
  use external libs (#133)
  use saturating number for gas and balance (#135)
  pvm accumulate functions and inovcation (#132)
  • Loading branch information
MacOMNI committed Oct 4, 2024
2 parents 2b67767 + 4ad83f3 commit 7d33357
Show file tree
Hide file tree
Showing 111 changed files with 9,191 additions and 1,200 deletions.
4 changes: 4 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,7 @@ insert_final_newline=true
[*.swift]
indent_style=space
tab_width=4

[*.yml]
indent_style=tab
indent_size=2
35 changes: 4 additions & 31 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,32 +25,19 @@ jobs:
test:
name: Build and Test
runs-on: [self-hosted, linux]

steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
submodules: recursive
- name: MsQuic Install Dependencies
run: |
export DEBIAN_FRONTEND=noninteractive
sudo apt-add-repository -y ppa:lttng/stable-2.13
sudo apt-get update
sudo apt-get install -y lttng-tools lttng-modules-dkms babeltrace2 liblttng-ust-dev python3-babeltrace
sudo apt-get install -y cmake
sudo apt-get install -y build-essential
- name: Get msquic submodule commit hash
id: msquic-commit-hash
run: |
echo "commit-hash=$(git submodule status Networking/Sources/msquic/ | cut -c2-41)" >> $GITHUB_OUTPUT
- run: sudo apt-get update
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: librocksdb-dev libzstd-dev libbz2-dev liblz4-dev
- name: Get blst submodule commit hash
id: blst-commit-hash
run: |
echo "commit-hash=$(git submodule status Utils/Sources/blst/ | cut -c2-41)" >> $GITHUB_OUTPUT
- name: Get rocksdb submodule commit hash
id: rocksdb-commit-hash
run: |
echo "commit-hash=$(git submodule status Database/Sources/rocksdb/ | cut -c2-41)" >> $GITHUB_OUTPUT
- name: Cache SPM
uses: actions/cache@v4
with:
Expand All @@ -68,13 +55,6 @@ jobs:
~/.cargo/git/db/
Utils/Sources/bandersnatch/target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Cache msquic static lib
uses: actions/cache@v4
with:
path: .lib/libmsquic.a
key: ${{ runner.os }}-libs-msquic-${{ steps.msquic-commit-hash.outputs.commit-hash }}
restore-keys: |
${{ runner.os }}-libs-msquic
- name: Cache blst static lib
uses: actions/cache@v4
with:
Expand All @@ -89,13 +69,6 @@ jobs:
key: ${{ runner.os }}-libs-libbandersnatch-${{ hashFiles('Utils/Sources/bandersnatch/**') }}
restore-keys: |
${{ runner.os }}-libs-libbandersnatch
- name: Cache rocksdb static lib
uses: actions/cache@v4
with:
path: .lib/librocksdb.a
key: ${{ runner.os }}-libs-librocksdb-${{ steps.rocksdb-commit-hash.outputs.commit-hash }}
restore-keys: |
${{ runner.os }}-libs-librocksdb
- name: Cache erasure-coding static lib
uses: actions/cache@v4
with:
Expand Down
6 changes: 0 additions & 6 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,3 @@
[submodule "blst"]
path = Utils/Sources/blst
url = https://github.com/supranational/blst.git
[submodule "Database/Sources/rocksdb"]
path = Database/Sources/rocksdb
url = https://github.com/facebook/rocksdb.git
[submodule "Networking/Sources/msquic"]
path = Networking/Sources/msquic
url = https://github.com/microsoft/msquic.git
11 changes: 10 additions & 1 deletion Blockchain/Package.resolved
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"originHash" : "2a91ceec1663a1ed3fc9b58333d3d32c6ce1131b7d1874346655dbfae0af61f4",
"originHash" : "64fa76cb48bfb721e9426cf0d246c06245d80b6c98b16ca1a7c9ba00acfabb1b",
"pins" : [
{
"identity" : "blake2.swift",
Expand Down Expand Up @@ -55,6 +55,15 @@
"version" : "2.5.0"
}
},
{
"identity" : "swift-numerics",
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-numerics",
"state" : {
"branch" : "main",
"revision" : "e30276bff2ff5ed80566fbdca49f50aa160b0e83"
}
},
{
"identity" : "swift-service-context",
"kind" : "remoteSourceControl",
Expand Down
78 changes: 10 additions & 68 deletions Blockchain/Sources/Blockchain/Blockchain.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,9 @@ import Utils

private let logger = Logger(label: "Blockchain")

private struct BlockchainStorage: Sendable {
var bestHead: Data32?
var bestHeadTimeslot: TimeslotIndex?
var finalizedHead: Data32
}

/// Holds the state of the blockchain.
/// Includes the canonical chain as well as pending forks.
/// Assume all blocks and states are valid and have been validated.
public final class Blockchain: ServiceBase, @unchecked Sendable {
private let storage: ThreadSafeContainer<BlockchainStorage>
private let dataProvider: BlockchainDataProvider
private let timeProvider: TimeProvider
public let dataProvider: BlockchainDataProvider
public let timeProvider: TimeProvider

public init(
config: ProtocolConfigRef,
Expand All @@ -27,24 +17,6 @@ public final class Blockchain: ServiceBase, @unchecked Sendable {
self.dataProvider = dataProvider
self.timeProvider = timeProvider

let heads = try await dataProvider.getHeads()
var bestHead: (HeaderRef, Data32)?
for head in heads {
guard let header = try? await dataProvider.getHeader(hash: head) else {
continue
}
if bestHead == nil || header.value.timeslot > bestHead!.0.value.timeslot {
bestHead = (header, head)
}
}
let finalizedHead = try await dataProvider.getFinalizedHead()

storage = ThreadSafeContainer(.init(
bestHead: bestHead?.1,
bestHeadTimeslot: bestHead?.0.value.timeslot,
finalizedHead: finalizedHead
))

super.init(config, eventBus)

await subscribe(RuntimeEvents.BlockAuthored.self) { [weak self] event in
Expand All @@ -57,58 +29,28 @@ public final class Blockchain: ServiceBase, @unchecked Sendable {
}

public func importBlock(_ block: BlockRef) async throws {
logger.debug("importing block: \(block.hash)")

try await withSpan("importBlock") { span in
span.attributes.blockHash = block.hash.description

let runtime = Runtime(config: config)
let parent = try await dataProvider.getState(hash: block.header.parentHash)
let timeslot = timeProvider.getTime() / UInt32(config.value.slotPeriodSeconds)
let timeslot = timeProvider.getTimeslot()
let state = try runtime.apply(block: block, state: parent, context: .init(timeslot: timeslot))
try await dataProvider.add(state: state)

// update best head
if state.value.timeslot > storage.value.bestHeadTimeslot ?? 0 {
storage.write { storage in
storage.bestHead = block.hash
storage.bestHeadTimeslot = state.value.timeslot
}
}
try await dataProvider.blockImported(block: block, state: state)

await publish(RuntimeEvents.BlockImported(block: block, state: state, parentState: parent))
publish(RuntimeEvents.BlockImported(block: block, state: state, parentState: parent))
}
}

public func finalize(hash: Data32) async throws {
logger.debug("finalizing block: \(hash)")

// TODO: purge forks
try await dataProvider.setFinalizedHead(hash: hash)

storage.write { storage in
storage.finalizedHead = hash
}

await publish(RuntimeEvents.BlockFinalized(hash: hash))
}

public func getBestBlock() async throws -> BlockRef {
guard let hash = try await dataProvider.getHeads().first else {
try throwUnreachable("no head")
}
return try await dataProvider.getBlock(hash: hash)
}

public func getBlock(hash: Data32) async throws -> BlockRef? {
try await dataProvider.getBlock(hash: hash)
}

public func getState(hash: Data32) async throws -> StateRef? {
try await dataProvider.getState(hash: hash)
}

public var bestHead: Data32 {
storage.value.bestHead ?? Data32()
}

public var finalizedHead: Data32 {
storage.value.finalizedHead
publish(RuntimeEvents.BlockFinalized(hash: hash))
}
}
Original file line number Diff line number Diff line change
@@ -1,37 +1,132 @@
import TracingUtils
import Utils

public enum BlockchainDataProviderError: Error {
case noData
private let logger = Logger(label: "BlockchainDataProvider")

private struct BlockchainStorage: Sendable {
var bestHead: Data32
var bestHeadTimeslot: TimeslotIndex
var finalizedHead: Data32
}

public final class BlockchainDataProvider {
private let storage: ThreadSafeContainer<BlockchainStorage>
private let dataProvider: BlockchainDataProviderProtocol

public init(_ dataProvider: BlockchainDataProviderProtocol) async throws {
let heads = try await dataProvider.getHeads()
var bestHead: (HeaderRef, Data32)?
for head in heads {
guard let header = try? await dataProvider.getHeader(hash: head) else {
continue
}
if bestHead == nil || header.value.timeslot > bestHead!.0.value.timeslot {
bestHead = (header, head)
}
}
let finalizedHead = try await dataProvider.getFinalizedHead()

storage = ThreadSafeContainer(.init(
bestHead: bestHead?.1 ?? Data32(),
bestHeadTimeslot: bestHead?.0.value.timeslot ?? 0,
finalizedHead: finalizedHead
))

self.dataProvider = dataProvider
}

public var bestHead: Data32 {
storage.value.bestHead
}

public var finalizedHead: Data32 {
storage.value.finalizedHead
}

public func blockImported(block: BlockRef, state: StateRef) async throws {
try await add(block: block)
try await add(state: state)
try await updateHead(hash: block.hash, parent: block.header.parentHash)

if block.header.timeslot > storage.value.bestHeadTimeslot {
storage.write { storage in
storage.bestHead = block.hash
storage.bestHeadTimeslot = block.header.timeslot
}
}

logger.debug("block imported: \(block.hash)")
}
}

public protocol BlockchainDataProvider: Sendable {
func hasBlock(hash: Data32) async throws -> Bool
func hasState(hash: Data32) async throws -> Bool
func isHead(hash: Data32) async throws -> Bool
// expose BlockchainDataProviderProtocol
extension BlockchainDataProvider {
public func hasBlock(hash: Data32) async throws -> Bool {
try await dataProvider.hasBlock(hash: hash)
}

public func hasState(hash: Data32) async throws -> Bool {
try await dataProvider.hasState(hash: hash)
}

public func isHead(hash: Data32) async throws -> Bool {
try await dataProvider.isHead(hash: hash)
}

public func getHeader(hash: Data32) async throws -> HeaderRef {
try await dataProvider.getHeader(hash: hash)
}

public func getBlock(hash: Data32) async throws -> BlockRef {
try await dataProvider.getBlock(hash: hash)
}

public func getState(hash: Data32) async throws -> StateRef {
try await dataProvider.getState(hash: hash)
}

public func getFinalizedHead() async throws -> Data32 {
try await dataProvider.getFinalizedHead()
}

public func getHeads() async throws -> Set<Data32> {
try await dataProvider.getHeads()
}

public func getBlockHash(byTimeslot timeslot: TimeslotIndex) async throws -> Set<Data32> {
try await dataProvider.getBlockHash(byTimeslot: timeslot)
}

public func add(block: BlockRef) async throws {
logger.debug("adding block: \(block.hash)")

try await dataProvider.add(block: block)
}

/// throw BlockchainDataProviderError.noData if not found
func getHeader(hash: Data32) async throws -> HeaderRef
public func add(state: StateRef) async throws {
logger.debug("adding state: \(state.value.lastBlockHash)")

/// throw BlockchainDataProviderError.noData if not found
func getBlock(hash: Data32) async throws -> BlockRef
try await dataProvider.add(state: state)
}

/// throw BlockchainDataProviderError.noData if not found
func getState(hash: Data32) async throws -> StateRef
public func setFinalizedHead(hash: Data32) async throws {
logger.debug("setting finalized head: \(hash)")

/// throw BlockchainDataProviderError.noData if not found
func getFinalizedHead() async throws -> Data32
func getHeads() async throws -> Set<Data32>
try await dataProvider.setFinalizedHead(hash: hash)
storage.write { storage in
storage.finalizedHead = hash
}
}

/// return empty set if not found
func getBlockHash(byTimeslot timeslot: TimeslotIndex) async throws -> Set<Data32>
public func updateHead(hash: Data32, parent: Data32) async throws {
logger.debug("updating head: \(hash) with parent: \(parent)")

func add(block: BlockRef) async throws
func add(state: StateRef) async throws
func setFinalizedHead(hash: Data32) async throws
try await dataProvider.updateHead(hash: hash, parent: parent)
}

/// throw BlockchainDataProviderError.noData if parent is not a head
func updateHead(hash: Data32, parent: Data32) async throws
public func remove(hash: Data32) async throws {
logger.debug("removing block: \(hash)")

/// remove header, block and state
func remove(hash: Data32) async throws
try await dataProvider.remove(hash: hash)
}
}
Loading

0 comments on commit 7d33357

Please sign in to comment.