cl+el poc[WiP]

.gitmodules

@@ -171,7 +171,7 @@
 [submodule "vendor/nimbus-eth2"]
 	path = vendor/nimbus-eth2
 	url = https://github.com/status-im/nimbus-eth2.git
-	branch = unstable
+	branch = dev/pedro/unified_changes
 [submodule "vendor/nim-taskpools"]
 	path = vendor/nim-taskpools
 	url = https://github.com/status-im/nim-taskpools.git

Makefile

@@ -103,6 +103,7 @@ VERIF_PROXY_OUT_PATH ?= build/libverifproxy/
 	deps \
 	update \
 	nimbus \
+	nimbus_unified \
 	nimbus_execution_client \
 	fluffy \
 	nimbus_verified_proxy \
@@ -366,6 +367,13 @@ ifneq ($(USE_LIBBACKTRACE), 0)
 	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
 endif
 
+# Nimbus unified related targets
+
+# builds the unified client
+NIM_PARAMS := -d:release --parallelBuild:1 -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku,grandine $(NIM_PARAMS)
+nimbus_unified: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:disable_libbacktrace -d:libp2p_pki_schemes=secp256k1 -d:unified -o:build/$@ "nimbus_unified/$@.nim"
 # Note about building Nimbus as a library:
 #
 # There were `wrappers`, `wrappers-static`, `libnimbus.so` and `libnimbus.a`

config.nims

@@ -130,7 +130,9 @@ if not defined(windows):
 --opt:speed
 --excessiveStackTrace:on
 # enable metric collection
---define:metrics
+when not defined(unified):
+  --define:metrics
+
 # for heap-usage-by-instance-type metrics and object base-type strings
 --define:nimTypeNames
 --styleCheck:usages

nimbus.nimble

@@ -31,7 +31,8 @@ requires "nim >= 1.6.0",
   "blscurve",
   "evmc",
   "web3",
-  "minilru"
+  "minilru",
+  "secp256k1"
 
 binDir = "build"
 
@@ -40,6 +41,7 @@ when declared(namedBin):
     "nimbus/nimbus_execution_client": "nimbus_execution_client",
     "fluffy/fluffy": "fluffy",
     "nimbus_verified_proxy/nimbus_verified_proxy": "nimbus_verified_proxy",
+    "nimbus_unified":"nimbus_unified"
   }.toTable()
 
 import std/os
@@ -129,3 +131,9 @@ task nimbus_verified_proxy, "Build Nimbus verified proxy":
 
 task nimbus_verified_proxy_test, "Run Nimbus verified proxy tests":
   test "nimbus_verified_proxy/tests", "test_proof_validation", "-d:chronicles_log_level=ERROR -d:nimbus_db_backend=sqlite"
+
+
+## nimbus unified tasks
+
+task nimbus_unified, "Build nimbus unified":
+  buildBinary "nimbus_unified", "nimbus_unified/", "-d:chronicles_log_level=TRACE"

nimbus_unified/README.md

@@ -0,0 +1,50 @@
+# Nimbus Unified
+
+<!-- [![Github Actions CI](tbd) -->
+[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+[![Discord: Nimbus](https://img.shields.io/badge/discord-nimbus-orange.svg)](https://discord.gg/XRxWahP)
+[![Status: #nimbus-general](https://img.shields.io/badge/status-nimbus--general-orange.svg)](https://join.status.im/nimbus-general)
+
+
+# NOTE - whole document to be concluded...
+
+# description
+Nimbus Unified combines Ethereum execution and consensus layer functionalities, featuring a fully integrated beacon node, validator duties, and execution layer support. This setup allows the Nimbus client to handle both Ethereum consensus (Eth2) and execution (Eth1) tasks within a single package.
+
+--> meh, requires  improvement
+# documentation
+For in-depth configuration and functionality of Nimbus Eth1 and Nimbus Eth2, refer to:
+
+- [Nimbus-eth1 - Execution layer client](https://github.com/status-im/nimbus-eth1) Documentation
+- [Nimbus-eth2 - Consensus layer client](https://github.com/status-im/nimbus-eth2) Documentation
+
+--- to be concluded
+# dependencies
+tbd
+# how to
+## configuration
+    todo
+## commands
+    todo
+## compile
+tbd
+    - mac os, windows, and linux
+## colaborate
+We welcome contributions to Nimbus Unified! Please adhere to the following guidelines:
+
+- Follow the [Nimbus Code of Conduct](https://github.com/status-im/nimbus-eth2/blob/master/CODE_OF_CONDUCT.md).
+- Use the [Nimbus Code Style Guide](https://github.com/status-im/nimbus-eth2/blob/master/docs/code_style.md) to maintain code consistency.
+- Format your code using the [Nim Pretty Printer (nph)](https://github.com/nim-lang/nimpretty) to ensure consistency across the codebase. Run it as part of your pull request process.
+## License
+
+Licensed and distributed under either of
+
+* MIT license: [LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT
+
+or
+
+* Apache License, Version 2.0: [LICENSE-APACHEv2](LICENSE-APACHEv2) or https://www.apache.org/licenses/LICENSE-2.0
+
+These files may not be copied, modified, or distributed except according to those terms.

nimbus_unified/configs/nimbus_configs.nim

@@ -0,0 +1,51 @@
+# nimbus_unified
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import std/[os, atomics, tables], beacon_chain/nimbus_binary_common
+
+## Exceptions
+type NimbusTasksError* = object of CatchableError
+
+## Configuration
+## TODO: implement a json (or other format like yaml) config reader for config reading (file config scenarios)
+##  1) implement a command line reader to read arguments
+##  2) good option to adhere to other projects conventions and use the in place support to read and load
+type NimbusConfig* = object
+  configTable: Table[string, string]
+
+## Nimbus workers arguments (thread arguments)
+type TaskParameters* = object
+  name*: string
+  configs*: string
+  beaconNodeConfigs*: BeaconNodeConf
+
+## Task shutdown flag
+##
+## The behaviour required: this thread needs to atomically change the flag value when
+##  a shutdown is required or when detects a stopped thread.
+##
+## Given the behaviour wanted, atomic operations are sufficient without barriers or fences. Compilers
+##  may reorder instructions, but given that the order is not important, this does not affect
+##  the semantic wanted: If instructions are reordered, the worker will fail to read on the current iteration
+##  but will read it correctly on the next iteration ( this thread is the only on which changes the flag behaviour,
+##  and will always change it to true)
+##
+## With this we avoid the overhead of locks
+var isShutDownRequired*: Atomic[bool]
+isShutDownRequired.store(false)
+
+# TODO: move this into config.nim file once we have the file in place
+proc defaultDataDir*(): string =
+  let dataDir =
+    when defined(windows):
+      "AppData" / "Roaming" / "Nimbus_unified"
+    elif defined(macosx):
+      "Library" / "Application Support" / "Nimbus_unified"
+    else:
+      ".cache" / "nimbus_unified"
+
+  getHomeDir() / dataDir

nimbus_unified/consensus/consensus_wrapper.nim

@@ -0,0 +1,64 @@
+# nimbus_unified
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+#TODO: Clean these imports
+import
+  std/atomics,
+  metrics,
+  chronos,
+  chronicles,
+  ../configs/nimbus_configs,
+  beacon_chain/[beacon_chain_db, beacon_node, nimbus_beacon_node, nimbus_binary_common],
+  beacon_chain/rpc/[rest_beacon_api, rest_api],
+  beacon_chain/networking/[network_metadata, network_metadata_downloads],
+  beacon_chain/validators/[keystore_management]
+
+export nimbus_configs
+
+## log
+logScope:
+  topics = "Consensus layer"
+
+# handles option of eth2 beacon node
+proc handleStartingOption*(config: var BeaconNodeConf) {.raises: [CatchableError].} =
+  let rng = HmacDrbgContext.new()
+
+  # More options can be added, might be out of scope given that they exist in eth2
+  case config.cmd
+  of BNStartUpCmd.noCommand:
+    doRunBeaconNode(config, rng)
+  of BNStartUpCmd.trustedNodeSync:
+    if config.blockId.isSome():
+      raise newException(
+        ValueError, "--blockId option has been removed - use --state-id instead!"
+      )
+
+    let
+      metadata = loadEth2Network(config)
+      db = BeaconChainDB.new(config.databaseDir, metadata.cfg, inMemory = false)
+      genesisState = waitFor fetchGenesisState(metadata)
+    waitFor db.doRunTrustedNodeSync(
+      metadata, config.databaseDir, config.eraDir, config.trustedNodeUrl,
+      config.stateId, config.lcTrustedBlockRoot, config.backfillBlocks, config.reindex,
+      config.downloadDepositSnapshot, genesisState,
+    )
+    db.close()
+  else:
+    notice("unknonw option")
+    isShutDownRequired.store(true)
+
+proc consensusWrapper*(parameters: TaskParameters) {.raises: [CatchableError].} =
+  var config = parameters.beaconNodeConfigs
+
+  try:
+    handleStartingOption(config)
+  except CatchableError as e:
+    fatal "error", message = e.msg
+    isShutDownRequired.store(true)
+
+  isShutDownRequired.store(true)
+  warn "\tExiting consensus wrapper"

nimbus_unified/execution/execution_wrapper.nim

@@ -0,0 +1,26 @@
+# nimbus_unified
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import chronicles, std/[os, atomics], ../configs/nimbus_configs
+export nimbus_configs
+
+## log
+logScope:
+  topics = "Execution layer"
+
+const cTempExecutionTimeoutMs = 5000
+proc executionWrapper*(parameters: TaskParameters) =
+  info "Execution wrapper:", worker = parameters.name
+
+  while true:
+    sleep(cTempExecutionTimeoutMs)
+    info "looping execution"
+    if isShutDownRequired.load() == true:
+      break
+
+  isShutDownRequired.store(true)
+  warn "\tExiting execution:", worker = parameters.name

nimbus_unified/nimbus_unified.cfg

@@ -0,0 +1,17 @@
+# nimbus_unified
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+-d:"libp2p_pki_schemes=secp256k1"
+
+-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
+-d:"chronicles_runtime_filtering=on"
+-d:"chronicles_disable_thread_id"
+
+@if release:
+  -d:"chronicles_line_numbers:0"
+@end
+