Add crate metadata for crates.io publishing (#40)

* add crate metadata for crates.io publishing

* apt-get?

* update readme

* introduce SlotTimestampProvider with fork support

* remove unneccessary cast

* restructure repo

* allow forking parachain

* just use stable

* specify exact version

* pin pallet-collator-selection

.github/workflows/ci.yml

@@ -22,10 +22,11 @@ jobs:
         with:
           toolchain: stable
 
+      - uses: Swatinem/rust-cache@v1
       - name: Install protoc
         run: |
-          sudo apt update
-          sudo apt install protobuf-compiler
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler
 
       - name: cargo check simnode
         run: cargo check -p sc-simnode
@@ -38,9 +39,7 @@ jobs:
 
       - uses: actions-rs/toolchain@v1
         with:
-          profile: minimal
           toolchain: stable
-          override: true
 
       - uses: Swatinem/rust-cache@v1
       - run: |
@@ -49,8 +48,8 @@ jobs:
 
       - name: Install protoc
         run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler build-essential
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler build-essential
 
       - name: Build all binaries
         run: |

Cargo.toml

@@ -3,21 +3,20 @@ resolver = "2"
 members = [
     "simnode",
     "runtime-api",
-
-    "integration/tests",
+    "tests",
 
     # parachain
-    "integration/parachain/node",
-    "integration/parachain/runtime",
+    "examples/parachain/node",
+    "examples/parachain/runtime",
 
     # aura
-    "integration/aura/node",
-    "integration/aura/runtime",
+    "examples/aura/node",
+    "examples/aura/runtime",
 
      # babe
-    "integration/babe/node",
-    "integration/babe/runtime",
-    "integration/babe/rpc",
+    "examples/babe/node",
+    "examples/babe/runtime",
+    "examples/babe/rpc",
 ]
 
 [workspace.dependencies]
@@ -41,7 +40,7 @@ pallet-offences = { version = "27.0.0", default-features = false }
 pallet-authorship = { version = "28.0.0", default-features = false }
 pallet-balances = { version = "28.0.0", default-features = false }
 pallet-session = { version = "28.0.0", default-features = false }
-pallet-staking = { version = "28.0.1", default-features = false }
+pallet-staking = { version = "28.0.0", default-features = false }
 pallet-session-benchmarking = { version = "28.0.0", default-features = false }
 pallet-sudo = { version = "28.0.0", default-features = false }
 pallet-timestamp = { version = "27.0.0", default-features = false }
@@ -50,7 +49,7 @@ pallet-transaction-payment-rpc-runtime-api = { version = "28.0.0", default-featu
 pallet-message-queue = { version = "31.0.0", default-features = false }
 pallet-beefy-mmr = { version = "28.0.0", default-features = false }
 pallet-assets = { version = "29.0.0", default-features = false }
-pallet-staking-reward-curve = { version = "8.0.0" }
+pallet-staking-reward-curve = { version = "11.0.0" }
 
 sp-api = { version = "26.0.0", default-features = false }
 sp-blockchain = { version = "28.0.0", default-features = false }
@@ -60,14 +59,15 @@ sp-block-builder = { version = "26.0.0", default-features = false }
 sp-consensus-aura = { version = "0.32.0", default-features = false }
 sp-consensus-babe = { version = "0.32.0", default-features = false }
 sp-consensus-grandpa = { version = "13.0.0", default-features = false }
+sp-consensus-slots = { version = "0.32.0", default-features = false }
 sp-consensus = { version = "0.32.0", default-features = false }
 sp-consensus-beefy = { version = "13.0.0", default-features = false }
 sp-core = { version = "28.0.0", default-features = false }
 sp-inherents = { version = "26.0.0", default-features = false }
 sp-externalities = { version = "0.25.0", default-features = false }
 sp-offchain = { version = "26.0.0", default-features = false }
 sp-keyring = { version = "31.0.0", default-features = false }
-sp-runtime = { version = "31.0.1", default-features = false }
+sp-runtime = { version = "31.0.0", default-features = false }
 sp-runtime-interface = { version = "24.0.0", default-features = false }
 sp-timestamp = { version = "26.0.0", default-features = false }
 sp-keystore = { version = "0.34.0", default-features = false }
@@ -87,8 +87,8 @@ pallet-xcm = { version = "7.0.0", default-features = false }
 polkadot-parachain-primitives = { version = "6.0.0", default-features = false }
 polkadot-runtime-common = { version = "7.0.0", default-features = false }
 staging-xcm = { version = "7.0.0", default-features = false }
-staging-xcm-builder = { version = "7.0.0", default-features = false }
-staging-xcm-executor = { version = "7.0.0", default-features = false }
+staging-xcm-builder = { version = "7.0.3", default-features = false }
+staging-xcm-executor = { version = "7.0.3", default-features = false }
 cumulus-primitives-aura = { version = "0.7.0", default-features = false }
 cumulus-pallet-session-benchmarking = { version = "9.0.0", default-features = false }
 cumulus-pallet-aura-ext = { version = "0.7.0", default-features = false }
@@ -98,20 +98,20 @@ cumulus-pallet-xcm = { version = "0.7.0", default-features = false }
 cumulus-pallet-xcmp-queue = { version = "0.7.0", default-features = false }
 cumulus-primitives-core = { version = "0.7.0", default-features = false }
 cumulus-primitives-timestamp = { version = "0.7.0", default-features = false }
-cumulus-primitives-utility = { version = "0.7.0", default-features = false }
-pallet-collator-selection = { version = "9.0.0", default-features = false }
-parachain-info = { package = "staging-parachain-info", version = "0.7.0", default-features = false }
+cumulus-primitives-utility = { version = "0.7.3", default-features = false }
+pallet-collator-selection = { version = "=9.0.0", default-features = false }
+parachain-info = { version = "0.7.0", package = "staging-parachain-info", default-features = false }
 parachains-common = { version = "7.0.0", default-features = false }
 
 # client
-parachain-inherent = {  package = "cumulus-primitives-parachain-inherent", version = "0.7.0" }
-sproof-builder = { package = "cumulus-test-relay-sproof-builder", version = "0.7.0" }
+parachain-inherent = { version = "0.7.0", package = "cumulus-primitives-parachain-inherent" }
+sproof-builder = { version = "0.7.0", package = "cumulus-test-relay-sproof-builder" }
 frame-benchmarking-cli = "32.0.0"
 pallet-transaction-payment-rpc = "30.0.0"
 sc-basic-authorship = "0.34.0"
 sc-chain-spec = "27.0.0"
 sc-cli = "0.36.0"
-sc-keystore = "24.0.0"
+sc-keystore = "25.0.0"
 sc-informant = "0.33.0"
 sc-block-builder = "0.33.0"
 sc-client-api = "28.0.0"
@@ -142,7 +142,7 @@ sc-tracing = "28.0.0"
 sc-transaction-pool = "28.0.0"
 sc-transaction-pool-api = "28.0.0"
 sc-offchain = "29.0.0"
-manual-seal = { package = "sc-consensus-manual-seal", version = "0.35.0" }
+manual-seal = { version = "0.35.0", package = "sc-consensus-manual-seal" }
 substrate-frame-rpc-system = "28.0.0"
 substrate-prometheus-endpoint = "0.17.0"
 try-runtime-cli = "0.38.0"

README.md

@@ -8,13 +8,22 @@
 
 This library exists to allow substrate developers to:
  - [x] Test complex pallets that require a full runtime, not a mocked runtime.
- - [x] Execute transactions from any origin on their live chain state.
+ - [x] Test a combination of both onchain & offchain components (eg offchain workers)
+ - [x] Fork your live chain state and execute transactions from any origin.
  - [x] Simulate runtime upgrades & migrations.
 
 
 ## Documentation
 
-Installation and integration guides can be found in the [book](https://simnode.polytope.technology).
+Installation and integration guides can be found in the [book](https://simnode.polytope.technology). Example integrations are provided for runtimes of all different kinds:
+
+- [x] [Standalone Chain](/examples/aura), AURA leader election, GRANDPA consensus.
+- [X] [Standalone Chain](/examples/babe), BABE leader election, GRANDPA consensus.
+- [X] [Parachain](/examples/parachain), AURA leader election, Polkadot Consensus.
+
+## Releases
+
+`sc-simnode` exclusively follows the official [paritytech/polkadot-sdk](https://github.com/paritytech/polkadot-sdk/releases) releases. Therefore new versions of this crate will be published with the same semver version of the corresponding polkadot-sdk release.
 
 ## License
 

runtime-api/Cargo.toml

@@ -1,12 +1,15 @@
 [package]
-edition = "2021"
 name = "simnode-runtime-api"
+version = "1.6.0"
 authors = ["Polytope Labs <hello@polytope.technology>"]
-version = "0.1.0"
-publish = false
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]
+edition = "2021"
+license = "Apache-2.0"
+repository = "https://github.com/polytope-labs/sc-simnode"
+homepage = "https://simnode.polytope.technology/"
+documentation = "https://docs.rs/sc-simnode"
+description = "Runtime APIs required for simnode integration"
+keywords = ["substrate", "polkadot-sdk", "integration-testing", "e2e-tests"]
+readme = "../README.md"
 
 [dependencies]
 codec = { workspace = true }

runtime-api/src/lib.rs

@@ -14,6 +14,12 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 #![cfg_attr(not(feature = "std"), no_std)]
 
+//! # simnode-runtime-api
+//!
+//!
+//! Exports a substrate runtime API that allows simnode create extrinsics that are always valid for
+//! the runtime
+
 use codec::Codec;
 use sp_std::vec::Vec;
 

simnode/Cargo.toml

@@ -1,8 +1,15 @@
 [package]
 name = "sc-simnode"
-version = "0.1.0"
+version = "1.6.1"
 authors = ["Polytope Labs <hello@polytope.technology>"]
 edition = "2021"
+license = "Apache-2.0"
+repository = "https://github.com/polytope-labs/sc-simnode"
+homepage = "https://simnode.polytope.technology/"
+documentation = "https://docs.rs/sc-simnode"
+description = "Simnode is an e2e integration testing environment for substrate-based blockchains"
+keywords = ["substrate", "polkadot-sdk", "integration-testing", "e2e-tests"]
+readme = "../README.md"
 
 [dependencies]
 # client deps
@@ -23,13 +30,16 @@ sc-client-api = { workspace = true }
 sc-rpc-server = { workspace = true }
 
 # consensus
-sc-consensus-babe = { workspace = true, optional = true }
+sc-consensus-babe = { workspace = true }
 manual-seal = { workspace = true }
 sc-consensus = { workspace = true }
+sc-consensus-slots = { workspace = true }
+sc-consensus-aura = { workspace = true, default-features = true }
 
 # primitive deps
 sp-consensus-aura = { workspace = true, default-features = true }
-sp-consensus-babe = { workspace = true, optional = true, default-features = true }
+sp-consensus-slots = { workspace = true, default-features = true }
+sp-consensus-babe = { workspace = true, default-features = true }
 sp-core = { workspace = true, default-features = true }
 sp-blockchain = { workspace = true, default-features = true }
 sp-block-builder = { workspace = true, default-features = true }
@@ -45,6 +55,7 @@ sp-keyring = { workspace = true, default-features = true }
 sp-externalities = { workspace = true, default-features = true }
 sp-state-machine = { workspace = true, default-features = true }
 sp-trie = { workspace = true, default-features = true }
+sp-timestamp = { workspace = true, default-features = true }
 sp-wasm-interface = { workspace = true, default-features = true }
 sp-runtime-interface = { workspace = true, default-features = true }
 
@@ -73,7 +84,7 @@ jsonrpsee = { version = "0.16.2", features = ["server"] }
 num-traits = "0.2.14"
 async-trait = "0.1.57"
 # local deps
-simnode-runtime-api = { path = "../runtime-api" }
+simnode-runtime-api = { version = "1.6.0", path = "../runtime-api" }
 
 
 [features]
@@ -84,4 +95,4 @@ std = []
 # consensus
 parachain = []
 aura = []
-babe = ["sp-consensus-babe", "sc-consensus-babe"]
+babe = []

simnode/src/client.rs

@@ -24,6 +24,7 @@ pub mod parachain;
 
 #[cfg(feature = "babe")]
 pub mod babe;
+pub mod timestamp;
 
 use crate::{ChainInfo, SignatureVerificationOverride};
 use jsonrpsee::RpcModule;

simnode/src/client/aura.rs

@@ -17,10 +17,9 @@
 //! Simnode for Standalone runtimes with Aura Consensus
 
 use super::*;
-use crate::{ChainInfo, SimnodeApiServer, SimnodeRpcHandler};
+use crate::{timestamp::SlotTimestampProvider, ChainInfo, SimnodeApiServer, SimnodeRpcHandler};
 use futures::{channel::mpsc, future::Either, FutureExt, StreamExt};
 use manual_seal::{
-	consensus::timestamp::SlotTimestampProvider,
 	rpc::{ManualSeal, ManualSealApiServer},
 	run_manual_seal, EngineCommand, ManualSealParams,
 };
@@ -201,17 +200,16 @@ where
 		consensus_data_provider: Some(Box::new(AuraConsensusDataProvider::new(client.clone()))),
 		create_inherent_data_providers: {
 			let client = client.clone();
-			move |_, _| {
+			move |parent, _| {
 				let client = client.clone();
 				async move {
 					let client = client.clone();
 
-					let timestamp = SlotTimestampProvider::new_aura(client.clone())
+					let timestamp = SlotTimestampProvider::new_aura(client.clone(), parent)
 						.map_err(|err| format!("{:?}", err))?;
 
-					let aura = sp_consensus_aura::inherents::InherentDataProvider::new(
-						timestamp.slot().into(),
-					);
+					let aura =
+						sp_consensus_aura::inherents::InherentDataProvider::new(timestamp.slot());
 
 					Ok((timestamp, aura))
 				}

simnode/src/client/babe.rs

@@ -18,7 +18,7 @@
 
 use futures::{channel::mpsc, future::Either, FutureExt, StreamExt};
 use manual_seal::{
-	consensus::{babe::BabeConsensusDataProvider, timestamp::SlotTimestampProvider},
+	consensus::babe::BabeConsensusDataProvider,
 	rpc::{ManualSeal, ManualSealApiServer},
 	run_manual_seal, EngineCommand, ManualSealParams,
 };
@@ -43,7 +43,7 @@ use sp_transaction_pool::runtime_api::TaggedTransactionQueue;
 
 use simnode_runtime_api::CreateTransactionApi;
 
-use crate::{ChainInfo, SimnodeApiServer, SimnodeRpcHandler};
+use crate::{timestamp::SlotTimestampProvider, ChainInfo, SimnodeApiServer, SimnodeRpcHandler};
 
 use super::*;
 
@@ -213,17 +213,16 @@ where
 		consensus_data_provider: Some(Box::new(babe_consensus)),
 		create_inherent_data_providers: {
 			let client = client.clone();
-			move |_, _| {
+			move |parent, _| {
 				let client = client.clone();
 				async move {
 					let client = client.clone();
 
-					let timestamp = SlotTimestampProvider::new_babe(client.clone())
+					let timestamp = SlotTimestampProvider::new_babe(client.clone(), parent)
 						.map_err(|err| format!("{:?}", err))?;
 
-					let babe = sp_consensus_babe::inherents::InherentDataProvider::new(
-						timestamp.slot().into(),
-					);
+					let babe =
+						sp_consensus_babe::inherents::InherentDataProvider::new(timestamp.slot());
 
 					Ok((timestamp, babe))
 				}

simnode/src/client/parachain.rs

@@ -17,12 +17,14 @@
 //! Simnode for Standalone runtimes with Parachain Consensus
 
 use super::*;
-use crate::{ChainInfo, ParachainSproofInherentProvider, SimnodeApiServer, SimnodeRpcHandler};
+use crate::{
+	timestamp::SlotTimestampProvider, ChainInfo, ParachainSproofInherentProvider, SimnodeApiServer,
+	SimnodeRpcHandler,
+};
 use async_trait::async_trait;
 use futures::{channel::mpsc, future::Either, lock::Mutex, FutureExt, StreamExt};
 use jsonrpsee::core::{Error as RpcError, RpcResult};
 use manual_seal::{
-	consensus::timestamp::SlotTimestampProvider,
 	rpc::{ManualSeal, ManualSealApiServer},
 	run_manual_seal, EngineCommand, ManualSealParams,
 };
@@ -331,22 +333,23 @@ where
 		create_inherent_data_providers: {
 			let client = client.clone();
 			let parachain_inherent_provider = parachain_inherent_provider.clone();
-			move |_, _| {
+			move |parent, _| {
 				let client = client.clone();
 				let parachain_sproof = parachain_inherent_provider.clone();
 				async move {
 					let client = client.clone();
 					let parachain_sproof = parachain_sproof.clone();
 
-					let timestamp = SlotTimestampProvider::new_aura(client.clone())
+					let timestamp = SlotTimestampProvider::new_aura(client.clone(), parent)
 						.map_err(|err| format!("{:?}", err))?;
 
-					let aura = sp_consensus_aura::inherents::InherentDataProvider::new(
-						timestamp.slot().into(),
-					);
+					let aura =
+						sp_consensus_aura::inherents::InherentDataProvider::new(timestamp.slot());
 
-					let parachain_system =
-						parachain_sproof.lock().await.create_inherent(timestamp.slot().into())?;
+					let parachain_system = parachain_sproof
+						.lock()
+						.await
+						.create_inherent(timestamp.slot().into(), parent)?;
 					Ok((timestamp, aura, parachain_system))
 				}
 			}