Refactor the host <-> runtime interface machinery

.gitlab/pipeline/test.yml

@@ -358,9 +358,9 @@ test-frame-examples-compile-to-wasm:
     RUST_BACKTRACE: 1
   script:
     - cd ./substrate/frame/examples/offchain-worker/
-    - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+    - RUSTFLAGS="--cfg substrate_runtime" cargo build --locked --target=wasm32-unknown-unknown --no-default-features
     - cd ../basic
-    - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+    - RUSTFLAGS="--cfg substrate_runtime" cargo build --locked --target=wasm32-unknown-unknown --no-default-features
   # FIXME
   allow_failure: true
 

polkadot/runtime/metrics/src/with_runtime_metrics.rs

@@ -143,5 +143,5 @@ impl Histogram {
 
 /// Returns current time in ns
 pub fn get_current_time() -> u128 {
-	frame_benchmarking::benchmarking::current_time()
+	frame_benchmarking::current_time()
 }

prdoc/pr_3689.prdoc

@@ -0,0 +1,82 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# yaml-language-server: $schema=https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: Refactor the host <-> runtime interface machinery (the `#[runtime_interface]` macro) and the way host functions are defined
+
+doc:
+  - audience: Node Dev
+    description: |
+      This PR refactors the way the host functions are defined.
+
+      Previously the way a given type was marshalled through the host <-> runtime boundary was
+      hardcoded for every type by the virtue of it implementing the relevant conversion traits.
+
+      This had two major consequences:
+        * It was not obvious how a given type is going to be passed just by looking at its type alone,
+          masking potentially expensive marshalling strategies. (For example, returning `Option<u32>`
+          was done through the SCALE codec and involved extra memory allocations!)
+        * It was not possible to use multiple marshalling strategies for a single type, making some
+          of the future improvements we'd like to do (e.g. move the runtime memory allocator into the runtime)
+          very hard to do.
+
+      So this PR disentangles this mess and makes the marshalling strategies explicit. This makes it
+      much more clear how a given type in a given host function is marshalled, and also makes it possible
+      to use different marshalling strategies for the same type in different host functions.
+
+      Before this PR you'd define a host function like this:
+
+      ```rust
+          #[runtime_interface]
+          trait MyInterface {
+              fn say_hello_world(name: &str) {
+                  println!("Hello {name}!");
+              }
+          }
+      ```
+
+      and after this PR you'll define it like this:
+
+      ```rust
+          #[runtime_interface]
+          trait MyInterface {
+              fn say_hello_world(name: PassFatPointerAndRead<&str>) {
+                  println!("Hello {name}!", name);
+              }
+          }
+      ```
+
+      In this case the strategy for passing the `&str` is now explicitly specified (`PassFatPointerAndRead`).
+      Note that the *actual* API generated by this macro and the way arguments are accessed is completely unchanged!
+      The `#[runtime_interface]` machinery automatically "strips" away the marshalling strategy wrappers,
+      so neither the body of the `say_hello_world` function here nor its callers need to be changed.
+crates:
+  - name: sp-runtime-interface
+    bump: major
+    note: Rework of the `#[runtime_interface]` macro and associated types/trait.
+  - name: sp-runtime-interface-proc-macro
+    bump: major
+    note: Rework of the `#[runtime_interface]` macro.
+  - name: sp-wasm-interface
+    bump: major
+    note: The `Pointer` type now implements `Copy` and `Clone` unconditionally. The `Result` now always returns a `String`.
+  - name: sp-core
+    bump: major
+    note: Some types don't implement the traits related to the old `#[runtime_interface]` anymore. A few extra conversion impls.
+  - name: sp-io
+    bump: major
+    note: Requires the new `#[runtime_interface]` macro and associated machinery. Some types don't implement the traits related to the old `#[runtime_interface]` anymore.
+  - name: sp-statement-store
+    bump: major
+    note: Requires the new `#[runtime_interface]` macro and associated machinery. Some types don't implement the traits related to the old `#[runtime_interface]` anymore.
+  - name: sp-crypto-ec-utils
+    bump: minor
+    note: Requires the new `#[runtime_interface]` macro and associated machinery.
+  - name: frame-benchmarking
+    bump: major
+    note: Requires the new `#[runtime_interface]` macro and associated machinery. `Benchmarking::current_time` host call was changed.
+  - name: frame-support-procedural
+    bump: minor
+    note: Needs new `frame-benchmarking` due to the change to `current_time`.
+  - name: polkadot-runtime-metrics
+    bump: minor
+    note: Needs new `frame-benchmarking` due to the change to `current_time`.

substrate/client/executor/src/executor.rs

@@ -741,11 +741,11 @@ impl<D: NativeExecutionDispatch> sp_core::traits::ReadRuntimeVersion for NativeE
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use sp_runtime_interface::runtime_interface;
+	use sp_runtime_interface::{pass_by::PassFatPointerAndRead, runtime_interface};
 
 	#[runtime_interface]
 	trait MyInterface {
-		fn say_hello_world(data: &str) {
+		fn say_hello_world(data: PassFatPointerAndRead<&str>) {
 			println!("Hello world from: {}", data);
 		}
 	}

substrate/frame/benchmarking/src/utils.rs

@@ -23,6 +23,9 @@ use scale_info::TypeInfo;
 use serde::{Deserialize, Serialize};
 use sp_io::hashing::blake2_256;
 use sp_runtime::{traits::TrailingZeroInput, DispatchError};
+use sp_runtime_interface::pass_by::{
+	AllocateAndReturnByCodec, PassFatPointerAndDecode, PassFatPointerAndRead, PassPointerAndWrite,
+};
 use sp_std::vec::Vec;
 use sp_storage::TrackedStorageKey;
 
@@ -242,18 +245,29 @@ sp_api::decl_runtime_apis! {
 	}
 }
 
+/// Get the number of nanoseconds passed since the UNIX epoch
+///
+/// WARNING! This is a non-deterministic call. Do not use this within
+/// consensus critical logic.
+pub fn current_time() -> u128 {
+	let mut out = [0; 16];
+	self::benchmarking::current_time(&mut out);
+	u128::from_le_bytes(out)
+}
+
 /// Interface that provides functions for benchmarking the runtime.
 #[sp_runtime_interface::runtime_interface]
 pub trait Benchmarking {
 	/// Get the number of nanoseconds passed since the UNIX epoch
 	///
 	/// WARNING! This is a non-deterministic call. Do not use this within
 	/// consensus critical logic.
-	fn current_time() -> u128 {
-		std::time::SystemTime::now()
+	fn current_time(out: PassPointerAndWrite<&mut [u8; 16], 16>) {
+		*out = std::time::SystemTime::now()
 			.duration_since(std::time::SystemTime::UNIX_EPOCH)
 			.expect("Unix time doesn't go backwards; qed")
 			.as_nanos()
+			.to_le_bytes();
 	}
 
 	/// Reset the trie database to the genesis state.
@@ -267,7 +281,7 @@ pub trait Benchmarking {
 	}
 
 	/// Get the read/write count.
-	fn read_write_count(&self) -> (u32, u32, u32, u32) {
+	fn read_write_count(&self) -> AllocateAndReturnByCodec<(u32, u32, u32, u32)> {
 		self.read_write_count()
 	}
 
@@ -277,17 +291,17 @@ pub trait Benchmarking {
 	}
 
 	/// Get the DB whitelist.
-	fn get_whitelist(&self) -> Vec<TrackedStorageKey> {
+	fn get_whitelist(&self) -> AllocateAndReturnByCodec<Vec<TrackedStorageKey>> {
 		self.get_whitelist()
 	}
 
 	/// Set the DB whitelist.
-	fn set_whitelist(&mut self, new: Vec<TrackedStorageKey>) {
+	fn set_whitelist(&mut self, new: PassFatPointerAndDecode<Vec<TrackedStorageKey>>) {
 		self.set_whitelist(new)
 	}
 
 	// Add a new item to the DB whitelist.
-	fn add_to_whitelist(&mut self, add: TrackedStorageKey) {
+	fn add_to_whitelist(&mut self, add: PassFatPointerAndDecode<TrackedStorageKey>) {
 		let mut whitelist = self.get_whitelist();
 		match whitelist.iter_mut().find(|x| x.key == add.key) {
 			// If we already have this key in the whitelist, update to be the most constrained
@@ -306,18 +320,20 @@ pub trait Benchmarking {
 	}
 
 	// Remove an item from the DB whitelist.
-	fn remove_from_whitelist(&mut self, remove: Vec<u8>) {
+	fn remove_from_whitelist(&mut self, remove: PassFatPointerAndRead<Vec<u8>>) {
 		let mut whitelist = self.get_whitelist();
 		whitelist.retain(|x| x.key != remove);
 		self.set_whitelist(whitelist);
 	}
 
-	fn get_read_and_written_keys(&self) -> Vec<(Vec<u8>, u32, u32, bool)> {
+	fn get_read_and_written_keys(
+		&self,
+	) -> AllocateAndReturnByCodec<Vec<(Vec<u8>, u32, u32, bool)>> {
 		self.get_read_and_written_keys()
 	}
 
 	/// Get current estimated proof size.
-	fn proof_size(&self) -> Option<u32> {
+	fn proof_size(&self) -> AllocateAndReturnByCodec<Option<u32>> {
 		self.proof_size()
 	}
 }
@@ -397,11 +413,11 @@ impl<'a> Recording for BenchmarkRecording<'a> {
 	fn start(&mut self) {
 		(self.on_before_start.take().expect("start called more than once"))();
 		self.start_pov = crate::benchmarking::proof_size();
-		self.start_extrinsic = Some(crate::benchmarking::current_time());
+		self.start_extrinsic = Some(current_time());
 	}
 
 	fn stop(&mut self) {
-		self.finish_extrinsic = Some(crate::benchmarking::current_time());
+		self.finish_extrinsic = Some(current_time());
 		self.end_pov = crate::benchmarking::proof_size();
 	}
 }

substrate/frame/benchmarking/src/v1.rs

@@ -1132,9 +1132,9 @@ macro_rules! impl_benchmark {
 					);
 
 					// Time the storage root recalculation.
-					let start_storage_root = $crate::benchmarking::current_time();
+					let start_storage_root = $crate::current_time();
 					$crate::__private::storage_root($crate::__private::StateVersion::V1);
-					let finish_storage_root = $crate::benchmarking::current_time();
+					let finish_storage_root = $crate::current_time();
 					let elapsed_storage_root = finish_storage_root - start_storage_root;
 
 					let skip_meta = [ $( stringify!($name_skip_meta).as_ref() ),* ];

substrate/frame/support/procedural/src/benchmark.rs

@@ -807,9 +807,9 @@ pub fn benchmarks(
 						);
 
 						// Time the storage root recalculation.
-						let start_storage_root = #krate::benchmarking::current_time();
+						let start_storage_root = #krate::current_time();
 						#krate::__private::storage_root(#krate::__private::StateVersion::V1);
-						let finish_storage_root = #krate::benchmarking::current_time();
+						let finish_storage_root = #krate::current_time();
 						let elapsed_storage_root = finish_storage_root - start_storage_root;
 
 						let skip_meta = [ #(#skip_meta_benchmark_names_str),* ];

substrate/primitives/application-crypto/check-features-variants.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env -S bash -eux
 
 export RUSTFLAGS="-Cdebug-assertions=y -Dwarnings"
+cargo check --release
+
+export RUSTFLAGS="$RUSTFLAGS --cfg substrate_runtime"
 T=wasm32-unknown-unknown
-cargo check --release 
 cargo check --release --target=$T --no-default-features
 cargo check --release --target=$T --no-default-features  --features="full_crypto"
 cargo check --release --target=$T --no-default-features  --features="serde"

substrate/primitives/core/Cargo.toml

@@ -37,9 +37,6 @@ ss58-registry = { version = "1.34.0", default-features = false }
 sp-std = { path = "../std", default-features = false }
 sp-debug-derive = { path = "../debug-derive", default-features = false }
 sp-storage = { path = "../storage", default-features = false }
-sp-externalities = { path = "../externalities", optional = true }
-futures = { version = "0.3.30", optional = true }
-dyn-clonable = { version = "0.9.0", optional = true }
 thiserror = { optional = true, workspace = true }
 tracing = { version = "0.1.29", optional = true }
 bitflags = "1.3"
@@ -65,6 +62,11 @@ w3f-bls = { version = "0.1.3", default-features = false, optional = true }
 # bandersnatch crypto
 bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "e9782f9", default-features = false, features = ["substrate-curves"], optional = true }
 
+[target.'cfg(not(substrate_runtime))'.dependencies]
+sp-externalities = { path = "../externalities", default-features = false }
+futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
+dyn-clone = "1.0.17"
+
 [dev-dependencies]
 criterion = "0.4.0"
 serde_json = { workspace = true, default-features = true }
@@ -89,10 +91,9 @@ std = [
 	"bounded-collections/std",
 	"bs58/std",
 	"codec/std",
-	"dyn-clonable",
 	"ed25519-zebra/std",
 	"full_crypto",
-	"futures",
+	"futures/std",
 	"futures/thread-pool",
 	"hash-db/std",
 	"hash256-std-hasher/std",

substrate/primitives/core/check-features-variants.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env -S bash -eux
 
-export RUSTFLAGS="-Cdebug-assertions=y -Dwarnings"
+export RUSTFLAGS="-Cdebug-assertions=y -Dwarnings --cfg substrate_runtime"
 T=wasm32-unknown-unknown
 
 cargo check --target=$T --release --no-default-features  --features="bls-experimental"

substrate/primitives/core/src/crypto.rs

@@ -26,7 +26,6 @@ use itertools::Itertools;
 use rand::{rngs::OsRng, RngCore};
 use scale_info::TypeInfo;
 pub use secrecy::{ExposeSecret, SecretString};
-use sp_runtime_interface::pass_by::PassByInner;
 #[doc(hidden)]
 pub use sp_std::ops::Deref;
 #[cfg(all(not(feature = "std"), feature = "serde"))]
@@ -1032,11 +1031,11 @@ pub trait CryptoType {
 	Hash,
 	Encode,
 	Decode,
-	PassByInner,
 	crate::RuntimeDebug,
 	TypeInfo,
 )]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[repr(transparent)]
 pub struct KeyTypeId(pub [u8; 4]);
 
 impl From<u32> for KeyTypeId {
@@ -1051,6 +1050,18 @@ impl From<KeyTypeId> for u32 {
 	}
 }
 
+impl From<[u8; 4]> for KeyTypeId {
+	fn from(value: [u8; 4]) -> Self {
+		Self(value)
+	}
+}
+
+impl AsRef<[u8]> for KeyTypeId {
+	fn as_ref(&self) -> &[u8] {
+		&self.0
+	}
+}
+
 impl<'a> TryFrom<&'a str> for KeyTypeId {
 	type Error = ();