Remark storage

Cargo.toml

@@ -109,6 +109,7 @@ members = [
 	"frame/randomness-collective-flip",
 	"frame/recovery",
 	"frame/referenda",
+	"frame/remark",
 	"frame/scheduler",
 	"frame/scored-pool",
 	"frame/session",

bin/node/runtime/Cargo.toml

@@ -83,6 +83,7 @@ pallet-proxy = { version = "4.0.0-dev", default-features = false, path = "../../
 pallet-randomness-collective-flip = { version = "4.0.0-dev", default-features = false, path = "../../../frame/randomness-collective-flip" }
 pallet-recovery = { version = "4.0.0-dev", default-features = false, path = "../../../frame/recovery" }
 pallet-referenda = { version = "4.0.0-dev", default-features = false, path = "../../../frame/referenda" }
+pallet-remark = { version = "4.0.0-dev", default-features = false, path = "../../../frame/remark" }
 pallet-session = { version = "4.0.0-dev", features = [ "historical" ], path = "../../../frame/session", default-features = false }
 pallet-session-benchmarking = { version = "4.0.0-dev", path = "../../../frame/session/benchmarking", default-features = false, optional = true }
 pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/staking" }
@@ -173,6 +174,7 @@ std = [
 	"sp-version/std",
 	"pallet-society/std",
 	"pallet-referenda/std",
+	"pallet-remark/std",
 	"pallet-recovery/std",
 	"pallet-uniques/std",
 	"pallet-vesting/std",
@@ -213,6 +215,7 @@ runtime-benchmarks = [
 	"pallet-proxy/runtime-benchmarks",
 	"pallet-scheduler/runtime-benchmarks",
 	"pallet-referenda/runtime-benchmarks",
+	"pallet-remark/runtime-benchmarks",
 	"pallet-session-benchmarking",
 	"pallet-society/runtime-benchmarks",
 	"pallet-staking/runtime-benchmarks",

bin/node/runtime/src/lib.rs

@@ -772,6 +772,12 @@ impl pallet_referenda::Config for Runtime {
 	type Tracks = TracksInfo;
 }
 
+impl pallet_remark::Config for Runtime {
+	type WeightInfo = pallet_remark::weights::SubstrateWeight<Self>;
+	type Call = Call;
+	type Event = Event;
+}
+
 parameter_types! {
 	pub const LaunchPeriod: BlockNumber = 28 * 24 * 60 * MINUTES;
 	pub const VotingPeriod: BlockNumber = 28 * 24 * 60 * MINUTES;
@@ -1446,6 +1452,7 @@ construct_runtime!(
 		StateTrieMigration: pallet_state_trie_migration,
 		ChildBounties: pallet_child_bounties,
 		Referenda: pallet_referenda,
+		Remark: pallet_remark,
 		ConvictionVoting: pallet_conviction_voting,
 		Whitelist: pallet_whitelist,
 	}
@@ -1535,6 +1542,7 @@ mod benches {
 		[pallet_preimage, Preimage]
 		[pallet_proxy, Proxy]
 		[pallet_referenda, Referenda]
+		[pallet_remark, Remark]
 		[pallet_scheduler, Scheduler]
 		[pallet_session, SessionBench::<Runtime>]
 		[pallet_staking, Staking]

frame/remark/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "pallet-remark"
+version = "4.0.0-dev"
+authors = ["Parity Technologies <[email protected]>"]
+edition = "2021"
+license = "Apache-2.0"
+homepage = "https://substrate.io"
+repository = "https://github.com/paritytech/substrate/"
+description = "Remark storage pallet"
+readme = "README.md"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+serde = { version = "1.0.136", optional = true }
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false }
+scale-info = { version = "2.0.1", default-features = false, features = ["derive"] }
+frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
+frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" }
+sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" }
+sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" }
+sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" }
+sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" }
+frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true }
+
+[dev-dependencies]
+sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false }
+
+[features]
+default = ["std"]
+runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"]
+std = [
+	"serde",
+	"codec/std",
+	"scale-info/std",
+	"sp-runtime/std",
+	"frame-support/std",
+	"frame-system/std",
+	"sp-io/std",
+	"sp-std/std",
+]

frame/remark/README.md

@@ -0,0 +1,82 @@
+# Transaction Storage Pallet
+
+Indexes transactions and manages storage proofs.
+
+Allows storing arbitrary data on the chain. Data is automatically removed after `StoragePeriod` blocks, unless the storage is renewed.
+Validators must submit proof of storing a random chunk of data for block `N - StoragePeriod` when producing block `N`.
+
+# Running a chain
+
+The following describes how to set up a new storage chain.
+
+Start with generating a chain spec.
+
+```bash
+cargo run --release -- build-spec --chain=local > sc_init.json
+```
+
+Edit the json chain spec file to customise the chain. The storage chain genesis params are configured in the `transactionStorage` section.
+Note that `storagePeriod` is specified in blocks and changing it also requires code changes at the moment.
+
+Build a raw spec from the init spec.
+
+```bash
+cargo run --release build-spec --chain=sc_init.json --raw > sc.json
+```
+
+Run a few validator nodes.
+
+```bash
+cargo run --release -- --chain=sc.json -d /tmp/alice --storage-chain --keep-blocks=100800 --ipfs-server --validator --alice
+cargo run --release -- --chain=sc.json -d /tmp/bob --storage-chain --keep-blocks=100800 --ipfs-server --validator --bob
+```
+
+`--storage-chain` enables transaction indexing.
+`--keep-blocks=100800` enables block pruning. The value here should be greater or equal than the storage period.
+`--ipfs-server` enables serving stored content over IPFS.
+
+Once the network is started, any other joining nodes need to sync with `--sync=fast`. Regular sync will fail because block pruning removes old blocks. The chain does not keep full block history.
+
+```bash
+cargo run --release -- --chain=sc.json -d /tmp/charlie --storage-chain --keep-blocks=100800 --ipfs-server --validator --charlie --sync=fast
+```
+
+# Making transactions
+
+To store data use the `transactionStorage.store` extrinsic. And IPFS CID can be generated from the Blake2-256 hash of the data.
+
+```js
+const util_crypto = require('@polkadot/util-crypto');
+const keyring_api = require('@polkadot/keyring');
+const polkadot_api = require('@polkadot/api');
+const fs = require('fs');
+const multihash = require('multihashes');
+const CID = require('cids')
+
+const wsProvider = new polkadot_api.WsProvider();
+const api = await polkadot_api.ApiPromise.create({ provider: wsProvider });
+
+const keyring = new keyring_api.Keyring({ type: "sr25519" });
+const alice = keyring.addFromUri("//Alice");
+
+const file = fs.readFileSync('cute_kitten.jpeg');
+const hash = util_crypto.blake2AsU8a(file)
+const encoded_hash = multihash.encode(hash, 'blake2b-256');
+
+const cid = new CID(1, 'blake2b-256', encoded_hash)
+console.log(cid.toString());
+
+const txHash = await api.tx.transactionStorage.store('0x' + file.toString('hex')).signAndSend(alice);
+```
+Data can be queried over IPFS
+
+```bash
+ipfs swarm connect <substrate peer address>
+ipfs block get /ipfs/<CID> > kitten.jpeg
+```
+
+To renew data and prevent it from being disposed after the storage period, use `transactionStorage.renew(block, index)`
+where `block` is the block number of the previous store or renew transction, and index is the index of that transaction in the block.
+
+
+License: Apache-2.0

frame/remark/src/benchmarking.rs

@@ -0,0 +1,44 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Benchmarks for transaction-storage Pallet
+
+#![cfg(feature = "runtime-benchmarks")]
+
+use super::*;
+use frame_benchmarking::{benchmarks, whitelisted_caller};
+use frame_system::{EventRecord, Pallet as System, RawOrigin};
+use sp_std::*;
+
+fn assert_last_event<T: Config>(generic_event: <T as Config>::Event) {
+	let events = System::<T>::events();
+	let system_event: <T as frame_system::Config>::Event = generic_event.into();
+	let EventRecord { event, .. } = &events[events.len() - 1];
+	assert_eq!(event, &system_event);
+}
+
+benchmarks! {
+	store {
+		let l in 1 .. 1024*1024;
+		let caller: T::AccountId = whitelisted_caller();
+	}: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize])
+	verify {
+		assert_last_event::<T>(Event::Stored { sender: caller, content_hash: sp_io::hashing::blake2_256(&vec![0u8; l as usize]).into() }.into());
+	}
+
+	impl_benchmark_test_suite!(Remark, crate::mock::new_test_ext(), crate::mock::Test);
+}

frame/remark/src/lib.rs

@@ -0,0 +1,91 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Remark storage pallet. Indexes remarks and stores them off chain.
+
+// Ensure we're `no_std` when compiling for Wasm.
+#![cfg_attr(not(feature = "std"), no_std)]
+
+mod benchmarking;
+pub mod weights;
+
+#[cfg(test)]
+mod mock;
+#[cfg(test)]
+mod tests;
+
+use frame_support::dispatch::{Dispatchable, GetDispatchInfo};
+use sp_std::prelude::*;
+
+// Re-export pallet items so that they can be accessed from the crate namespace.
+pub use pallet::*;
+pub use weights::WeightInfo;
+
+#[frame_support::pallet]
+pub mod pallet {
+	use super::*;
+	use frame_support::pallet_prelude::*;
+	use frame_system::pallet_prelude::*;
+
+	#[pallet::config]
+	pub trait Config: frame_system::Config {
+		/// The overarching event type.
+		type Event: From<Event<Self>> + IsType<<Self as frame_system::Config>::Event>;
+		/// A dispatchable call.
+		type Call: Parameter
+			+ Dispatchable<Origin = Self::Origin>
+			+ GetDispatchInfo
+			+ From<frame_system::Call<Self>>;
+		/// Weight information for extrinsics in this pallet.
+		type WeightInfo: WeightInfo;
+	}
+
+	#[pallet::error]
+	pub enum Error<T> {
+		/// Attempting to store empty data.
+		Empty,
+		/// Attempted to call `store` outside of block execution.
+		BadContext,
+	}
+
+	#[pallet::pallet]
+	#[pallet::without_storage_info]
+	pub struct Pallet<T>(_);
+
+	#[pallet::call]
+	impl<T: Config> Pallet<T> {
+		/// Index and store data off chain.
+		#[pallet::weight(T::WeightInfo::store(remark.len() as u32))]
+		pub fn store(origin: OriginFor<T>, remark: Vec<u8>) -> DispatchResultWithPostInfo {
+			ensure!(!remark.is_empty(), Error::<T>::Empty);
+			let sender = ensure_signed(origin)?;
+			let content_hash = sp_io::hashing::blake2_256(&remark);
+			let extrinsic_index = <frame_system::Pallet<T>>::extrinsic_index()
+				.ok_or_else(|| Error::<T>::BadContext)?;
+			sp_io::transaction_index::index(extrinsic_index, remark.len() as u32, content_hash);
+			Self::deposit_event(Event::Stored { sender, content_hash: content_hash.into() });
+			Ok(().into())
+		}
+	}
+
+	#[pallet::event]
+	#[pallet::generate_deposit(pub(super) fn deposit_event)]
+	pub enum Event<T: Config> {
+		/// Stored data off chain.
+		Stored { sender: T::AccountId, content_hash: sp_core::H256 },
+	}
+}