From f9cdf41ad9c146324eb1c9d40048daa400501a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=B3nal=20Murray?= Date: Sat, 21 Dec 2024 22:29:13 +0100 Subject: [PATCH] [pallet-broker] add extrinsic to reserve a system core without having to wait two sale boundaries (#4273) When calling the reserve extrinsic after sales have started, the assignment will be reserved, but two sale period boundaries must pass before the core is actually assigned. Since this can take between 28 and 56 days on production networks, a new extrinsic is introduced to shorten the timeline. This essentially performs three actions: 1. Reserve it (applies after two sale boundaries) 2. Add it to the Workplan for the next sale period 3. Add it to the Workplan for the rest of the current sale period from the next timeslice to be commmitted. The caller must ensure that a core is first added, with most relay chain implementations having a delay of two session boundaries until it comes into effect. Alternatively the extrinsic can be called on a core whose workload can be clobbered from now until the reservation kicks in (the sale period after the next). Any workplan entries for that core at other timeslices should be first removed by the caller. --------- Co-authored-by: command-bot <> --- .../src/weights/pallet_broker.rs | 18 ++ .../src/weights/pallet_broker.rs | 18 ++ prdoc/pr_4273.prdoc | 19 ++ substrate/frame/broker/src/benchmarking.rs | 41 +++ .../frame/broker/src/dispatchable_impls.rs | 21 ++ substrate/frame/broker/src/lib.rs | 26 ++ substrate/frame/broker/src/tests.rs | 303 ++++++++++++++++++ substrate/frame/broker/src/weights.rs | 35 ++ 8 files changed, 481 insertions(+) create mode 100644 prdoc/pr_4273.prdoc diff --git a/cumulus/parachains/runtimes/coretime/coretime-rococo/src/weights/pallet_broker.rs b/cumulus/parachains/runtimes/coretime/coretime-rococo/src/weights/pallet_broker.rs index 5cb01f62cd26..3e4bbf379c3f 100644 --- a/cumulus/parachains/runtimes/coretime/coretime-rococo/src/weights/pallet_broker.rs +++ b/cumulus/parachains/runtimes/coretime/coretime-rococo/src/weights/pallet_broker.rs @@ -555,6 +555,24 @@ impl pallet_broker::WeightInfo for WeightInfo { .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(1)) } + /// Storage: `Broker::SaleInfo` (r:1 w:0) + /// Proof: `Broker::SaleInfo` (`max_values`: Some(1), `max_size`: Some(57), added: 552, mode: `MaxEncodedLen`) + /// Storage: `Broker::Reservations` (r:1 w:1) + /// Proof: `Broker::Reservations` (`max_values`: Some(1), `max_size`: Some(12021), added: 12516, mode: `MaxEncodedLen`) + /// Storage: `Broker::Status` (r:1 w:0) + /// Proof: `Broker::Status` (`max_values`: Some(1), `max_size`: Some(18), added: 513, mode: `MaxEncodedLen`) + /// Storage: `Broker::Workplan` (r:0 w:2) + /// Proof: `Broker::Workplan` (`max_values`: None, `max_size`: Some(1216), added: 3691, mode: `MaxEncodedLen`) + fn force_reserve() -> Weight { + // Proof Size summary in bytes: + // Measured: `11125` + // Estimated: `13506` + // Minimum execution time: 32_286_000 picoseconds. + Weight::from_parts(33_830_000, 0) + .saturating_add(Weight::from_parts(0, 13506)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(3)) + } /// Storage: `Broker::Leases` (r:1 w:1) /// Proof: `Broker::Leases` (`max_values`: Some(1), `max_size`: Some(401), added: 896, mode: `MaxEncodedLen`) fn swap_leases() -> Weight { diff --git a/cumulus/parachains/runtimes/coretime/coretime-westend/src/weights/pallet_broker.rs b/cumulus/parachains/runtimes/coretime/coretime-westend/src/weights/pallet_broker.rs index ad71691b2174..a0eee2d99efa 100644 --- a/cumulus/parachains/runtimes/coretime/coretime-westend/src/weights/pallet_broker.rs +++ b/cumulus/parachains/runtimes/coretime/coretime-westend/src/weights/pallet_broker.rs @@ -553,6 +553,24 @@ impl pallet_broker::WeightInfo for WeightInfo { .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(1)) } + /// Storage: `Broker::SaleInfo` (r:1 w:0) + /// Proof: `Broker::SaleInfo` (`max_values`: Some(1), `max_size`: Some(57), added: 552, mode: `MaxEncodedLen`) + /// Storage: `Broker::Reservations` (r:1 w:1) + /// Proof: `Broker::Reservations` (`max_values`: Some(1), `max_size`: Some(12021), added: 12516, mode: `MaxEncodedLen`) + /// Storage: `Broker::Status` (r:1 w:0) + /// Proof: `Broker::Status` (`max_values`: Some(1), `max_size`: Some(18), added: 513, mode: `MaxEncodedLen`) + /// Storage: `Broker::Workplan` (r:0 w:2) + /// Proof: `Broker::Workplan` (`max_values`: None, `max_size`: Some(1216), added: 3691, mode: `MaxEncodedLen`) + fn force_reserve() -> Weight { + // Proof Size summary in bytes: + // Measured: `11125` + // Estimated: `13506` + // Minimum execution time: 31_464_000 picoseconds. + Weight::from_parts(32_798_000, 0) + .saturating_add(Weight::from_parts(0, 13506)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(3)) + } /// Storage: `Broker::Leases` (r:1 w:1) /// Proof: `Broker::Leases` (`max_values`: Some(1), `max_size`: Some(81), added: 576, mode: `MaxEncodedLen`) fn swap_leases() -> Weight { diff --git a/prdoc/pr_4273.prdoc b/prdoc/pr_4273.prdoc new file mode 100644 index 000000000000..1ff0a5782a41 --- /dev/null +++ b/prdoc/pr_4273.prdoc @@ -0,0 +1,19 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: "[pallet-broker] add extrinsic to reserve a system core without having to wait two sale boundaries" + +doc: + - audience: Runtime User + description: | + When calling the reserve extrinsic after sales have started, the assignment will be reserved, + but two sale period boundaries must pass before the core is actually assigned. A new + `force_reserve` extrinsic is introduced to allow a core to be immediately assigned. + +crates: + - name: pallet-broker + bump: major + - name: coretime-rococo-runtime + bump: patch + - name: coretime-westend-runtime + bump: patch diff --git a/substrate/frame/broker/src/benchmarking.rs b/substrate/frame/broker/src/benchmarking.rs index 044689b254c5..516518740f7d 100644 --- a/substrate/frame/broker/src/benchmarking.rs +++ b/substrate/frame/broker/src/benchmarking.rs @@ -1016,6 +1016,47 @@ mod benches { Ok(()) } + #[benchmark] + fn force_reserve() -> Result<(), BenchmarkError> { + Configuration::::put(new_config_record::()); + // Assume Reservations to be almost filled for worst case. + let reservation_count = T::MaxReservedCores::get().saturating_sub(1); + setup_reservations::(reservation_count); + + // Assume leases to be filled for worst case + setup_leases::(T::MaxLeasedCores::get(), 1, 10); + + let origin = + T::AdminOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; + + // Sales must be started. + Broker::::do_start_sales(100u32.into(), CoreIndex::try_from(reservation_count).unwrap()) + .map_err(|_| BenchmarkError::Weightless)?; + + // Add a core. + let status = Status::::get().unwrap(); + Broker::::do_request_core_count(status.core_count + 1).unwrap(); + + advance_to::(T::TimeslicePeriod::get().try_into().ok().unwrap()); + let schedule = new_schedule(); + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, schedule.clone(), status.core_count); + + assert_eq!(Reservations::::decode_len().unwrap(), T::MaxReservedCores::get() as usize); + + let sale_info = SaleInfo::::get().unwrap(); + assert_eq!( + Workplan::::get((sale_info.region_begin, status.core_count)), + Some(schedule.clone()) + ); + // We called at timeslice 1, therefore 2 was already processed and 3 is the next possible + // assignment point. + assert_eq!(Workplan::::get((3, status.core_count)), Some(schedule)); + + Ok(()) + } + #[benchmark] fn swap_leases() -> Result<(), BenchmarkError> { let admin_origin = diff --git a/substrate/frame/broker/src/dispatchable_impls.rs b/substrate/frame/broker/src/dispatchable_impls.rs index 733d96625da0..489be12bdd15 100644 --- a/substrate/frame/broker/src/dispatchable_impls.rs +++ b/substrate/frame/broker/src/dispatchable_impls.rs @@ -60,6 +60,27 @@ impl Pallet { Ok(()) } + pub(crate) fn do_force_reserve(workload: Schedule, core: CoreIndex) -> DispatchResult { + // Sales must have started, otherwise reserve is equivalent. + let sale = SaleInfo::::get().ok_or(Error::::NoSales)?; + + // Reserve - starts at second sale period boundary from now. + Self::do_reserve(workload.clone())?; + + // Add to workload - grants one region from the next sale boundary. + Workplan::::insert((sale.region_begin, core), &workload); + + // Assign now until the next sale boundary unless the next timeslice is already the sale + // boundary. + let status = Status::::get().ok_or(Error::::Uninitialized)?; + let timeslice = status.last_committed_timeslice.saturating_add(1); + if timeslice < sale.region_begin { + Workplan::::insert((timeslice, core), &workload); + } + + Ok(()) + } + pub(crate) fn do_set_lease(task: TaskId, until: Timeslice) -> DispatchResult { let mut r = Leases::::get(); ensure!(until > Self::current_timeslice(), Error::::AlreadyExpired); diff --git a/substrate/frame/broker/src/lib.rs b/substrate/frame/broker/src/lib.rs index ed16b98d26cc..01368fd6404d 100644 --- a/substrate/frame/broker/src/lib.rs +++ b/substrate/frame/broker/src/lib.rs @@ -585,6 +585,9 @@ pub mod pallet { /// Reserve a core for a workload. /// + /// The workload will be given a reservation, but two sale period boundaries must pass + /// before the core is actually assigned. + /// /// - `origin`: Must be Root or pass `AdminOrigin`. /// - `workload`: The workload which should be permanently placed on a core. #[pallet::call_index(1)] @@ -943,6 +946,29 @@ pub mod pallet { Ok(()) } + /// Reserve a core for a workload immediately. + /// + /// - `origin`: Must be Root or pass `AdminOrigin`. + /// - `workload`: The workload which should be permanently placed on a core starting + /// immediately. + /// - `core`: The core to which the assignment should be made until the reservation takes + /// effect. It is left to the caller to either add this new core or reassign any other + /// tasks to this existing core. + /// + /// This reserves the workload and then injects the workload into the Workplan for the next + /// two sale periods. This overwrites any existing assignments for this core at the start of + /// the next sale period. + #[pallet::call_index(23)] + pub fn force_reserve( + origin: OriginFor, + workload: Schedule, + core: CoreIndex, + ) -> DispatchResultWithPostInfo { + T::AdminOrigin::ensure_origin_or_root(origin)?; + Self::do_force_reserve(workload, core)?; + Ok(Pays::No.into()) + } + #[pallet::call_index(99)] #[pallet::weight(T::WeightInfo::swap_leases())] pub fn swap_leases(origin: OriginFor, id: TaskId, other: TaskId) -> DispatchResult { diff --git a/substrate/frame/broker/src/tests.rs b/substrate/frame/broker/src/tests.rs index f3fd5234e4ca..a130a2050d9a 100644 --- a/substrate/frame/broker/src/tests.rs +++ b/substrate/frame/broker/src/tests.rs @@ -1837,3 +1837,306 @@ fn start_sales_sets_correct_core_count() { System::assert_has_event(Event::::CoreCountRequested { core_count: 9 }.into()); }) } + +// Reservations currently need two sale period boundaries to pass before coming into effect. +#[test] +fn reserve_works() { + TestExt::new().execute_with(|| { + assert_ok!(Broker::do_start_sales(100, 0)); + // Advance forward from start_sales, but not into the first sale. + advance_to(1); + + let system_workload = Schedule::truncate_from(vec![ScheduleItem { + mask: CoreMask::complete(), + assignment: Task(1004), + }]); + + // This shouldn't work, as the reservation will never be assigned a core unless one is + // available. + // assert_noop!(Broker::do_reserve(system_workload.clone()), Error::::Unavailable); + + // Add another core and create the reservation. + let status = Status::::get().unwrap(); + assert_ok!(Broker::request_core_count(RuntimeOrigin::root(), status.core_count + 1)); + assert_ok!(Broker::reserve(RuntimeOrigin::root(), system_workload.clone())); + + // This is added to reservations. + System::assert_last_event( + Event::ReservationMade { index: 0, workload: system_workload.clone() }.into(), + ); + assert_eq!(Reservations::::get(), vec![system_workload.clone()]); + + // But not yet in workplan for any of the next few regions. + for i in 0..20 { + assert_eq!(Workplan::::get((i, 0)), None); + } + // And it hasn't been assigned a core. + assert_eq!(CoretimeTrace::get(), vec![]); + + // Go to next sale. Rotate sale puts it in the workplan. + advance_sale_period(); + assert_eq!(Workplan::::get((7, 0)), Some(system_workload.clone())); + // But it still hasn't been assigned a core. + assert_eq!(CoretimeTrace::get(), vec![]); + + // Go to the second sale after reserving. + advance_sale_period(); + // Core is assigned at block 14 (timeslice 7) after being reserved all the way back at + // timeslice 1! Since the mock periods are 3 timeslices long, this means that reservations + // made in period 0 will only come into effect in period 2. + assert_eq!( + CoretimeTrace::get(), + vec![( + 12, + AssignCore { + core: 0, + begin: 14, + assignment: vec![(Task(1004), 57600)], + end_hint: None + } + )] + ); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 14, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + + // And it's in the workplan for the next period. + assert_eq!(Workplan::::get((10, 0)), Some(system_workload.clone())); + }); +} + +// We can use a hack to accelerate this by injecting it into the workplan. +#[test] +fn can_reserve_workloads_quickly() { + TestExt::new().execute_with(|| { + // Start sales. + assert_ok!(Broker::do_start_sales(100, 0)); + advance_to(2); + + let system_workload = Schedule::truncate_from(vec![ScheduleItem { + mask: CoreMask::complete(), + assignment: Task(1004), + }]); + + // This shouldn't work, as the reservation will never be assigned a core unless one is + // available. + // assert_noop!(Broker::do_reserve(system_workload.clone()), Error::::Unavailable); + + // Add another core and create the reservation. + let core_count = Status::::get().unwrap().core_count; + assert_ok!(Broker::request_core_count(RuntimeOrigin::root(), core_count + 1)); + assert_ok!(Broker::reserve(RuntimeOrigin::root(), system_workload.clone())); + + // These are the additional steps to onboard this immediately. + let core_index = core_count; + // In a real network this would call the relay chain + // `assigner_coretime::assign_core` extrinsic directly. + ::assign_core( + core_index, + 2, + vec![(Task(1004), 57600)], + None, + ); + // Inject into the workplan to ensure it's scheduled in the next rotate_sale. + Workplan::::insert((4, core_index), system_workload.clone()); + + // Reservation is added for the workload. + System::assert_has_event( + Event::ReservationMade { index: 0, workload: system_workload.clone() }.into(), + ); + System::assert_has_event(Event::CoreCountRequested { core_count: 1 }.into()); + + // It is also in the workplan for the next region. + assert_eq!(Workplan::::get((4, 0)), Some(system_workload.clone())); + + // Go to next sale. Rotate sale puts it in the workplan. + advance_sale_period(); + assert_eq!(Workplan::::get((7, 0)), Some(system_workload.clone())); + + // Go to the second sale after reserving. + advance_sale_period(); + + // Check the trace to ensure it has a core in every region. + assert_eq!( + CoretimeTrace::get(), + vec![ + ( + 2, + AssignCore { + core: 0, + begin: 2, + assignment: vec![(Task(1004), 57600)], + end_hint: None + } + ), + ( + 6, + AssignCore { + core: 0, + begin: 8, + assignment: vec![(Task(1004), 57600)], + end_hint: None + } + ), + ( + 12, + AssignCore { + core: 0, + begin: 14, + assignment: vec![(Task(1004), 57600)], + end_hint: None + } + ) + ] + ); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 8, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 14, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 14, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + + // And it's in the workplan for the next period. + assert_eq!(Workplan::::get((10, 0)), Some(system_workload.clone())); + }); +} + +// Add an extrinsic to do it properly. +#[test] +fn force_reserve_works() { + TestExt::new().execute_with(|| { + let system_workload = Schedule::truncate_from(vec![ScheduleItem { + mask: CoreMask::complete(), + assignment: Task(1004), + }]); + + // Not intended to work before sales are started. + assert_noop!( + Broker::force_reserve(RuntimeOrigin::root(), system_workload.clone(), 0), + Error::::NoSales + ); + + // Start sales. + assert_ok!(Broker::do_start_sales(100, 0)); + advance_to(1); + + // Add a new core. With the mock this is instant, with current relay implementation it + // takes two sessions to come into effect. + assert_ok!(Broker::do_request_core_count(1)); + + // Force reserve should now work. + assert_ok!(Broker::force_reserve(RuntimeOrigin::root(), system_workload.clone(), 0)); + + // Reservation is added for the workload. + System::assert_has_event( + Event::ReservationMade { index: 0, workload: system_workload.clone() }.into(), + ); + System::assert_has_event(Event::CoreCountRequested { core_count: 1 }.into()); + assert_eq!(Reservations::::get(), vec![system_workload.clone()]); + + // Advance to where that timeslice will be committed. + advance_to(3); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 4, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + + // It is also in the workplan for the next region. + assert_eq!(Workplan::::get((4, 0)), Some(system_workload.clone())); + + // Go to next sale. Rotate sale puts it in the workplan. + advance_sale_period(); + assert_eq!(Workplan::::get((7, 0)), Some(system_workload.clone())); + + // Go to the second sale after reserving. + advance_sale_period(); + + // Check the trace to ensure it has a core in every region. + assert_eq!( + CoretimeTrace::get(), + vec![ + ( + 2, + AssignCore { + core: 0, + begin: 4, + assignment: vec![(Task(1004), 57600)], + end_hint: None + } + ), + ( + 6, + AssignCore { + core: 0, + begin: 8, + assignment: vec![(Task(1004), 57600)], + end_hint: None + } + ), + ( + 12, + AssignCore { + core: 0, + begin: 14, + assignment: vec![(Task(1004), 57600)], + end_hint: None + } + ) + ] + ); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 8, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 14, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + System::assert_has_event( + Event::CoreAssigned { + core: 0, + when: 14, + assignment: vec![(CoreAssignment::Task(1004), 57600)], + } + .into(), + ); + + // And it's in the workplan for the next period. + assert_eq!(Workplan::::get((10, 0)), Some(system_workload.clone())); + }); +} diff --git a/substrate/frame/broker/src/weights.rs b/substrate/frame/broker/src/weights.rs index 894fed5a6a00..87e588551661 100644 --- a/substrate/frame/broker/src/weights.rs +++ b/substrate/frame/broker/src/weights.rs @@ -77,6 +77,7 @@ pub trait WeightInfo { fn notify_core_count() -> Weight; fn notify_revenue() -> Weight; fn do_tick_base() -> Weight; + fn force_reserve() -> Weight; fn swap_leases() -> Weight; fn enable_auto_renew() -> Weight; fn disable_auto_renew() -> Weight; @@ -487,6 +488,23 @@ impl WeightInfo for SubstrateWeight { .saturating_add(T::DbWeight::get().reads(4_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } + /// Storage: `Broker::SaleInfo` (r:1 w:0) + /// Proof: `Broker::SaleInfo` (`max_values`: Some(1), `max_size`: Some(57), added: 552, mode: `MaxEncodedLen`) + /// Storage: `Broker::Reservations` (r:1 w:1) + /// Proof: `Broker::Reservations` (`max_values`: Some(1), `max_size`: Some(6011), added: 6506, mode: `MaxEncodedLen`) + /// Storage: `Broker::Status` (r:1 w:0) + /// Proof: `Broker::Status` (`max_values`: Some(1), `max_size`: Some(18), added: 513, mode: `MaxEncodedLen`) + /// Storage: `Broker::Workplan` (r:0 w:2) + /// Proof: `Broker::Workplan` (`max_values`: None, `max_size`: Some(1216), added: 3691, mode: `MaxEncodedLen`) + fn force_reserve() -> Weight { + // Proof Size summary in bytes: + // Measured: `5253` + // Estimated: `7496` + // Minimum execution time: 28_363_000 picoseconds. + Weight::from_parts(29_243_000, 7496) + .saturating_add(T::DbWeight::get().reads(3_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) + } /// Storage: `Broker::Leases` (r:1 w:1) /// Proof: `Broker::Leases` (`max_values`: Some(1), `max_size`: Some(41), added: 536, mode: `MaxEncodedLen`) fn swap_leases() -> Weight { @@ -944,6 +962,23 @@ impl WeightInfo for () { .saturating_add(RocksDbWeight::get().reads(4_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } + /// Storage: `Broker::SaleInfo` (r:1 w:0) + /// Proof: `Broker::SaleInfo` (`max_values`: Some(1), `max_size`: Some(57), added: 552, mode: `MaxEncodedLen`) + /// Storage: `Broker::Reservations` (r:1 w:1) + /// Proof: `Broker::Reservations` (`max_values`: Some(1), `max_size`: Some(6011), added: 6506, mode: `MaxEncodedLen`) + /// Storage: `Broker::Status` (r:1 w:0) + /// Proof: `Broker::Status` (`max_values`: Some(1), `max_size`: Some(18), added: 513, mode: `MaxEncodedLen`) + /// Storage: `Broker::Workplan` (r:0 w:2) + /// Proof: `Broker::Workplan` (`max_values`: None, `max_size`: Some(1216), added: 3691, mode: `MaxEncodedLen`) + fn force_reserve() -> Weight { + // Proof Size summary in bytes: + // Measured: `5253` + // Estimated: `7496` + // Minimum execution time: 28_363_000 picoseconds. + Weight::from_parts(29_243_000, 7496) + .saturating_add(RocksDbWeight::get().reads(3_u64)) + .saturating_add(RocksDbWeight::get().writes(3_u64)) + } /// Storage: `Broker::Leases` (r:1 w:1) /// Proof: `Broker::Leases` (`max_values`: Some(1), `max_size`: Some(41), added: 536, mode: `MaxEncodedLen`) fn swap_leases() -> Weight {