From 29585b54b987289d4de2dbb841db1952bec5f50f Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:19:59 +0200 Subject: [PATCH 1/2] A draw of many promised frames takes the allocator's lock once Every frame a shared-memory region or a fault's populated run drew took the frame allocator's one lock on its own: 512 holds for a 2 MiB region, up to 16 for a fault. Under the build-shaped set at one child a CPU the lock was 99 % of every contended lock wait, its waits 1.05 of the T14's 8 CPUs and 7.1 of the AMD bench's 16 for the whole round, 82 % of the spin in the children's region creations and 13 to 14 % in their faults. `Frames::alloc_promised_each` draws `count` promised frames in one call, checking first that as many are promised; promises and ledgers move as for as many single draws. `pmm::alloc_promised_many` draws a region's zeroed frames and `pmm::alloc_promised_into` a run's unzeroed ones, a superframe's worth (512) to a hold, so a large region still lets other CPUs' draws in between. `SharedMemObject::create` takes a piece at a time, and a fault counts its run's empty pages, draws them together and writes each as its page starts (`fill_new`), which replaces `owned_frame` and `pmm::alloc_promised_unzeroed`. Frees already went back in batches. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/frames.rs | 25 ++++++++++++++++++++ kernel/src/mm/pmm.rs | 49 ++++++++++++++++++++++++++++++++-------- kernel/src/mm/space.rs | 34 ++++++++++++++++++---------- kernel/src/object/shm.rs | 6 +---- 4 files changed, 88 insertions(+), 26 deletions(-) diff --git a/kernel/pure/frames.rs b/kernel/pure/frames.rs index 32eb6c5457b..a0bad903584 100644 --- a/kernel/pure/frames.rs +++ b/kernel/pure/frames.rs @@ -769,6 +769,16 @@ impl<'m> Frames<'m> { phys } + /// `count` frames promises held, each handed to `each`: as many + /// [`alloc_promised`](Self::alloc_promised)s, which a caller makes under + /// one hold of the allocator's lock. + pub fn alloc_promised_each(&mut self, count: u64, mut each: impl FnMut(u64)) { + assert!(count <= self.promised, "frames: {count} promised frames drawn of {} promised", self.promised); + for _ in 0..count { + each(self.alloc_promised()); + } + } + /// One frame for the kernel's own use: one nothing holds back, or else /// one of a kept superframe, which is kept no longer. pub fn alloc_frame(&mut self) -> Option { @@ -1341,6 +1351,21 @@ mod tests { assert_eq!(m.frames.alloc_frame(), Some(first[7])); } + /// A batch of promised frames is as many single draws: distinct free + /// frames, each promise drawn once, and the ledger as it was. + #[test] + fn a_batch_of_promised_frames_draws_each_promise_once() { + let mut m = whole(2); + assert!(m.frames.promise(&m.l, 40)); + let (free, usage) = (m.frames.free_frames(), m.l.usage()); + let mut drawn = Vec::new(); + m.frames.alloc_promised_each(30, |phys| drawn.push(phys)); + assert_eq!(drawn.iter().collect::>().len(), 30, "no frame twice"); + assert_eq!((m.frames.promised(), m.frames.free_frames(), m.l.usage()), (10, free - 30, usage)); + let single = m.frames.alloc_promised(); + assert!(!drawn.contains(&single), "a single draw after the batch took one of its frames"); + } + /// What is promised is refused to every other allocation and drawn by /// its holder alone; a promise past what is free, less the superframes /// kept, is refused whole. diff --git a/kernel/src/mm/pmm.rs b/kernel/src/mm/pmm.rs index b822f0b4dd9..cd2154c7284 100644 --- a/kernel/src/mm/pmm.rs +++ b/kernel/src/mm/pmm.rs @@ -13,7 +13,9 @@ //! keeps. A job refused memory can wait for it to come back: [`ready`] //! answers whether it has, and [`returned`] is posted once it has. +use alloc::boxed::Box; use alloc::sync::Arc; +use alloc::vec::Vec; use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use kernel::frames::{self, Frames, Pins, PoolSplit, Super, Unpinned, FRAME, SUPER}; @@ -142,7 +144,7 @@ pub unsafe fn free_drawn(job: &Job, frames: &[u64]) { /// Give back every frame of `frames`, drawn for pages `job` committed, under /// one hold of the allocator's lock. -pub fn free_all(job: &Job, frames: alloc::boxed::Box<[Frame]>) { +pub fn free_all(job: &Job, frames: Box<[Frame]>) { let pins = pins(); with(|f| f.free_drawn(job.memory(), frames.iter().map(|frame| frame.phys), &pins)); // Given up only once the allocator's lock is let go: the list goes back @@ -261,9 +263,9 @@ fn kept_supers(frames: &Frames) -> u64 { } /// Promise `frames` to a commitment of `job`'s, which draws each with -/// [`alloc_promised`] or [`alloc_promised_unzeroed`] when it is first -/// touched; `false`, and nothing promised, where that would take what is -/// held back. +/// [`alloc_promised`], [`alloc_promised_many`] or [`alloc_promised_into`] +/// when it is first touched; `false`, and nothing promised, where that would +/// take what is held back. pub fn promise(job: &Job, frames: u64) -> bool { frames == 0 || with(|f| f.promise(job.memory(), frames)) } @@ -306,11 +308,40 @@ pub fn alloc_promised() -> Frame { frame } -/// [`alloc_promised`]'s frame holding what its last owner left, by its -/// address: its caller writes every byte of it before anything maps it, and -/// gives it back through [`free_drawn`] or [`discard`]. -pub(super) fn alloc_promised_unzeroed() -> u64 { - with(Frames::alloc_promised) +/// The most frames a batch draws under one hold of the allocator's lock, a +/// superframe's: a large region lets other CPUs' draws in between its holds. +const DRAWS_A_HOLD: usize = frames::PER_SUPER; + +/// `count` zeroed frames promises held, drawn [`DRAWS_A_HOLD`] to a hold of +/// the allocator's lock. +pub fn alloc_promised_many(count: usize) -> Box<[Frame]> { + let mut frames = Vec::with_capacity(count); + while frames.len() < count { + let n = (count - frames.len()).min(DRAWS_A_HOLD); + // Inside the capacity reserved above: nothing allocates under the lock. + with(|f| f.alloc_promised_each(n as u64, |phys| frames.push(Frame { phys }))); + } + for frame in &frames { + // SAFETY: just taken off the allocator, so unaliased, and inside the direct map. + unsafe { core::ptr::write_bytes(frame.direct_map().as_mut_ptr::(), 0, FRAME as usize) }; + } + frames.into_boxed_slice() +} + +/// `out.len()` frames promises held, drawn [`DRAWS_A_HOLD`] to a hold of the +/// allocator's lock, each holding what its last owner left: its caller +/// writes every byte of each before anything maps it, and gives them back +/// through [`free_drawn`] or [`discard`]. +pub(super) fn alloc_promised_into(out: &mut [u64]) { + for slots in out.chunks_mut(DRAWS_A_HOLD) { + let mut filled = 0; + with(|f| { + f.alloc_promised_each(slots.len() as u64, |phys| { + slots[filled] = phys; + filled += 1; + }); + }); + } } /// A zeroed frame for a page table of `job`'s, refused rather than take what diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 3c5f07c0b1e..42b7d763c5e 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -881,17 +881,25 @@ impl AddressSpace { phys(page) } Kind::Private { committed: true } | Kind::Copy { .. } => { - let (mut filled, mut own) = (0, 0); + // Every empty page of the run takes a frame, all drawn + // under one hold of the allocator's lock. + let empty = (lo..hi).step_by(PAGE_SIZE as usize).filter(|&at| leaves.get(at) == Leaf::Empty).count(); + let mut drawn = [0u64; (vm::AROUND / PAGE_SIZE) as usize]; + pmm::alloc_promised_into(&mut drawn[..empty]); + let mut frames = drawn[..empty].iter().copied(); + let mut own = 0; leaves.fill(lo, hi, |at| { - let phys = owned_frame(start, entry, at); - filled += 1; + let phys = frames.next().expect("space: a run filled past its empty pages"); + // SAFETY: a frame just drawn for this page, which no + // leaf maps yet, so nothing else reaches it. + unsafe { fill_new(start, entry, at, phys) }; if at == page { own = phys; } Leaf::Mapped(Mapping { phys, prot, owned: true, cache: CachePolicy::Normal }) }); assert_ne!(own, 0, "space: {page:#x} touched and not filled"); - self.usage.populated(filled); + self.usage.populated(empty as u64); own } } @@ -918,20 +926,22 @@ fn copyable(entry: &SpaceEntry) -> bool { } } -/// A fresh frame for `page` of the committed range `entry` at `start`, every -/// byte of it written: zeros, or for a copied range its source's bytes as far -/// as the range's copy and the source both go, and zeros after them. -fn owned_frame(start: u64, entry: &SpaceEntry, page: u64) -> u64 { - let phys = pmm::alloc_promised_unzeroed(); - // SAFETY: a frame just drawn for this page, which no leaf maps yet, so - // nothing else reaches it. +/// Write every byte of `phys`, a frame just drawn for `page` of the committed +/// range `entry` at `start`, as the page starts: zeros, or for a copied range +/// its source's bytes as far as the range's copy and the source both go, and +/// zeros after them. +/// +/// # Safety +/// +/// No leaf maps `phys` yet, so nothing else reaches it. +unsafe fn fill_new(start: u64, entry: &SpaceEntry, page: u64, phys: u64) { + // SAFETY: the caller's; a frame is a page of RAM the direct map reaches. let frame = unsafe { core::slice::from_raw_parts_mut(super::DirectMap::from_phys(phys).as_mut_ptr::(), PAGE_SIZE as usize) }; match &entry.kind { Kind::Private { .. } => frame.fill(0), Kind::Copy { source, offset, len } => vm::copied_page(frame, page - start, *offset, *len, source.size(), |from, bytes| source.read(from, bytes)), Kind::Object { .. } => unreachable!("space: an object's page owns no frame"), } - phys } /// Every frame the space owns goes back; its tables go with `UserTables`. diff --git a/kernel/src/object/shm.rs b/kernel/src/object/shm.rs index 44e5e60254a..d3dc64a73ee 100644 --- a/kernel/src/object/shm.rs +++ b/kernel/src/object/shm.rs @@ -91,11 +91,7 @@ impl SharedMemObject { let mut left = pages as usize; while left > 0 { let n = left.min(CHUNK); - let mut piece = Vec::with_capacity(n); - for _ in 0..n { - piece.push(pmm::alloc_promised()); - } - pieces.push(piece.into_boxed_slice()); + pieces.push(pmm::alloc_promised_many(n)); left -= n; } Ok(Self::new(core, Backing::Frames(pieces), pages * PAGE_SIZE, CachePolicy::Normal)) From 3eef788ab61b98969a07bc8b4b914d689d474b25 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 21:59:01 +0200 Subject: [PATCH 2/2] issues: the batched draw is built, and what #1268 owes on metal The frame allocator's record says the first decision is built here, #1265 having landed first, and names what main must show on metal for it: the benches are down, so the A/B and the lock census are owed rather than run. Co-Authored-By: Claude Opus 5.5 --- ...pmm-state-shares-lines-every-cpu-writes.md | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md b/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md index 95879749e97..aa28b7310e4 100644 --- a/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md +++ b/issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md @@ -59,25 +59,34 @@ build's 256 children, one a CPU, it is 99 % of every contended lock wait on both benches: 418,000 contended acquisitions and 243 ms of wait a round on the T14 (`j8`), 1.05 of its 8 CPUs for the whole round, and 605,000 and 2,348 ms on the AMD bench (`j16`), 7.1 of its 16. 82 % of the spin is in the children's -`SHM_CREATE`s, which draw each frame of what are, by inference from the SDK, +`SHM_CREATE`s, which drew each frame of what are, by inference from the SDK, their 2 MiB directory windows under its own hold (`SharedMemObject::create`), -and 13 to 14 % in their faults, which draw each frame of a populated run under -its own hold (`owned_frame`). The next contended lock waits under a hundredth +and 13 to 14 % in their faults, which drew each frame of a populated run under +its own hold. The next contended lock waits under a hundredth as long. On the T14 the spin is a third of the children's growth in CPU, so the lock bounds less of the build there than on the AMD bench, where it is 72 %. -**Decided** (the orchestrator, 2026-11-11), first of three: **a draw of many -frames takes the lock once.** A promised batch serves a shared-memory region's -frames and a fault's whole run, the way frees already go back, a region's drop -through `pmm::free_all` and a space's teardown through `free_drawn`: one -acquisition a window or a run instead of one a frame, with promises and ledgers -moving as for as many single draws. Per-CPU frame caches in front of the lock -are held until the batched build is measured; they would take single draws off -it too, but every promise must stay backed by a frame free now, so a draw that -finds the shared pool empty while another CPU's cache holds frames has to reach -them. The second, a region's frames drawn at first touch, and the third, the +**Decided** (the orchestrator, 2026-11-11), first of three, and built (#1268): +**a draw of many frames takes the lock once.** A shared-memory region's frames +(`pmm::alloc_promised_many`) and a fault's whole run (`pmm::alloc_promised_into`) +are drawn a superframe's worth to a hold, as a space's teardown already gave +frames back (`free_drawn`), with promises and ledgers moving as for as many +single draws. Per-CPU frame caches in front of the lock are held until the +batched build is measured; they would take single draws off it too, but every +promise must stay backed by a frame free now, so a draw that finds the shared +pool empty while another CPU's cache holds frames has to reach them. The second, a region's frames drawn at first touch, and the third, the window's size, are in their own records (the spawn record names them). +- **Owed on metal:** #1268, a draw of many promised frames under one hold of + the frame allocator's lock. An A/B of main before and after it on the T14 + and the AMD bench, of `osbench work` (`work.build_shaped.j1`, and `.j8` on + the T14 and `.j16` on the AMD bench), `spawn_cost` and `osbench procs`. Main + after must show `work.build_shaped.j` faster on both benches and no + `j1` or `proc` row slower beyond the before run's spread. And the lock + census of the build-shaped set (the spawn diagnosis's boot 3 instrument, + branch `spawnperf-split`, carried onto main after) must show this lock's + share of contended waits, 99 % before, fallen on both. + **Exit**: that attribution, posted, and read again after the change; the all-CPU build's lock waits read again with the batched draw; and each of the three either written per CPU on a line no other CPU writes, or kept with its