From 46e058d15e6fe41b0eb16df98cc5e19c001b0b7a Mon Sep 17 00:00:00 2001 From: b7r6 Date: Wed, 7 Oct 2026 16:21:11 -0400 Subject: [PATCH 1/2] ci: defer the main.yaml integration-tests OCI lane off per-PR Completes the no-per-PR-OCI-builds intent: image-off-pr moved image.yaml off PRs and #2877 moved the nix integration job off PRs, but main.yaml's integration-tests (docker-compose image build + run_integration_tests.sh) fell between them and still ran on every PR. Defer to nightly + merge-to-main, matching #2880's pattern. Fork-local until folded into the upstream OCI/nightly set. --- .github/workflows/main.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 43dacaf1ea0..232952a3a33 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,17 +1,23 @@ name: CI # Controls when the workflow will run. +# +# The only live job here (integration-tests) builds the trace_machina/nativelink +# OCI image from the docker-compose Dockerfile and runs run_integration_tests.sh. +# That is a per-PR OCI build — the same class of heavy work #image-off-pr moved +# off PRs for image.yaml and #2877 moved off PRs for the nix integration job. +# This lane fell in the crack between those two, so defer it the same way: +# nightly cross-check + on every merge to main, not on every PR. Per-PR +# build/test signal is carried by the Nix "Bazel Dev" lane. on: push: branches: [ main ] paths-ignore: - '.github/styles/**' - 'web/**' - pull_request: - branches: [ main ] - paths-ignore: - - '.github/styles/**' - - 'web/**' + schedule: + - cron: '0 7 * * *' # 07:00 UTC daily (off-peak) + workflow_dispatch: permissions: read-all From 25dd77fe84d3facb33d370b853c21a85fac837ca Mon Sep 17 00:00:00 2001 From: b7r6 Date: Fri, 2 Oct 2026 16:08:10 -0400 Subject: [PATCH 2/2] fix(store): wake in-flight slow-write waiters when any upload completes the digest An existence check that found a slow-store write in flight parked on that specific writer's completion mutex. A ByteStream upload that is interrupted (cancel, half-close, client deadline) is parked for resumption, keeping its FastSlowStore::update() future -- and thus the locked mutex -- alive. A later registration for the same key overwrote the map entry, so completing the digest under a fresh upload id never woke the original waiters: they starved until their client deadline even though the blob was present and new lookups succeeded. Replace the per-writer mutex with a per-key slot shared by all concurrent writers of that key, broadcasting Pending/Done over a watch channel. Any writer completing the digest wakes every waiter; the entry is removed (and waiters released with "not written") only when the last writer deregisters, which also fixes the stale-remove where a second writer's guard evicted a slot the first writer still occupied. Regression: stalled_upload_waiter_released_by_fresh_upload_of_same_digest -- fails on v1.7.3 (waiter hangs past a 2s deadline), passes with the fix; also asserts a genuinely-absent digest still resolves promptly. Co-Authored-By: Claude Opus 4.8 --- nativelink-store/src/fast_slow_store.rs | 103 ++++++++++++++---- .../tests/fast_slow_store_test.rs | 67 ++++++++++++ 2 files changed, 149 insertions(+), 21 deletions(-) diff --git a/nativelink-store/src/fast_slow_store.rs b/nativelink-store/src/fast_slow_store.rs index 08d91800559..d1b97bd6a9f 100644 --- a/nativelink-store/src/fast_slow_store.rs +++ b/nativelink-store/src/fast_slow_store.rs @@ -77,7 +77,33 @@ pub struct FastSlowStore { // concurrent writer that has not yet finished pushing to the slow store // is still visible to a concurrent existence check, preventing redundant // duplicate uploads of the same blob. - in_flight_slow_writes: Mutex, Arc>>>, + // + // Each key maps to a single shared slot that *all* concurrent writers of + // that key register on. This is load-bearing for liveness: waiters key on + // the digest, not on any individual upload, so a second upload (e.g. a + // retry under a fresh ByteStream upload id after the first was + // interrupted and parked for resumption) completing the blob wakes + // waiters that subscribed while the first upload was in flight. + in_flight_slow_writes: Mutex, InFlightSlowWrite>>, +} + +/// State of an in-flight slow-store write, broadcast to existence-check +/// waiters through a `watch` channel. +#[derive(Clone, Copy, Debug)] +enum InFlightWriteState { + /// At least one writer is still streaming this key to the slow store. + Pending, + /// The write finished: `Some(size)` if a writer completed successfully, + /// `None` if every writer gave up without completing. + Done(MaybeSize), +} + +/// Per-key slot shared by every concurrent writer of that key. +#[derive(Debug)] +struct InFlightSlowWrite { + tx: tokio::sync::watch::Sender, + /// Number of live [`InFlightSlowWriteGuard`]s registered on this slot. + writers: usize, } // This guard ensures that the populating_digests is cleared even if the future @@ -104,19 +130,22 @@ impl LoaderGuard<'_> { } } -/// Cancel-safe RAII guard that removes an entry from -/// `FastSlowStore::in_flight_slow_writes` when dropped. This ensures the -/// map does not leak entries if the surrounding `update()` future is -/// cancelled before the slow-store write completes. +/// Cancel-safe RAII guard that deregisters a writer from the key's shared +/// slot in `FastSlowStore::in_flight_slow_writes` when dropped. The map entry +/// is removed only when the *last* writer for the key deregisters, so a +/// parked/cancelled upload cannot orphan waiters nor evict a slot a newer +/// upload of the same key is still using. struct InFlightSlowWriteGuard { weak_store: Weak, key: Option>, - write_complete_guard: tokio::sync::OwnedMutexGuard, + tx: tokio::sync::watch::Sender, } impl InFlightSlowWriteGuard { - fn complete(mut self, size: MaybeSize) { - *self.write_complete_guard = size; + /// Marks the write done and wakes all waiters; deregistration happens in + /// `Drop`. + fn complete(self, size: MaybeSize) { + self.tx.send_replace(InFlightWriteState::Done(size)); } } @@ -128,7 +157,27 @@ impl Drop for InFlightSlowWriteGuard { let Some(key) = self.key.take() else { return; }; - store.in_flight_slow_writes.lock().remove(&key); + let mut map = store.in_flight_slow_writes.lock(); + let std::collections::hash_map::Entry::Occupied(mut occupied_entry) = map.entry(key) else { + return; + }; + // Only touch the slot this guard registered on (defense against a + // same-key slot that was replaced out from under us). + if !occupied_entry.get().tx.same_channel(&self.tx) { + return; + } + let slot = occupied_entry.get_mut(); + slot.writers -= 1; + if slot.writers == 0 { + // Last writer for this key. If nobody completed the write, wake + // waiters with "not written" so they do not hang forever. + slot.tx.send_modify(|state| { + if matches!(state, InFlightWriteState::Pending) { + *state = InFlightWriteState::Done(None); + } + }); + occupied_entry.remove(); + } } } @@ -175,18 +224,18 @@ impl FastSlowStore { fn register_in_flight_slow_write(&self, key: StoreKey<'_>) -> InFlightSlowWriteGuard { let owned = key.into_owned(); - let write_complete = Arc::new(tokio::sync::Mutex::new(None)); - let write_complete_guard = write_complete - .clone() - .try_lock_owned() - .expect("Newly created mutex is locked"); - self.in_flight_slow_writes - .lock() - .insert(owned.borrow().into_owned(), write_complete); + let mut map = self.in_flight_slow_writes.lock(); + let slot = map + .entry(owned.borrow().into_owned()) + .or_insert_with(|| InFlightSlowWrite { + tx: tokio::sync::watch::channel(InFlightWriteState::Pending).0, + writers: 0, + }); + slot.writers += 1; InFlightSlowWriteGuard { weak_store: self.weak_self.clone(), key: Some(owned), - write_complete_guard, + tx: slot.tx.clone(), } } @@ -560,9 +609,21 @@ impl StoreDriver for FastSlowStore { for (i, (k, result)) in key.iter().zip(results.iter_mut()).enumerate() { if result.is_none() { let owned = k.borrow().into_owned(); - if let Some(maybe_size) = in_flight.get(&owned) { - let maybe_size = maybe_size.clone(); - in_flight_futs.push(async move { (i, *maybe_size.lock().await) }); + if let Some(slot) = in_flight.get(&owned) { + let mut rx = slot.tx.subscribe(); + in_flight_futs.push(async move { + loop { + let state = *rx.borrow_and_update(); + match state { + InFlightWriteState::Done(size) => return (i, size), + InFlightWriteState::Pending => {} + } + if rx.changed().await.is_err() { + // All writers dropped without completing. + return (i, None); + } + } + }); } } } diff --git a/nativelink-store/tests/fast_slow_store_test.rs b/nativelink-store/tests/fast_slow_store_test.rs index 35a04d2511c..1514b815f67 100644 --- a/nativelink-store/tests/fast_slow_store_test.rs +++ b/nativelink-store/tests/fast_slow_store_test.rs @@ -2347,3 +2347,70 @@ async fn incomplete_slow_read_never_populates_memory_or_filesystem() -> Result<( } Ok(()) } + +// Regression test for N14: an interrupted ByteStream upload is parked for +// resumption, keeping its FastSlowStore::update() future (and in-flight +// slow-write registration) alive. Existence-check waiters that subscribed to +// that in-flight write must be woken when the SAME digest is completed by a +// DIFFERENT upload (fresh upload id), not starve until their client deadline. +#[nativelink_test] +async fn stalled_upload_waiter_released_by_fresh_upload_of_same_digest() -> Result<(), Error> { + let (fast_slow_store, _fast_store, _slow_store) = make_stores(); + let data = make_random_data(100); + let digest = DigestInfo::try_new(VALID_HASH, data.len())?; + + // Upload A: starts streaming, then stalls forever mid-stream (simulates a + // cancelled/half-closed ByteStream upload parked for resumption). We keep + // `stalled_tx` alive so the update() future stays pending. + let (mut stalled_tx, stalled_rx) = make_buf_channel_pair(); + let store_a = fast_slow_store.clone(); + let upload_a = tokio::spawn(async move { + store_a + .update(digest, stalled_rx, UploadSizeInfo::ExactSize(100)) + .await + }); + stalled_tx.send(Bytes::from(make_random_data(10))).await?; + // Let upload A register its in-flight slow write. + tokio::task::yield_now().await; + + // Waiter: FindMissingBlobs-style existence check that parks on the + // in-flight write of upload A. + let store_w = fast_slow_store.clone(); + let waiter = tokio::spawn(async move { store_w.has(digest).await }); + // Let the waiter subscribe to the in-flight write. + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiter.is_finished(), "Waiter should be parked on upload A"); + + // Upload B: completes the same digest under a "fresh upload id". + fast_slow_store + .update_oneshot(digest, data.clone().into()) + .await?; + + // The waiter must be released promptly by upload B's completion. + let result = tokio::time::timeout(Duration::from_secs(2), waiter) + .await + .map_err(|_| { + make_err!( + Code::DeadlineExceeded, + "Waiter hung after same digest was completed by a fresh upload" + ) + })? + .expect("waiter task panicked")?; + assert_eq!( + result, + Some(100), + "Waiter should see the completed blob size" + ); + + // A genuinely-absent digest must still resolve promptly once its only + // (stalled) writer goes away. + drop(stalled_tx); + drop(upload_a.await); + let absent = DigestInfo::try_new(VALID_HASH, 12345)?; + let absent_result = tokio::time::timeout(Duration::from_secs(2), fast_slow_store.has(absent)) + .await + .map_err(|_| make_err!(Code::DeadlineExceeded, "Absent digest check hung"))??; + assert_eq!(absent_result, None); + + Ok(()) +}