Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
@@ -1,17 +1,23 @@
name: CI

# Controls when the workflow will run.
#
# The only live job here (integration-tests) builds the trace_machina/nativelink
# OCI image from the docker-compose Dockerfile and runs run_integration_tests.sh.
# That is a per-PR OCI build — the same class of heavy work #image-off-pr moved
# off PRs for image.yaml and #2877 moved off PRs for the nix integration job.
# This lane fell in the crack between those two, so defer it the same way:
# nightly cross-check + on every merge to main, not on every PR. Per-PR
# build/test signal is carried by the Nix "Bazel Dev" lane.
on:
push:
branches: [ main ]
paths-ignore:
- '.github/styles/**'
- 'web/**'
pull_request:
branches: [ main ]
paths-ignore:
- '.github/styles/**'
- 'web/**'
schedule:
- cron: '0 7 * * *' # 07:00 UTC daily (off-peak)
workflow_dispatch:

permissions: read-all

Expand Down
103 changes: 82 additions & 21 deletions nativelink-store/src/fast_slow_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,33 @@ pub struct FastSlowStore {
// concurrent writer that has not yet finished pushing to the slow store
// is still visible to a concurrent existence check, preventing redundant
// duplicate uploads of the same blob.
in_flight_slow_writes: Mutex<HashMap<StoreKey<'static>, Arc<tokio::sync::Mutex<MaybeSize>>>>,
//
// Each key maps to a single shared slot that *all* concurrent writers of
// that key register on. This is load-bearing for liveness: waiters key on
// the digest, not on any individual upload, so a second upload (e.g. a
// retry under a fresh ByteStream upload id after the first was
// interrupted and parked for resumption) completing the blob wakes
// waiters that subscribed while the first upload was in flight.
in_flight_slow_writes: Mutex<HashMap<StoreKey<'static>, InFlightSlowWrite>>,
}

/// State of an in-flight slow-store write, broadcast to existence-check
/// waiters through a `watch` channel.
#[derive(Clone, Copy, Debug)]
enum InFlightWriteState {
/// At least one writer is still streaming this key to the slow store.
Pending,
/// The write finished: `Some(size)` if a writer completed successfully,
/// `None` if every writer gave up without completing.
Done(MaybeSize),
}

/// Per-key slot shared by every concurrent writer of that key.
#[derive(Debug)]
struct InFlightSlowWrite {
tx: tokio::sync::watch::Sender<InFlightWriteState>,
/// Number of live [`InFlightSlowWriteGuard`]s registered on this slot.
writers: usize,
}

// This guard ensures that the populating_digests is cleared even if the future
Expand All @@ -104,19 +130,22 @@ impl LoaderGuard<'_> {
}
}

/// Cancel-safe RAII guard that removes an entry from
/// `FastSlowStore::in_flight_slow_writes` when dropped. This ensures the
/// map does not leak entries if the surrounding `update()` future is
/// cancelled before the slow-store write completes.
/// Cancel-safe RAII guard that deregisters a writer from the key's shared
/// slot in `FastSlowStore::in_flight_slow_writes` when dropped. The map entry
/// is removed only when the *last* writer for the key deregisters, so a
/// parked/cancelled upload cannot orphan waiters nor evict a slot a newer
/// upload of the same key is still using.
struct InFlightSlowWriteGuard {
weak_store: Weak<FastSlowStore>,
key: Option<StoreKey<'static>>,
write_complete_guard: tokio::sync::OwnedMutexGuard<MaybeSize>,
tx: tokio::sync::watch::Sender<InFlightWriteState>,
}

impl InFlightSlowWriteGuard {
fn complete(mut self, size: MaybeSize) {
*self.write_complete_guard = size;
/// Marks the write done and wakes all waiters; deregistration happens in
/// `Drop`.
fn complete(self, size: MaybeSize) {
self.tx.send_replace(InFlightWriteState::Done(size));
}
}

Expand All @@ -128,7 +157,27 @@ impl Drop for InFlightSlowWriteGuard {
let Some(key) = self.key.take() else {
return;
};
store.in_flight_slow_writes.lock().remove(&key);
let mut map = store.in_flight_slow_writes.lock();
let std::collections::hash_map::Entry::Occupied(mut occupied_entry) = map.entry(key) else {
return;
};
// Only touch the slot this guard registered on (defense against a
// same-key slot that was replaced out from under us).
if !occupied_entry.get().tx.same_channel(&self.tx) {
return;
}
let slot = occupied_entry.get_mut();
slot.writers -= 1;
if slot.writers == 0 {
// Last writer for this key. If nobody completed the write, wake
// waiters with "not written" so they do not hang forever.
slot.tx.send_modify(|state| {
if matches!(state, InFlightWriteState::Pending) {
*state = InFlightWriteState::Done(None);
}
});
occupied_entry.remove();
}
}
}

Expand Down Expand Up @@ -175,18 +224,18 @@ impl FastSlowStore {

fn register_in_flight_slow_write(&self, key: StoreKey<'_>) -> InFlightSlowWriteGuard {
let owned = key.into_owned();
let write_complete = Arc::new(tokio::sync::Mutex::new(None));
let write_complete_guard = write_complete
.clone()
.try_lock_owned()
.expect("Newly created mutex is locked");
self.in_flight_slow_writes
.lock()
.insert(owned.borrow().into_owned(), write_complete);
let mut map = self.in_flight_slow_writes.lock();
let slot = map
.entry(owned.borrow().into_owned())
.or_insert_with(|| InFlightSlowWrite {
tx: tokio::sync::watch::channel(InFlightWriteState::Pending).0,
writers: 0,
});
slot.writers += 1;
InFlightSlowWriteGuard {
weak_store: self.weak_self.clone(),
key: Some(owned),
write_complete_guard,
tx: slot.tx.clone(),
}
}

Expand Down Expand Up @@ -560,9 +609,21 @@ impl StoreDriver for FastSlowStore {
for (i, (k, result)) in key.iter().zip(results.iter_mut()).enumerate() {
if result.is_none() {
let owned = k.borrow().into_owned();
if let Some(maybe_size) = in_flight.get(&owned) {
let maybe_size = maybe_size.clone();
in_flight_futs.push(async move { (i, *maybe_size.lock().await) });
if let Some(slot) = in_flight.get(&owned) {
let mut rx = slot.tx.subscribe();
in_flight_futs.push(async move {
loop {
let state = *rx.borrow_and_update();
match state {
InFlightWriteState::Done(size) => return (i, size),
InFlightWriteState::Pending => {}
}
if rx.changed().await.is_err() {
// All writers dropped without completing.
return (i, None);
}
}
});
}
}
}
Expand Down
67 changes: 67 additions & 0 deletions nativelink-store/tests/fast_slow_store_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2347,3 +2347,70 @@ async fn incomplete_slow_read_never_populates_memory_or_filesystem() -> Result<(
}
Ok(())
}

// Regression test for N14: an interrupted ByteStream upload is parked for
// resumption, keeping its FastSlowStore::update() future (and in-flight
// slow-write registration) alive. Existence-check waiters that subscribed to
// that in-flight write must be woken when the SAME digest is completed by a
// DIFFERENT upload (fresh upload id), not starve until their client deadline.
#[nativelink_test]
async fn stalled_upload_waiter_released_by_fresh_upload_of_same_digest() -> Result<(), Error> {
let (fast_slow_store, _fast_store, _slow_store) = make_stores();
let data = make_random_data(100);
let digest = DigestInfo::try_new(VALID_HASH, data.len())?;

// Upload A: starts streaming, then stalls forever mid-stream (simulates a
// cancelled/half-closed ByteStream upload parked for resumption). We keep
// `stalled_tx` alive so the update() future stays pending.
let (mut stalled_tx, stalled_rx) = make_buf_channel_pair();
let store_a = fast_slow_store.clone();
let upload_a = tokio::spawn(async move {
store_a
.update(digest, stalled_rx, UploadSizeInfo::ExactSize(100))
.await
});
stalled_tx.send(Bytes::from(make_random_data(10))).await?;
// Let upload A register its in-flight slow write.
tokio::task::yield_now().await;

// Waiter: FindMissingBlobs-style existence check that parks on the
// in-flight write of upload A.
let store_w = fast_slow_store.clone();
let waiter = tokio::spawn(async move { store_w.has(digest).await });
// Let the waiter subscribe to the in-flight write.
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(!waiter.is_finished(), "Waiter should be parked on upload A");

// Upload B: completes the same digest under a "fresh upload id".
fast_slow_store
.update_oneshot(digest, data.clone().into())
.await?;

// The waiter must be released promptly by upload B's completion.
let result = tokio::time::timeout(Duration::from_secs(2), waiter)
.await
.map_err(|_| {
make_err!(
Code::DeadlineExceeded,
"Waiter hung after same digest was completed by a fresh upload"
)
})?
.expect("waiter task panicked")?;
assert_eq!(
result,
Some(100),
"Waiter should see the completed blob size"
);

// A genuinely-absent digest must still resolve promptly once its only
// (stalled) writer goes away.
drop(stalled_tx);
drop(upload_a.await);
let absent = DigestInfo::try_new(VALID_HASH, 12345)?;
let absent_result = tokio::time::timeout(Duration::from_secs(2), fast_slow_store.has(absent))
.await
.map_err(|_| make_err!(Code::DeadlineExceeded, "Absent digest check hung"))??;
assert_eq!(absent_result, None);

Ok(())
}
Loading