Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
@@ -1,17 +1,23 @@
name: CI

# Controls when the workflow will run.
#
# The only live job here (integration-tests) builds the trace_machina/nativelink
# OCI image from the docker-compose Dockerfile and runs run_integration_tests.sh.
# That is a per-PR OCI build — the same class of heavy work #image-off-pr moved
# off PRs for image.yaml and #2877 moved off PRs for the nix integration job.
# This lane fell in the crack between those two, so defer it the same way:
# nightly cross-check + on every merge to main, not on every PR. Per-PR
# build/test signal is carried by the Nix "Bazel Dev" lane.
on:
push:
branches: [ main ]
paths-ignore:
- '.github/styles/**'
- 'web/**'
pull_request:
branches: [ main ]
paths-ignore:
- '.github/styles/**'
- 'web/**'
schedule:
- cron: '0 7 * * *' # 07:00 UTC daily (off-peak)
workflow_dispatch:

permissions: read-all

Expand Down
323 changes: 323 additions & 0 deletions nativelink-scheduler/src/api_worker_scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2075,6 +2075,14 @@ pub struct ApiWorkerScheduler {

/// Woken when a worker joins or leaves this scheduler.
local_fleet_change_notify: Arc<Notify>,

/// Test-only rendezvous: `remove_timedout_workers` parks here between its
/// liveness snapshot and re-acquiring the eviction lock, at an existing
/// await boundary, so tests can deterministically interleave a keepalive.
/// `.0` is signalled when the sweep reaches the boundary, `.1` lets it
/// proceed.
#[cfg(test)]
test_evict_gate: std::sync::OnceLock<Arc<(Notify, Notify)>>,
}

impl ApiWorkerScheduler {
Expand Down Expand Up @@ -2160,6 +2168,8 @@ impl ApiWorkerScheduler {
local_fleet_change_notify,
capacity_generation,
scheduler_id,
#[cfg(test)]
test_evict_gate: std::sync::OnceLock::new(),
})
}

Expand Down Expand Up @@ -2748,10 +2758,41 @@ impl WorkerScheduler for ApiWorkerScheduler {
return Ok(());
}

// Test-only hold at this pre-existing await boundary (the inner lock
// was dropped after the snapshot and is only re-acquired below); lets
// a test run a concurrent keepalive here deterministically.
#[cfg(test)]
if let Some(gate) = self.test_evict_gate.get() {
gate.0.notify_one();
gate.1.notified().await;
}

let mut inner = self.inner.lock().await;
let mut result = Ok(());

for (worker_id, kill_overdue) in &worker_ids_to_remove {
// The staleness decision above was made from a snapshot taken
// before this lock was (re-)acquired; the registry read is an
// await boundary every sweep crosses with the lock released. A
// keepalive processed in that window refreshes
// `last_update_timestamp` under this same mutex, so re-validate
// liveness here and skip eviction if the worker heartbeated past
// the threshold. Kill-overdue evictions are intentionally exempt:
// keepalives from a worker that will not acknowledge a kill are
// exactly the wedged-worker symptom being handled.
if !*kill_overdue
&& inner
.workers
.peek(worker_id)
.is_some_and(|worker| worker.last_update_timestamp > timeout_threshold)
{
trace!(
?worker_id,
timeout_threshold,
"Worker heartbeated between staleness snapshot and eviction lock; keeping"
);
continue;
}
let err = if *kill_overdue {
make_err!(
Code::Internal,
Expand Down Expand Up @@ -3023,3 +3064,285 @@ mod peer_census_trusted_tests {
));
}
}

#[cfg(test)]
mod timedout_worker_eviction_race_tests {
use nativelink_metric::{
MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent,
};
use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::UpdateForWorker;

use super::*;
use crate::worker_registry::WorkerRegistry;

#[derive(Debug, Default)]
struct NoopWorkerStateManager;

impl MetricsComponent for NoopWorkerStateManager {
fn publish(
&self,
_kind: MetricKind,
_field_metadata: MetricFieldData,
) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
Ok(MetricPublishKnownKindData::Component)
}
}

#[async_trait]
impl WorkerStateManager for NoopWorkerStateManager {
async fn update_operation(
&self,
_operation_id: &OperationId,
_worker_id: &WorkerId,
_update: UpdateOperationType,
) -> Result<(), Error> {
Ok(())
}

async fn is_executing_on_worker(
&self,
_operation_id: &OperationId,
_worker_id: &WorkerId,
) -> Result<bool, Error> {
Ok(true)
}
}

/// repro: `remove_timedout_workers` evicts a worker whose keepalive
/// arrived between the liveness snapshot and the eviction lock.
///
/// The sweep snapshots `last_update_timestamp` under the inner lock,
/// drops the lock across the registry await, then re-locks and calls
/// `immediate_evict_worker` without re-checking the timestamp. A
/// keepalive that lands in that window (a real await boundary crossed by
/// every sweep) refreshes the worker under the same mutex, yet the
/// worker is still evicted. The test parks the sweep at that boundary
/// via a test-only gate, delivers the keepalive, and asserts the worker
/// survives; on current code the worker is evicted despite being
/// provably alive.
#[nativelink_macro::nativelink_test]
async fn keepalive_between_snapshot_and_evict_lock_still_evicts() {
const WORKER_TIMEOUT_S: u64 = 100;
const OLD_TS: WorkerTimestamp = 1_000;
const NOW_TS: WorkerTimestamp = 2_000;
// Fresh relative to NOW_TS: NOW_TS - FRESH_TS < WORKER_TIMEOUT_S.
const FRESH_TS: WorkerTimestamp = 1_999;

let registry = Arc::new(WorkerRegistry::new());
let scheduler = ApiWorkerScheduler::new(
Arc::new(NoopWorkerStateManager),
Arc::new(PlatformPropertyManager::new(HashMap::new())),
WorkerAllocationStrategy::default(),
None,
None,
None,
Arc::new(Notify::new()),
WORKER_TIMEOUT_S,
10,
0,
registry,
None,
false,
Duration::from_secs(15),
"test-scheduler".to_string(),
);
let gate = Arc::new((Notify::new(), Notify::new()));
scheduler
.test_evict_gate
.set(gate.clone())
.expect("gate set once");

let worker_id = WorkerId::from(String::from("worker1"));
// Keep rx alive so eviction's Disconnect send would succeed.
let (tx, _rx) = mpsc::channel::<UpdateForWorker>(16);
let worker = Worker::new(
worker_id.clone(),
PlatformProperties::default(),
tx,
OLD_TS,
1,
);
scheduler.add_worker(worker).await.expect("add worker");

// Start the sweep; it snapshots local_alive=false (OLD_TS is past
// the timeout), finds the registry stale too, and parks at the gate
// just before re-acquiring the inner lock to evict.
let sweep_scheduler = scheduler.clone();
let sweep = nativelink_util::background_spawn!("timedout_sweep", async move {
sweep_scheduler.remove_timedout_workers(NOW_TS).await
});
gate.0.notified().await;

// The worker's keepalive arrives now: it acquires the inner lock,
// advances last_update_timestamp to FRESH_TS, and updates the
// registry heartbeat. The worker is provably alive.
scheduler
.worker_keep_alive_received(&worker_id, FRESH_TS, None)
.await
.expect("keepalive");
assert!(scheduler.contains_worker_for_test(&worker_id).await);

// Let the sweep proceed to its eviction lock.
gate.1.notify_one();
sweep.await.expect("join").expect("sweep result");

// A correct sweep re-checks liveness under the lock and keeps the
// worker; current code evicts it unconditionally.
assert!(
scheduler.contains_worker_for_test(&worker_id).await,
"worker was evicted by remove_timedout_workers even though its \
keepalive (timestamp {FRESH_TS}, well within the {WORKER_TIMEOUT_S}s \
timeout at now={NOW_TS}) was fully processed before the eviction \
lock was taken"
);
}
}

#[cfg(test)]
mod keepalive_eviction_race_tests {
//! Deterministic reproduction of the check-then-act race in
//! `remove_timedout_workers`: a keepalive that lands after the sweep's
//! unlocked registry check but before it re-acquires the inner lock is
//! never re-consulted, so a demonstrably live worker is evicted.

use nativelink_config::schedulers::WorkerAllocationStrategy;

use super::{
ApiWorkerScheduler, Arc, Duration, Error, HashMap, MetricFieldData, MetricKind,
MetricPublishKnownKindData, MetricsComponent, Notify, OperationId, PlatformProperties,
UNIX_EPOCH, UpdateOperationType, Worker, WorkerId, WorkerScheduler, WorkerStateManager,
async_trait, mpsc,
};
use crate::platform_property_manager::PlatformPropertyManager;
use crate::worker_registry::WorkerRegistry;

#[derive(Debug)]
struct NoopWorkerStateManager;

impl MetricsComponent for NoopWorkerStateManager {
fn publish(
&self,
_kind: MetricKind,
_field_metadata: MetricFieldData,
) -> Result<MetricPublishKnownKindData, nativelink_metric::Error> {
Ok(MetricPublishKnownKindData::Component)
}
}

#[async_trait]
impl WorkerStateManager for NoopWorkerStateManager {
async fn update_operation(
&self,
_operation_id: &OperationId,
_worker_id: &WorkerId,
_update: UpdateOperationType,
) -> Result<(), Error> {
Ok(())
}

async fn is_executing_on_worker(
&self,
_operation_id: &OperationId,
_worker_id: &WorkerId,
) -> Result<bool, Error> {
Ok(true)
}
}

const WORKER_TIMEOUT_S: u64 = 100;
/// Worker registered (locally and in the registry) at t=0.
const REGISTERED_AT: u64 = 0;
/// The sweep and the keepalive both happen at t=200, i.e. one full
/// timeout past expiry, so both liveness sources initially say "dead".
const NOW: u64 = 200;

#[nativelink_macro::nativelink_test]
async fn keepalive_between_registry_check_and_eviction_lock_is_ignored() {
let registry = Arc::new(WorkerRegistry::new());
let scheduler = ApiWorkerScheduler::new(
Arc::new(NoopWorkerStateManager),
Arc::new(PlatformPropertyManager::new(HashMap::new())),
WorkerAllocationStrategy::default(),
None,
None,
None,
Arc::new(Notify::new()),
WORKER_TIMEOUT_S,
60, // unacknowledged_kill_timeout_s
0, // dispatch_ack_timeout_s
registry.clone(),
None,
false, // has_peers
Duration::from_secs(15), // record_ttl
"test-scheduler".to_string(),
);

let worker_id = WorkerId("stale_then_alive_worker".to_string());
let (tx, _rx) = mpsc::channel(64);
scheduler
.add_worker(Worker::new(
worker_id.clone(),
PlatformProperties::default(),
tx,
REGISTERED_AT,
4,
))
.await
.expect("add_worker failed");

// Gate 1: hold the registry lock so the sweep, after snapshotting the
// (stale) local state and RELEASING the inner lock, parks at its
// unlocked `is_worker_alive` await.
let registry_guard = registry.write_lock_for_test().await;

let sweep = nativelink_util::background_spawn!("timedout_sweep", {
let scheduler = scheduler.clone();
async move { scheduler.remove_timedout_workers(NOW).await }
});
// Let the sweep run: it takes the inner lock, snapshots the worker as
// locally stale, releases the lock, and parks on the registry read.
for _ in 0..8 {
tokio::task::yield_now().await;
}

// Gate 2: take the inner lock (currently free - the sweep released it)
// so that after its registry read the sweep parks again before it can
// evict. This models inner-lock contention on a busy scheduler.
let mut inner_guard = scheduler.inner.lock().await;

// Release gate 1: the sweep reads the registry (heartbeat still t=0,
// so "dead"), queues the worker for eviction, and parks on the inner
// lock we hold.
drop(registry_guard);
for _ in 0..8 {
tokio::task::yield_now().await;
}

// The keepalive lands NOW, while the sweep is parked awaiting the
// inner lock. These are exactly the two steps of
// `worker_keep_alive_received` -> `refresh_worker`: refresh the local
// lifetime under the inner lock, then update the registry heartbeat.
inner_guard
.refresh_lifetime(&worker_id, NOW, None, true)
.expect("refresh_lifetime failed");
registry
.update_worker_heartbeat(&worker_id, UNIX_EPOCH + Duration::from_secs(NOW))
.await;

// Release gate 2: the sweep re-acquires the inner lock and applies its
// stale decision without re-checking either liveness source.
drop(inner_guard);
sweep
.await
.expect("sweep panicked")
.expect("remove_timedout_workers failed");

// The worker heartbeated at t=NOW - both the local timestamp and the
// registry say it is alive - yet the stale sweep evicted it.
assert!(
scheduler.contains_worker_for_test(&worker_id).await,
"live worker (keepalive at t={NOW}, timeout {WORKER_TIMEOUT_S}s) \
was evicted by a stale timeout decision"
);
}
}
9 changes: 9 additions & 0 deletions nativelink-scheduler/src/worker_registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,15 @@ impl WorkerRegistry {
self.check_liveness(worker_id, timeout, now).await == WorkerLiveness::Alive
}

/// Test-only: hold the internal write lock so that concurrent liveness
/// reads park at a deterministic await point.
#[cfg(test)]
pub(crate) async fn write_lock_for_test(
&self,
) -> async_lock::RwLockWriteGuard<'_, HashMap<WorkerId, SystemTime>> {
self.workers.write().await
}

pub async fn get_worker_last_seen(&self, worker_id: &WorkerId) -> Option<SystemTime> {
let workers = self.workers.read().await;
workers.get(worker_id).copied()
Expand Down
Loading