Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions backend/app/scripts/ingestion/archive_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
_find_spec_matches,
)
from app.scripts.ingestion.archive_ingestor_core import (
ARCHIVE_ROOT_BASENAME,
DISCOVERY_PROGRESS_LOG_EVERY_DIRECTORIES,
STAGING_ROOT_BASENAME,
ArchiveSnapshotScan,
Expand Down Expand Up @@ -51,6 +52,40 @@
from app.scripts.ingestion.archive_logging import scan_finished

EXECUTION_DIR_PATTERN = re.compile(r"\d+\.\d+-\d+$")
# PACE leaves these under a staging root: processed batches waiting to move,
# upload work trees, and, on sites such as Aurora, the archive root itself.
STAGING_EXCLUDED_DIR_PREFIXES = ("performance_archive_", "pace-exps-")
STAGING_EXCLUDED_DIR_NAMES = frozenset({ARCHIVE_ROOT_BASENAME})


def _prune_staging_pace_dirnames(
dirpath: str, dirnames: list[str], *, archive_root: Path
) -> None:
"""Skip PACE batches, upload work trees, and a nested archive root."""
if Path(dirpath) != archive_root:
return
dirnames[:] = [
name
for name in dirnames
if not name.startswith(STAGING_EXCLUDED_DIR_PREFIXES)
and name not in STAGING_EXCLUDED_DIR_NAMES
]


def _prune_staging_walk_dirnames(
dirpath: str, dirnames: list[str], *, archive_root: Path
) -> None:
"""Limit staging walks to ``username/case/execution`` directories."""
_prune_staging_pace_dirnames(dirpath, dirnames, archive_root=archive_root)
depth = len(Path(dirpath).relative_to(archive_root).parts)
if depth < 2:
dirnames[:] = [
name for name in dirnames if not EXECUTION_DIR_PATTERN.fullmatch(name)
]
else:
dirnames[:] = [
name for name in dirnames if EXECUTION_DIR_PATTERN.fullmatch(name)
]


@dataclass(frozen=True)
Expand Down
2 changes: 1 addition & 1 deletion backend/app/scripts/ingestion/backfill/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
_timeout_request_error,
_url_request_error,
)
from app.scripts.ingestion.archive_discovery import _prune_staging_walk_dirnames
from app.scripts.ingestion.archive_ingestor_core import (
DISCOVERY_PROGRESS_LOG_EVERY_DIRECTORIES,
DISCOVERY_RESULT_BATCH_SIZE,
Expand All @@ -46,7 +47,6 @@
)
from app.scripts.ingestion.archive_ingestor_core import logger as ingestion_logger
from app.scripts.ingestion.archive_layout import _filter_archive_walk_dirnames
from app.scripts.ingestion.nersc_archive_ingestor import _prune_staging_walk_dirnames
from app.scripts.ingestion.v3_data.lcrc_v3_archive_ingestor import (
CHRYSALIS_ARCHIVE_ROOT,
V3_ARCHIVE_YEAR_START,
Expand Down
17 changes: 16 additions & 1 deletion backend/app/scripts/ingestion/hpc_upload_archive_ingestor.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import uuid
from collections.abc import Iterator
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Any, Callable

Expand All @@ -53,7 +54,10 @@
_timeout_request_error,
_url_request_error,
)
from app.scripts.ingestion.archive_discovery import _scan_archive
from app.scripts.ingestion.archive_discovery import (
_prune_staging_pace_dirnames,
_scan_archive,
)
from app.scripts.ingestion.archive_ingestor_core import (
ArchiveCheckpointPersistenceCallback,
CaseSubmissionCallback,
Expand Down Expand Up @@ -118,6 +122,16 @@ def _case_submission_callback(
)


def _default_dir_pruner(
config: IngestorConfig,
additional_dir_pruner: Callable[[str, list[str]], None] | None,
) -> Callable[[str, list[str]], None] | None:
"""Skip PACE-owned staging trees unless the caller supplies its own pruner."""
if additional_dir_pruner is not None or config.scan_mode != "staging":
return additional_dir_pruner
return partial(_prune_staging_pace_dirnames, archive_root=config.archive_root)


def _prepare_run_state(
config: IngestorConfig,
archive_checkpointing: bool = True,
Expand Down Expand Up @@ -205,6 +219,7 @@ def _run_ingestor(
"case_simulation_type requires the built-in HPC upload transport"
)
post_request_fn = _case_submission_callback(post_request_fn)
additional_dir_pruner = _default_dir_pruner(config, additional_dir_pruner)

if not _validate_run_preconditions(config, log_event_fn=_log_event):
return 1
Expand Down
26 changes: 4 additions & 22 deletions backend/app/scripts/ingestion/nersc_archive_ingestor.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@
_fetch_ingestion_state,
_post_ingestion_request,
)
from app.scripts.ingestion.archive_discovery import EXECUTION_DIR_PATTERN, _scan_archive
from app.scripts.ingestion.archive_discovery import (
_prune_staging_walk_dirnames,
_scan_archive,
)
from app.scripts.ingestion.archive_ingestor_core import (
ArchiveCheckpointPersistenceCallback,
CaseSubmissionCallback,
Expand Down Expand Up @@ -106,27 +109,6 @@ def _case_submission_callback(
return _post_ingestion_request if post_request_fn is None else post_request_fn


def _prune_staging_walk_dirnames(
dirpath: str, dirnames: list[str], *, archive_root: Path
) -> None:
"""Limit NERSC staging walks to username/case/execution directories."""
depth = len(Path(dirpath).relative_to(archive_root).parts)
if depth == 0:
dirnames[:] = [
name
for name in dirnames
if not name.startswith(("performance_archive_", "pace-exps-"))
]
if depth < 2:
dirnames[:] = [
name for name in dirnames if not EXECUTION_DIR_PATTERN.fullmatch(name)
]
else:
dirnames[:] = [
name for name in dirnames if EXECUTION_DIR_PATTERN.fullmatch(name)
]


def _prepare_run_state(
config: IngestorConfig,
) -> tuple[dict[str, Any], set[str], str] | None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import tarfile
import urllib.error
import urllib.request
from dataclasses import replace
from email.message import Message
from pathlib import Path
from types import SimpleNamespace
Expand Down Expand Up @@ -33,6 +34,7 @@
from app.scripts.ingestion.hpc_upload_archive_ingestor import (
_build_endpoint_url,
_create_case_archive,
_default_dir_pruner,
_post_hpc_upload_ingestion_request,
_prepared_hpc_case_submission,
_run_ingestor,
Expand Down Expand Up @@ -787,6 +789,80 @@ def fake_log_event(event: str, fields: dict[str, object] | None = None) -> None:
assert "accepted_execution_ids" not in scan_completed


def test_run_ingestor_staging_skips_pace_batches_and_nested_archive_root(
tmp_path: Path,
monkeypatch,
) -> None:
archive_root = tmp_path / "performance_archive"
for case_dir in (
archive_root / "user_a" / "case_a",
archive_root / "performance_archive_2026_10_15_10_03_11" / "user_a" / "case_b",
archive_root / "pace-exps-test" / "user_a" / "case_c",
archive_root / "OLD_PERF" / "2026-10" / "user_a" / "case_d",
):
(case_dir / "100.1-1").mkdir(parents=True)
logged_events: list[tuple[str, dict[str, object]]] = []

def fake_log_event(event: str, fields: dict[str, object] | None = None) -> None:
logged_events.append((event, {} if fields is None else fields))

monkeypatch.setattr(upload_ingestor_module, "_log_event", fake_log_event)
monkeypatch.setattr(discovery_module, "_log_event", fake_log_event)
monkeypatch.setattr(
upload_ingestor_module,
"_fetch_ingestion_state",
lambda *args, **kwargs: _fresh_state(),
)

config = IngestorConfig(
api_base_url="http://backend:8000",
api_token="token",
archive_root=archive_root,
machine_name="compy",
dry_run=True,
max_cases_per_run=None,
max_attempts=1,
request_timeout_seconds=30,
scan_mode="staging",
)

exit_code = _run_ingestor(
config,
metadata_locator=lambda *_: {},
sleep_fn=lambda *_: None,
)

scan_completed = [
fields for event, fields in logged_events if event == "scan_completed"
][0]

assert exit_code == 0
assert scan_completed["execution_dirs_scanned"] == 1
assert scan_completed["submission_qualified_cases"] == 1


def test_default_dir_pruner_keeps_caller_pruner_and_archive_scans(
tmp_path: Path,
) -> None:
config = IngestorConfig(
api_base_url="http://backend:8000",
api_token="token",
archive_root=tmp_path,
machine_name="compy",
dry_run=True,
max_cases_per_run=None,
max_attempts=1,
request_timeout_seconds=30,
scan_mode="staging",
)

def caller_pruner(dirpath: str, dirnames: list[str]) -> None:
del dirpath, dirnames

assert _default_dir_pruner(config, caller_pruner) is caller_pruner
assert _default_dir_pruner(replace(config, scan_mode="archive"), None) is None


def test_run_ingestor_missing_archive_root_returns_failure(
monkeypatch, tmp_path: Path
) -> None:
Expand Down
48 changes: 46 additions & 2 deletions backend/tests/features/ingestion/test_nersc_archive_ingestor.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,47 @@ def forbidden(*args, **kwargs):
)


@pytest.mark.parametrize(
"relative",
[
"performance_archive_2026_10_12_10_02_55/user/case/100.1-1",
"pace-exps-test/user/case/100.1-1",
"OLD_PERF/2026-10/user/case/100.1-1",
],
)
def test_upload_single_execution_rejects_staging_exclusions(
tmp_path, monkeypatch, relative
) -> None:
root = tmp_path / "performance_archive"
execution = root / relative
execution.mkdir(parents=True)
monkeypatch.setattr(
upload_module, "_fetch_ingestion_state", lambda *a, **kw: _fresh_state()
)
config = IngestorConfig(
api_base_url="https://example.test",
api_token="test-token",
archive_root=root,
machine_name="chrysalis",
dry_run=False,
max_cases_per_run=None,
max_attempts=1,
request_timeout_seconds=30,
scan_mode="staging",
execution_dir=execution,
)

def forbidden(*args, **kwargs):
pytest.fail("Excluded selections must not validate or submit executions")

assert (
upload_module._run_ingestor(
config, metadata_locator=forbidden, post_request_fn=forbidden
)
== 1
)


@pytest.mark.parametrize("runner", [ingestor_module, upload_module])
@pytest.mark.parametrize("rejection", [IncompleteArchiveError, ArchiveValidationError])
@pytest.mark.parametrize("move_to_archive", [False, True])
Expand Down Expand Up @@ -1970,11 +2011,14 @@ def test_staging_scan_excludes_batches_and_unrelated_layouts(
for case in case_paths
}
batch = root / "performance_archive_2026_10_15_10_03_11"
excluded_roots = [batch, root / "pace-exps-test"]
excluded_roots = [batch, root / "pace-exps-test", root / "OLD_PERF"]
excluded_cases = [
batch / status / "ndk" / "archived-case"
for status in ("COMPLETED", "FAILED", "NODE_FAIL", "TIMEOUT", "UNKNOWN")
] + [root / "pace-exps-test" / "experiment-case"]
] + [
root / "pace-exps-test" / "experiment-case",
root / "OLD_PERF" / "2026-10" / "ndk" / "archived-case",
]
for case in case_paths + excluded_cases:
(case / execution_id).mkdir(parents=True)
nested = case_paths[0] / "unrelated" / "nested-case"
Expand Down
6 changes: 4 additions & 2 deletions docs/architecture/metadata-ingestion.md
Original file line number Diff line number Diff line change
Expand Up @@ -478,8 +478,10 @@ rejects machines absent from its machine catalog; missing machine metadata is no
replaced with the scanner's configured machine.

NERSC staging scans only discover `<root>/<username>/<case>/<execution_id>`.
They exclude top-level `performance_archive_*` batches and `pace-exps-*`
directories and do not recurse into other layouts below a case directory.
They exclude top-level `performance_archive_*` batches, `pace-exps-*`
directories, and an `OLD_PERF` archive root nested in staging, and do not
recurse into other layouts below a case directory. Remote-upload staging scans
apply the same top-level exclusions.
Processed batches temporarily remaining under the staging root are ignored;
archive scans discover them after they move to `OLD_PERF_ARCHIVE_ROOT/YYYY-MM/`,
subject to the existing archive range and status rules.
Expand Down
Loading