Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/app/features/ingestion/job_health.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
"staging": (45, 60),
"archive": (1560, 720),
"diagnostics": (420, 300),
"refresh": (1020, 60),
"log_archive": (10200, 120),
}
UPLOAD_FAILURE_CATEGORIES: dict[str, ScheduledJobUploadFailureCategory] = {
**dict.fromkeys(["401", "403"], ScheduledJobUploadFailureCategory.AUTH),
Expand Down
5 changes: 4 additions & 1 deletion backend/app/features/ingestion/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@
SCHEDULED_JOB_ID_PATTERN = r"^[a-z0-9][a-z0-9_-]{0,99}$"
# One year keeps derived deadlines within datetime range.
SCHEDULED_JOB_MAX_THRESHOLD_MINUTES = 525_600
ScheduledJobScanMode = Literal["staging", "archive", "diagnostics"]
# Scan modes plus the site maintenance jobs that share the crontab.
ScheduledJobScanMode = Literal[
"staging", "archive", "diagnostics", "refresh", "log_archive"
]
UPLOAD_FAILURE_CASES_MAX = 50
UPLOAD_FAILURE_MESSAGE_MAX_LENGTH = 200
# Failure counts are keyed by HTTP status, or "network" for errors and timeouts.
Expand Down
2 changes: 1 addition & 1 deletion backend/app/scripts/ingestion/sites/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ remote-site ingestion jobs.
| `site_ingestion_launcher.sh` | Stable scheduler entrypoint. Loads one reviewed site config and starts the configured Python ingestor. |
| `configs/` | Reviewed site-specific defaults such as archive roots, machine name, ingestor module, and archive lower bound. Never store tokens or deployment paths here. |
| `templates/` | Committed templates for deployment-local API environment files and scheduler crontabs. |
| `operations/` | Helpers used by the `make ops-*` targets to provision or refresh a checkout and create protected environment files, plus weekly launcher-log archiving. |
| `operations/` | Helpers used by the `make ops-*` targets to provision or refresh a checkout and create protected environment files, plus weekly launcher-log archiving and `run_reported_job.sh`, which reports these cron jobs' health. |
| `diagnostics/` | Wrappers for diagnostics operations that are separate from performance ingestion. |

`SIMBOARD_ROOT`, API tokens, and installed crontabs are deployment-local. The
Expand Down
92 changes: 92 additions & 0 deletions backend/app/scripts/ingestion/sites/operations/run_reported_job.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#!/usr/bin/env bash
# Run a site maintenance command once and report it to scheduled job health.
#
# Usage: run_reported_job.sh <site> <job-type> <command> [args...]
#
# Live runs (DRY_RUN=false) that set SIMBOARD_JOB_ID report their start and
# finish to each API environment in SIMBOARD_ENV_FILES, a colon-separated list
# of env files. Only the report processes load the site config and env files,
# so the command never receives an API token. Reporting is best effort: a
# missing file or interpreter, or an unreachable API, is logged and never stops
# the command or changes its exit code.

warn() {
printf '[%s] run_reported_job: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$1" >&2
}

# Send one start or finish report to every API environment.
report_job() {
local env_file
local -a env_files
IFS=: read -r -a env_files <<< "${SIMBOARD_ENV_FILES}"
for env_file in "${env_files[@]}"; do
[[ -n "${env_file}" ]] || continue
if [[ ! -r "${env_file}" ]]; then
warn "API environment not readable; skipping ${1} report: ${env_file}"
continue
fi
# Silence both files: shell diagnostics can echo the token.
(
source "${site_config}" > /dev/null 2>&1
source "${env_file}" > /dev/null 2>&1 || exit 1
export MACHINE_NAME SIMBOARD_JOB_ID SIMBOARD_JOB_RUN_ID
export SIMBOARD_JOB_STARTED_AT SIMBOARD_JOB_SCAN_MODE
cd "${backend_dir}" && "${python_bin}" -m app.scripts.ingestion.job_health "$@"
) || warn "${1} report not sent: ${env_file}"
done
}

main() {
if (( $# < 3 )) || [[ ! $1 =~ ^[abcdefghijklmnopqrstuvwxyz0123456789_-]+$ ]]; then
echo "Usage: $0 <site> <job-type> <command> [args...]" >&2
return 1
fi

local site=$1
SIMBOARD_JOB_SCAN_MODE=$2
shift 2

local script_dir
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
backend_dir="$(cd -- "${script_dir}/../../../../.." && pwd)"
python_bin="${backend_dir}/.venv/bin/python"
# The site config provides MACHINE_NAME, as it does for the launcher.
site_config="${SIMBOARD_SITE_CONFIG:-${script_dir}/../configs/${site}.config}"

local reporting=false
shopt -s nocasematch
case "${DRY_RUN:-true}" in
0|false|no|off)
[[ -z "${SIMBOARD_JOB_ID:-}" || -z "${SIMBOARD_ENV_FILES:-}" ]] || reporting=true
;;
esac
shopt -u nocasematch

if [[ $reporting == true && ! -r "${site_config}" ]]; then
warn "site configuration not readable; running without job health reports: ${site_config}"
reporting=false
fi
if [[ $reporting == true ]] && ! SIMBOARD_JOB_RUN_ID="$("${python_bin}" -c 'import uuid; print(uuid.uuid4())' 2> /dev/null)"; then
warn "${python_bin} is unavailable; running without job health reports"
reporting=false
fi

if [[ $reporting == true ]]; then
SIMBOARD_JOB_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
report_job start
fi

local exit_code=0
"$@" || exit_code=$?

# A refresh may have replaced the reporter; the new one sends the finish.
if [[ $reporting == true ]]; then
report_job finish --exit-code "${exit_code}"
fi

return "${exit_code}"
}

# Keep the call and exit on one line: Bash parses the whole line before running
# it, so it never reads this file again after a refresh replaces it.
main "$@"; exit $?
17 changes: 12 additions & 5 deletions backend/app/scripts/ingestion/sites/templates/crontab.example
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,13 @@ SIMBOARD_ROOT=/path/to/simboard_root
# (5 PM and 8 AM Pacific daylight time; 4 PM and 7 AM standard time). The
# refresh installs backend dependencies only when the revision changed and uses
# `flock` for a process lock when it is available; review
# simboard-ingestion-provision.log after each update.
0 0,15 * * * PATH="$HOME/.local/bin:$PATH" "${SIMBOARD_ROOT}/repository/simboard/backend/app/scripts/ingestion/sites/operations/refresh_repository.sh" >> "${SIMBOARD_ROOT}/operations/simboard-ingestion-provision.log" 2>&1
# simboard-ingestion-provision.log after each update. Replace `chrysalis` with
# the site-config name. run_reported_job.sh runs the refresh once and reports
# it as the `refresh` job to each API environment in SIMBOARD_ENV_FILES; a
# missing or unreachable environment is skipped and never stops the refresh or
# changes its exit code. The refresh has no dry-run mode, so this entry always
# sets DRY_RUN=false.
0 0,15 * * * PATH="$HOME/.local/bin:$PATH" DRY_RUN=false SIMBOARD_ENV_FILES="${SIMBOARD_ROOT}/operations/env.dev.sh:${SIMBOARD_ROOT}/operations/env.prod.sh" SIMBOARD_JOB_ID=chrysalis-refresh "${SIMBOARD_ROOT}/repository/simboard/backend/app/scripts/ingestion/sites/operations/run_reported_job.sh" chrysalis refresh "${SIMBOARD_ROOT}/repository/simboard/backend/app/scripts/ingestion/sites/operations/refresh_repository.sh" >> "${SIMBOARD_ROOT}/operations/simboard-ingestion-provision.log" 2>&1

# =============================================================================
# API environment and run mode
Expand Down Expand Up @@ -91,7 +96,9 @@ SIMBOARD_ROOT=/path/to/simboard_root
# =============================================================================
# Run weekly at 01:00 UTC on Sunday, offset from the repository refresh.
# Inherit DRY_RUN: true or unset only previews; false archives completed weeks
# and deletes verified, unchanged raw logs. No API credentials are needed.
# Keep at least seven days of raw logs; optionally override this default below.
# and deletes verified, unchanged raw logs. Archiving needs no API credentials;
# the API environments are used only to report live runs as the `log_archive`
# job. Keep at least seven days of raw logs; optionally override
# this default below.
# LOG_RETENTION_DAYS=7
0 1 * * 0 "${SIMBOARD_ROOT}/repository/simboard/backend/.venv/bin/python" "${SIMBOARD_ROOT}/repository/simboard/backend/app/scripts/ingestion/sites/operations/archive_ingestion_logs.py" >> "${SIMBOARD_ROOT}/operations/simboard-log-archiving.log" 2>&1
0 1 * * 0 SIMBOARD_ENV_FILES="${SIMBOARD_ROOT}/operations/env.dev.sh:${SIMBOARD_ROOT}/operations/env.prod.sh" SIMBOARD_JOB_ID=chrysalis-log-archive "${SIMBOARD_ROOT}/repository/simboard/backend/app/scripts/ingestion/sites/operations/run_reported_job.sh" chrysalis log_archive "${SIMBOARD_ROOT}/repository/simboard/backend/.venv/bin/python" "${SIMBOARD_ROOT}/repository/simboard/backend/app/scripts/ingestion/sites/operations/archive_ingestion_logs.py" >> "${SIMBOARD_ROOT}/operations/simboard-log-archiving.log" 2>&1
8 changes: 7 additions & 1 deletion backend/tests/features/ingestion/test_job_health_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,13 @@ def test_only_the_bound_service_account_reports_runs(client, users, act_as, job,
@pytest.mark.parametrize("action", ["start", "finish"])
@pytest.mark.parametrize(
("scan_mode", "thresholds"),
[("staging", (45, 60)), ("archive", (1560, 720)), ("diagnostics", (420, 300))],
[
("staging", (45, 60)),
("archive", (1560, 720)),
("diagnostics", (420, 300)),
("refresh", (1020, 60)),
("log_archive", (10200, 120)),
],
)
def test_first_report_registers_the_job(
client, db, users, act_as, action, scan_mode, thresholds
Expand Down
Loading
Loading