From 8c0975ffba813604395de45a960bd5d08592ba6f Mon Sep 17 00:00:00 2001 From: jackaldenryan Date: Sat, 29 Aug 2026 22:36:55 -0500 Subject: [PATCH 1/2] feat(ingestion): attach Zep document_id to files, emails, and Slack (v0.4.0) Group related episodes so extraction can resolve pronouns across chunks and Slack threads, and add a non-blocking post-merge/release smoke suite. Co-authored-by: Cursor --- .github/workflows/smoke-ingestion.yml | 44 ++ .github/workflows/test-ingestion.yml | 2 +- ingestion/CHANGELOG.md | 18 + ingestion/examples/README.md | 2 +- ingestion/examples/slack_export_example.py | 5 +- ingestion/pyproject.toml | 5 +- ingestion/scripts/release_smoke_prod.py | 374 +---------------- ingestion/scripts/smoke.py | 390 ++++++++++++++++++ ingestion/src/zep_ingest/_graph_api.py | 74 ++++ ingestion/src/zep_ingest/documents.py | 84 ++++ ingestion/src/zep_ingest/loaders/email.py | 2 + .../src/zep_ingest/loaders/json_records.py | 7 +- ingestion/src/zep_ingest/loaders/slack.py | 65 +-- ingestion/src/zep_ingest/loaders/text.py | 2 + .../src/zep_ingest/loaders/transcript.py | 3 + ingestion/src/zep_ingest/pipeline.py | 10 +- ingestion/src/zep_ingest/result.py | 72 +++- .../src/zep_ingest/submitters/sequential.py | 20 +- .../zep_ingest/transforms/canonicalizer.py | 1 + .../src/zep_ingest/transforms/chunker.py | 1 + .../zep_ingest/transforms/contextualizer.py | 1 + ingestion/src/zep_ingest/transforms/limits.py | 1 + ingestion/src/zep_ingest/types.py | 34 +- ingestion/tests/test_document_poll.py | 32 ++ ingestion/tests/test_documents.py | 87 ++++ ingestion/tests/test_pipeline.py | 7 +- ingestion/tests/test_slack_loader.py | 142 ++++--- ingestion/tests/test_smoke.py | 25 ++ ingestion/tests/test_types.py | 5 +- 29 files changed, 1024 insertions(+), 491 deletions(-) create mode 100644 .github/workflows/smoke-ingestion.yml create mode 100644 ingestion/scripts/smoke.py create mode 100644 ingestion/src/zep_ingest/_graph_api.py create mode 100644 ingestion/src/zep_ingest/documents.py create mode 100644 ingestion/tests/test_document_poll.py create mode 100644 ingestion/tests/test_documents.py create mode 100644 ingestion/tests/test_smoke.py diff --git a/.github/workflows/smoke-ingestion.yml b/.github/workflows/smoke-ingestion.yml new file mode 100644 index 00000000..e7e053e7 --- /dev/null +++ b/.github/workflows/smoke-ingestion.yml @@ -0,0 +1,44 @@ +name: Smoke Ingestion Package + +on: + push: + branches: [main] + paths: + - 'ingestion/**' + - '.github/workflows/smoke-ingestion.yml' + workflow_run: + workflows: [Release Ingestion Package] + types: [completed] + +permissions: + contents: read + +jobs: + smoke: + if: >- + github.event_name == 'push' || + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') + runs-on: ubuntu-latest + continue-on-error: true + environment: integration + defaults: + run: + working-directory: ingestion + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install uv + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 + with: + version: "0.7.19" + + - name: Set up Python + run: uv python install 3.13 + + - name: Install dependencies + run: uv sync --extra dev + + - name: Run live smoke tests + run: uv run python scripts/smoke.py + env: + ZEP_API_KEY: ${{ secrets.ZEP_API_KEY }} diff --git a/.github/workflows/test-ingestion.yml b/.github/workflows/test-ingestion.yml index a580b100..40d0e5ea 100644 --- a/.github/workflows/test-ingestion.yml +++ b/.github/workflows/test-ingestion.yml @@ -61,7 +61,7 @@ jobs: name: ingestion-coverage integration: - if: github.event_name == 'push' + if: false # superseded by smoke-ingestion.yml (non-blocking post-merge smoke) runs-on: ubuntu-latest environment: integration defaults: diff --git a/ingestion/CHANGELOG.md b/ingestion/CHANGELOG.md index 4dc1c1fd..73175acb 100644 --- a/ingestion/CHANGELOG.md +++ b/ingestion/CHANGELOG.md @@ -4,6 +4,24 @@ All notable changes to `zep-ingest` are documented here. The project follows [Semantic Versioning](https://semver.org); while at `0.x` the public API may still change between minor versions. +## 0.4.0 + +- **Document grouping (`document_id`).** Text files, transcripts, and emails + always get a `document_id` (even as a single episode). Slack is ingested as + one episode per message with a shared `document_id` per Slack thread, plus a + second document of each channel's top-level messages in order (thread parents + are ingested twice; payload identical except `document_id`). Not a Zep + `thread_id`. The Slack `grouping` option is removed. JSON records omit + `document_id`. `TextChunker` preserves `document_id` across chunks. +- Sequential `graph.add` sends `document_id` via a thin API wrapper until the + installed `zep-cloud` SDK exposes the field. Batch items pass it through + `BatchAddItem` extras. +- `wait()` polls one tail episode per `document_id` saga (like multi-thread + polling). Ingestion order within a document follows `created_at`. +- Unified live smoke suite: `ingestion/scripts/smoke.py` (also + `pytest -m smoke`). Runs post-merge and post-release via + `.github/workflows/smoke-ingestion.yml` with `continue-on-error: true`. + ## 0.3.0 - **Submit everything, then wait once.** Multiple files or loaders destined for diff --git a/ingestion/examples/README.md b/ingestion/examples/README.md index 3330c447..e0050d04 100644 --- a/ingestion/examples/README.md +++ b/ingestion/examples/README.md @@ -24,7 +24,7 @@ the [dashboard](https://app.getzep.com) when you're done. | [`documents_example.py`](documents_example.py) | Markdown → ~500-char chunks; optional LLM contextualization (auto-detects `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`) | named graph | | [`fact_triples_example.py`](fact_triples_example.py) | molding a realistic directory export (no triple-shaped columns) into explicit fact triples; the manual create → set_ontology → seed lifecycle | named graph | | [`json_records_example.py`](json_records_example.py) | structured records with identity-field mapping — Zep extracts the relationships | named graph | -| [`slack_export_example.py`](slack_export_example.py) | free `preview()` first, then a Slack export with thread grouping, `skip_subtypes`, and the opt-in risky-alias guard | named graph | +| [`slack_export_example.py`](slack_export_example.py) | free `preview()` first, then a Slack export (one episode per message, thread as `document_id`), `skip_subtypes`, and the opt-in risky-alias guard | named graph | | [`user_graph_example.py`](user_graph_example.py) | **combined**: profile fact triples → chat-thread backfill → a document, all on one user's graph | user graph | | [`thread_backfill_example.py`](thread_backfill_example.py) | historic chat history (JSONL) → threads that power `thread.get_user_context()` | user graph | diff --git a/ingestion/examples/slack_export_example.py b/ingestion/examples/slack_export_example.py index c449a1f7..2c3e76fa 100644 --- a/ingestion/examples/slack_export_example.py +++ b/ingestion/examples/slack_export_example.py @@ -2,8 +2,9 @@ Self-contained and re-runnable: previews then ingests the bundled sample export under examples/data/slack_export/, creating a fresh graph and setting -the starter ontology first. Messages are grouped by thread, join/leave noise -is skipped, and each episode keeps its real Slack timestamp. +the starter ontology first. Each message is one episode; Slack threads share a +``document_id``. Join/leave noise is skipped, and each episode keeps its real +Slack timestamp. To run against real data, get an export from Slack (Workspace Admin → Settings & administration → Workspace settings → Import/Export Data → diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index 6ff9afd6..1f97d314 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "zep-ingest" -version = "0.3.0" +version = "0.4.0" description = "Bulk data ingestion pipeline for Zep: chunk, contextualize, canonicalize, and submit unstructured and structured data" readme = "README.md" requires-python = ">=3.11" @@ -57,7 +57,8 @@ python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] markers = [ - "integration: marks tests as integration tests (deselect with '-m \"not integration\"')" + "integration: marks tests as integration tests (deselect with '-m \"not integration\"')", + "smoke: live smoke tests against a real Zep account", ] [tool.ruff] diff --git a/ingestion/scripts/release_smoke_prod.py b/ingestion/scripts/release_smoke_prod.py index 3958f077..2f0169d9 100644 --- a/ingestion/scripts/release_smoke_prod.py +++ b/ingestion/scripts/release_smoke_prod.py @@ -1,377 +1,9 @@ -"""Production smoke tests for zep-ingest 0.3.0 release candidates. - -Exercises submit-all-then-wait-once, tail polling, thread message UUIDs, -multi-thread sagas, and phased node seeding. Requires ZEP_API_KEY (and -ZEP_API_URL for zep-cloud). Run via KeyBank: - - keybank run zep-prod -- env ZEP_API_URL=https://api.getzep.com \\ - uv run --directory ingestion python scripts/release_smoke_prod.py -""" +"""Backward-compatible entry point — use scripts/smoke.py instead.""" from __future__ import annotations -import json -import sys -import tempfile -import time -import uuid -from collections import Counter -from dataclasses import dataclass, field +import runpy from pathlib import Path -from typing import Any - -from zep_cloud.client import Zep -from zep_cloud.types import Message - -from zep_ingest import Episode, NodeItem, ingest, ingest_json_records, ingest_nodes -from zep_ingest.exceptions import BatchUnavailableError -from zep_ingest.threads import ThreadMessage, ingest_thread_messages - - -@dataclass -class PollStats: - batch_get: int = 0 - episode_get: Counter[str] = field(default_factory=Counter) - task_get: int = 0 - - -def instrument_client(client: Zep) -> PollStats: - stats = PollStats() - orig_batch_get = client.batch.get - orig_episode_get = client.graph.episode.get - orig_task_get = client.task.get - - def batch_get(batch_id: str, *args: Any, **kwargs: Any): - stats.batch_get += 1 - return orig_batch_get(batch_id, *args, **kwargs) - - def episode_get(*args: Any, **kwargs: Any): - uuid_ = kwargs.get("uuid_") or (args[0] if args else "?") - stats.episode_get[str(uuid_)] += 1 - return orig_episode_get(*args, **kwargs) - - def task_get(task_id: str, *args: Any, **kwargs: Any): - stats.task_get += 1 - return orig_task_get(task_id, *args, **kwargs) - - client.batch.get = batch_get # type: ignore[method-assign] - client.graph.episode.get = episode_get # type: ignore[method-assign] - client.task.get = task_get # type: ignore[method-assign] - return stats - - -class ListLoader: - def __init__(self, episodes: list[Episode]) -> None: - self.episodes = episodes - - def load(self): - yield from self.episodes - - -@dataclass -class CaseResult: - name: str - ok: bool - seconds: float - detail: str - - -def ok(name: str, seconds: float, detail: str) -> CaseResult: - return CaseResult(name=name, ok=True, seconds=seconds, detail=detail) - - -def fail(name: str, seconds: float, detail: str) -> CaseResult: - return CaseResult(name=name, ok=False, seconds=seconds, detail=detail) - - -def run_case(name: str, fn) -> CaseResult: - start = time.monotonic() - try: - detail = fn() - return ok(name, time.monotonic() - start, detail) - except Exception as exc: # noqa: BLE001 — smoke script reports all failures - return fail(name, time.monotonic() - start, f"{type(exc).__name__}: {exc}") - - -def main() -> int: - run_id = uuid.uuid4().hex[:10] - client = Zep() - stats_holder: list[PollStats] = [] - - graph_id = f"ingest-smoke-{run_id}" - user_id = f"ingest-smoke-user-{run_id}" - client.graph.create(graph_id=graph_id, name=f"ingest smoke {run_id}") - client.user.add( - user_id=user_id, - first_name="Ingest", - last_name="Smoke", - email=f"{user_id}@example.com", - ) - - results: list[CaseResult] = [] - - def case_thread_add_messages_no_task_id() -> str: - thread_id = f"probe-{run_id}" - client.thread.create(thread_id=thread_id, user_id=user_id) - response = client.thread.add_messages( - thread_id, - messages=[ - Message( - role="user", - name="Smoke Tester", - content="Probe: sequential thread.add_messages response shape.", - created_at="2025-06-01T12:00:00Z", - ) - ], - ) - uuids = getattr(response, "message_uuids", None) or [] - task_id = getattr(response, "task_id", None) - if not uuids: - raise AssertionError("expected message_uuids from thread.add_messages") - if task_id is not None: - raise AssertionError(f"expected task_id=None, got {task_id!r}") - return f"message_uuids={len(uuids)}, task_id=null" - - def case_batch_submit_all_wait_once() -> str: - stats = instrument_client(client) - stats_holder.append(stats) - episodes = [ - Episode( - data=f"Batch smoke episode {i}: employee {run_id} shipped feature {i}.", - created_at=f"2024-06-{10 + i:02d}T10:00:00Z", - ) - for i in range(5) - ] - t0 = time.monotonic() - try: - result = ingest(client, ListLoader(episodes), graph_id=graph_id, method="batch") - except BatchUnavailableError as exc: - raise AssertionError("production must support batch API") from exc - submit_s = time.monotonic() - t0 - assert result.items_submitted == 5 - assert result.batch_ids, "expected batch_ids" - assert not result.episode_uuids - pre_wait_batch_calls = stats.batch_get - t1 = time.monotonic() - result.wait(poll_interval=5.0, timeout=600) - wait_s = time.monotonic() - t1 - assert result.status in ("succeeded", "partial") - # Tail polling: should not call batch.get once per item per tick. - wait_batch_calls = stats.batch_get - pre_wait_batch_calls - return ( - f"submit={submit_s:.1f}s wait={wait_s:.1f}s batch_ids={len(result.batch_ids)} " - f"batch.get during wait={wait_batch_calls} status={result.status}" - ) - - def case_sequential_tail_episode_poll() -> str: - stats = instrument_client(client) - stats_holder.append(stats) - episodes = [ - Episode( - data=f"Sequential smoke {i}: project {run_id} milestone {i}.", - created_at=f"2024-07-{10 + i:02d}T09:00:00Z", - ) - for i in range(3) - ] - result = ingest(client, ListLoader(episodes), graph_id=graph_id, method="sequential") - assert len(result.episode_uuids) == 3 - tail = result.episode_uuids[-1] - pre = sum(stats.episode_get.values()) - result.wait(poll_interval=5.0, timeout=300) - polled_uuids = set(stats.episode_get.keys()) - assert result.status == "succeeded" - # Should poll mostly the tail until it completes. - assert tail in polled_uuids - assert stats.episode_get[tail] >= 1 - assert stats.task_get == 0 - return ( - f"episodes={len(result.episode_uuids)} tail={tail[:8]}… " - f"episode.get calls={sum(stats.episode_get.values()) - pre} " - f"unique_uuids_polled={len(polled_uuids)} status={result.status}" - ) - - def case_phased_nodes_then_episodes() -> str: - stats = instrument_client(client) - stats_holder.append(stats) - nodes = ingest_nodes( - client, - [ - NodeItem(name=f"Smoke Entity A {run_id}", label="Entity"), - NodeItem(name=f"Smoke Entity B {run_id}", label="Entity"), - ], - graph_id=graph_id, - ) - assert nodes.task_ids, "add_nodes should return task_ids" - assert not nodes.episode_uuids - t0 = time.monotonic() - nodes.wait(poll_interval=5.0, timeout=300) - nodes_wait = time.monotonic() - t0 - assert nodes.status == "succeeded" - assert all(nodes.node_uuids) - assert stats.task_get >= len(nodes.task_ids) - pre_tasks = stats.task_get - - episodes = ingest( - client, - ListLoader( - [ - Episode( - data=f"After seeding nodes, {run_id} closed the release checklist.", - created_at="2024-08-01T12:00:00Z", - ) - ] - ), - graph_id=graph_id, - method="batch", - ) - t1 = time.monotonic() - episodes.wait(poll_interval=5.0, timeout=300) - ep_wait = time.monotonic() - t1 - assert episodes.status in ("succeeded", "partial") - return ( - f"nodes.wait={nodes_wait:.1f}s node_uuids={nodes.node_uuids} " - f"episodes.wait={ep_wait:.1f}s tasks_during_ep_wait={stats.task_get - pre_tasks} " - f"status={episodes.status}" - ) - - def case_multi_file_one_wait() -> str: - stats = instrument_client(client) - stats_holder.append(stats) - with tempfile.TemporaryDirectory() as tmp: - paths = [] - for i in range(3): - path = Path(tmp) / f"records_{i}.jsonl" - path.write_text( - json.dumps( - { - "id": f"{run_id}-{i}", - "summary": f"JSONL file {i} for {run_id}: inventory update {i}.", - "date": f"2024-09-{10 + i:02d}T00:00:00Z", - } - ) - + "\n" - ) - paths.append(path) - t0 = time.monotonic() - result = ingest_json_records( - client, - paths, - graph_id=graph_id, - id_field="id", - created_at_field="date", - ) - submit_s = time.monotonic() - t0 - assert result.items_submitted == 3 - t1 = time.monotonic() - result.wait(poll_interval=5.0, timeout=600) - wait_s = time.monotonic() - t1 - assert result.status in ("succeeded", "partial") - return ( - f"files=3 submit={submit_s:.1f}s wait={wait_s:.1f}s " - f"batch.get={stats.batch_get} status={result.status}" - ) - - def case_multi_thread_sequential_wait() -> str: - stats = instrument_client(client) - stats_holder.append(stats) - t1 = f"thread-a-{run_id}" - t2 = f"thread-b-{run_id}" - client.thread.create(thread_id=t1, user_id=user_id) - client.thread.create(thread_id=t2, user_id=user_id) - messages = [ - ThreadMessage( - thread_id=t1, - role="user", - name="Alice", - content=f"Thread A message 1 for {run_id}.", - created_at="2025-01-01T10:00:00Z", - ), - ThreadMessage( - thread_id=t2, - role="user", - name="Bob", - content=f"Thread B message 1 for {run_id}.", - created_at="2025-01-01T10:01:00Z", - ), - ThreadMessage( - thread_id=t1, - role="user", - name="Alice", - content=f"Thread A message 2 for {run_id}.", - created_at="2025-01-01T10:02:00Z", - ), - ] - result = ingest_thread_messages(client, messages, user_id=user_id, method="sequential") - assert result.task_ids == [] - assert len(result.episode_uuids) == 2 - assert result._single_queue_episode_poll is False - result.wait(poll_interval=5.0, timeout=600) - assert result.status == "succeeded" - polled = set(stats.episode_get.keys()) - assert polled == set(result.episode_uuids), f"polled {polled} vs {result.episode_uuids}" - return ( - f"threads=2 poll_uuids={result.episode_uuids} " - f"episode.get total={sum(stats.episode_get.values())} status={result.status}" - ) - - def case_thread_batch_backfill() -> str: - stats = instrument_client(client) - stats_holder.append(stats) - thread_id = f"batch-thread-{run_id}" - client.thread.create(thread_id=thread_id, user_id=user_id) - messages = [ - ThreadMessage( - thread_id=thread_id, - role="user", - name="Carol", - content=f"Batch thread message {i} for {run_id}.", - created_at=f"2025-02-{10 + i:02d}T12:00:00Z", - ) - for i in range(4) - ] - try: - result = ingest_thread_messages(client, messages, user_id=user_id, method="batch") - except BatchUnavailableError as exc: - raise AssertionError("production must support batch for thread backfill") from exc - assert result.batch_ids - assert not result.episode_uuids - result.wait(poll_interval=5.0, timeout=600) - assert result.status in ("succeeded", "partial") - return ( - f"messages=4 batch_ids={len(result.batch_ids)} " - f"batch.get={stats.batch_get} status={result.status}" - ) - - cases = [ - ("thread.add_messages has message_uuids not task_id", case_thread_add_messages_no_task_id), - ("batch: submit 5 episodes then wait once", case_batch_submit_all_wait_once), - ("sequential graph.add: tail episode poll", case_sequential_tail_episode_poll), - ("phased: nodes.wait then episodes.wait", case_phased_nodes_then_episodes), - ("multi-file json: one submit one wait", case_multi_file_one_wait), - ("multi-thread sequential: poll each thread tail", case_multi_thread_sequential_wait), - ("thread batch backfill: batch.get wait", case_thread_batch_backfill), - ] - - print(f"zep-ingest production smoke run_id={run_id} graph_id={graph_id}") - print("-" * 72) - for name, fn in cases: - result = run_case(name, fn) - results.append(result) - mark = "PASS" if result.ok else "FAIL" - print(f"[{mark}] {result.name} ({result.seconds:.1f}s)") - print(f" {result.detail}") - - print("-" * 72) - passed = sum(1 for r in results if r.ok) - print(f"{passed}/{len(results)} passed") - - try: - client.graph.delete(graph_id) - except Exception: # noqa: BLE001 - print(f"warning: could not delete graph {graph_id}") - - return 0 if passed == len(results) else 1 - if __name__ == "__main__": - sys.exit(main()) + runpy.run_path(str(Path(__file__).with_name("smoke.py")), run_name="__main__") diff --git a/ingestion/scripts/smoke.py b/ingestion/scripts/smoke.py new file mode 100644 index 00000000..ea2d417d --- /dev/null +++ b/ingestion/scripts/smoke.py @@ -0,0 +1,390 @@ +"""Live smoke tests for zep-ingest against a real Zep account. + +Requires ``ZEP_API_KEY`` (and ``ZEP_API_URL`` for zep-cloud). Runnable locally: + + keybank run zep-prod -- env ZEP_API_URL=https://api.getzep.com \\ + uv run --directory ingestion python scripts/smoke.py + +Cases run in parallel where they share one throwaway graph. The document case +uses a single two-chunk ingestion to verify ``document_id`` grouping. +""" + +from __future__ import annotations + +import json +import sys +import tempfile +import time +import uuid +from collections import Counter +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from zep_cloud.client import Zep +from zep_cloud.types import Message + +from zep_ingest import Episode, NodeItem, ingest, ingest_json_records, ingest_nodes +from zep_ingest._graph_api import get_document_episodes +from zep_ingest.exceptions import BatchUnavailableError +from zep_ingest.threads import ThreadMessage, ingest_thread_messages + + +@dataclass +class PollStats: + batch_get: int = 0 + episode_get: Counter[str] = field(default_factory=Counter) + task_get: int = 0 + + +def instrument_client(client: Zep) -> PollStats: + stats = PollStats() + orig_batch_get = client.batch.get + orig_episode_get = client.graph.episode.get + orig_task_get = client.task.get + + def batch_get(batch_id: str, *args: Any, **kwargs: Any): + stats.batch_get += 1 + return orig_batch_get(batch_id, *args, **kwargs) + + def episode_get(*args: Any, **kwargs: Any): + uuid_ = kwargs.get("uuid_") or (args[0] if args else "?") + stats.episode_get[str(uuid_)] += 1 + return orig_episode_get(*args, **kwargs) + + def task_get(task_id: str, *args: Any, **kwargs: Any): + stats.task_get += 1 + return orig_task_get(task_id, *args, **kwargs) + + client.batch.get = batch_get # type: ignore[method-assign] + client.graph.episode.get = episode_get # type: ignore[method-assign] + client.task.get = task_get # type: ignore[method-assign] + return stats + + +class ListLoader: + def __init__(self, episodes: list[Episode]) -> None: + self.episodes = episodes + + def load(self): + yield from self.episodes + + +@dataclass +class CaseResult: + name: str + ok: bool + seconds: float + detail: str + + +def ok(name: str, seconds: float, detail: str) -> CaseResult: + return CaseResult(name=name, ok=True, seconds=seconds, detail=detail) + + +def fail(name: str, seconds: float, detail: str) -> CaseResult: + return CaseResult(name=name, ok=False, seconds=seconds, detail=detail) + + +def run_case(name: str, fn: Callable[[], str]) -> CaseResult: + start = time.monotonic() + try: + detail = fn() + return ok(name, time.monotonic() - start, detail) + except Exception as exc: # noqa: BLE001 — smoke script reports all failures + return fail(name, time.monotonic() - start, f"{type(exc).__name__}: {exc}") + + +def build_cases( + client: Zep, graph_id: str, user_id: str, run_id: str +) -> list[tuple[str, Callable[[], str]]]: + stats_holder: list[PollStats] = [] + + def case_document_two_chunks() -> str: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "handbook.txt" + path.write_text( + "Alice Chen joined Acme Corp as a product manager in January and owns the mobile " + "roadmap for the entire platform team across all regions.\n\n" + "In March she launched the Acme mobile app to enterprise customers and the launch " + "exceeded adoption targets set for the first quarter of the fiscal year." + ) + from zep_ingest.loaders.text import TextFileLoader + from zep_ingest.transforms.chunker import TextChunker + + episodes = list( + TextChunker(chunk_size=180, overlap=0).apply(TextFileLoader(path).load()) + ) + assert len(episodes) == 2 + document_id = episodes[0].document_id + assert document_id and episodes[1].document_id == document_id + t0 = time.monotonic() + result = ingest(client, ListLoader(episodes), graph_id=graph_id, method="batch") + submit_s = time.monotonic() - t0 + assert result.items_submitted == 2 + result.wait(poll_interval=5.0, timeout=600) + assert result.batch_ids, "expected batch_ids for document ingest" + items = client.batch.list_items(result.batch_ids[0], limit=10).items or [] + assert len(items) == 2, f"expected 2 batch items, got {len(items)}" + assert all(getattr(item, "document_id", None) == document_id for item in items) + listed = get_document_episodes(client, graph_id=graph_id, document_id=document_id) + list_detail = ( + f"list_api={len(listed)}" + if listed + else "list_api=empty (document list may lag on some deployments)" + ) + return ( + f"document_id={document_id!r} chunks=2 submit={submit_s:.1f}s " + f"batch_items={len(items)} {list_detail} status={result.status}" + ) + + def case_thread_add_messages_no_task_id() -> str: + thread_id = f"probe-{run_id}" + client.thread.create(thread_id=thread_id, user_id=user_id) + response = client.thread.add_messages( + thread_id, + messages=[ + Message( + role="user", + name="Smoke Tester", + content="Probe: sequential thread.add_messages response shape.", + created_at="2025-06-01T12:00:00Z", + ) + ], + ) + uuids = getattr(response, "message_uuids", None) or [] + task_id = getattr(response, "task_id", None) + if not uuids: + raise AssertionError("expected message_uuids from thread.add_messages") + if task_id is not None: + raise AssertionError(f"expected task_id=None, got {task_id!r}") + return f"message_uuids={len(uuids)}, task_id=null" + + def case_batch_submit_all_wait_once() -> str: + stats = instrument_client(client) + stats_holder.append(stats) + episodes = [ + Episode( + data=f"Batch smoke episode {i}: employee {run_id} shipped feature {i}.", + created_at=f"2024-06-{10 + i:02d}T10:00:00Z", + ) + for i in range(5) + ] + t0 = time.monotonic() + try: + result = ingest(client, ListLoader(episodes), graph_id=graph_id, method="batch") + except BatchUnavailableError as exc: + raise AssertionError("production must support batch API") from exc + submit_s = time.monotonic() - t0 + assert result.items_submitted == 5 + pre_wait_batch_calls = stats.batch_get + t1 = time.monotonic() + result.wait(poll_interval=5.0, timeout=600) + wait_s = time.monotonic() - t1 + wait_batch_calls = stats.batch_get - pre_wait_batch_calls + return ( + f"submit={submit_s:.1f}s wait={wait_s:.1f}s batch_ids={len(result.batch_ids)} " + f"batch.get during wait={wait_batch_calls} status={result.status}" + ) + + def case_sequential_tail_episode_poll() -> str: + stats = instrument_client(client) + stats_holder.append(stats) + episodes = [ + Episode( + data=f"Sequential smoke {i}: project {run_id} milestone {i}.", + created_at=f"2024-07-{10 + i:02d}T09:00:00Z", + ) + for i in range(3) + ] + result = ingest(client, ListLoader(episodes), graph_id=graph_id, method="sequential") + assert len(result.episode_uuids) == 3 + tail = result.episode_uuids[-1] + pre = sum(stats.episode_get.values()) + result.wait(poll_interval=5.0, timeout=300) + assert result.status == "succeeded" + return ( + f"episodes={len(result.episode_uuids)} tail={tail[:8]}… " + f"episode.get calls={sum(stats.episode_get.values()) - pre} status={result.status}" + ) + + def case_phased_nodes_then_episodes() -> str: + stats = instrument_client(client) + stats_holder.append(stats) + nodes = ingest_nodes( + client, + [ + NodeItem(name=f"Smoke Entity A {run_id}", label="Entity"), + NodeItem(name=f"Smoke Entity B {run_id}", label="Entity"), + ], + graph_id=graph_id, + ) + nodes.wait(poll_interval=5.0, timeout=300) + assert nodes.status == "succeeded" + episodes = ingest( + client, + ListLoader( + [ + Episode( + data=f"After seeding nodes, {run_id} closed the release checklist.", + created_at="2024-08-01T12:00:00Z", + ) + ] + ), + graph_id=graph_id, + method="batch", + ) + t1 = time.monotonic() + episodes.wait(poll_interval=5.0, timeout=300) + ep_wait = time.monotonic() - t1 + return f"nodes.status={nodes.status} episodes.wait={ep_wait:.1f}s status={episodes.status}" + + def case_multi_file_one_wait() -> str: + with tempfile.TemporaryDirectory() as tmp: + paths = [] + for i in range(3): + path = Path(tmp) / f"records_{i}.jsonl" + path.write_text( + json.dumps( + { + "id": f"{run_id}-{i}", + "summary": f"JSONL file {i} for {run_id}: inventory update {i}.", + "date": f"2024-09-{10 + i:02d}T00:00:00Z", + } + ) + + "\n" + ) + paths.append(path) + result = ingest_json_records( + client, + paths, + graph_id=graph_id, + id_field="id", + created_at_field="date", + ) + assert result.items_submitted == 3 + t1 = time.monotonic() + result.wait(poll_interval=5.0, timeout=600) + wait_s = time.monotonic() - t1 + return f"files=3 wait={wait_s:.1f}s status={result.status}" + + def case_multi_thread_sequential_wait() -> str: + stats = instrument_client(client) + stats_holder.append(stats) + t1 = f"thread-a-{run_id}" + t2 = f"thread-b-{run_id}" + client.thread.create(thread_id=t1, user_id=user_id) + client.thread.create(thread_id=t2, user_id=user_id) + messages = [ + ThreadMessage( + thread_id=t1, + role="user", + name="Alice", + content=f"Thread A message 1 for {run_id}.", + created_at="2025-01-01T10:00:00Z", + ), + ThreadMessage( + thread_id=t2, + role="user", + name="Bob", + content=f"Thread B message 1 for {run_id}.", + created_at="2025-01-01T10:01:00Z", + ), + ThreadMessage( + thread_id=t1, + role="user", + name="Alice", + content=f"Thread A message 2 for {run_id}.", + created_at="2025-01-01T10:02:00Z", + ), + ] + result = ingest_thread_messages(client, messages, user_id=user_id, method="sequential") + assert len(result.episode_uuids) == 2 + result.wait(poll_interval=5.0, timeout=600) + assert result.status == "succeeded" + return f"threads=2 status={result.status}" + + def case_thread_batch_backfill() -> str: + thread_id = f"batch-thread-{run_id}" + client.thread.create(thread_id=thread_id, user_id=user_id) + messages = [ + ThreadMessage( + thread_id=thread_id, + role="user", + name="Carol", + content=f"Batch thread message {i} for {run_id}.", + created_at=f"2025-02-{10 + i:02d}T12:00:00Z", + ) + for i in range(4) + ] + try: + result = ingest_thread_messages(client, messages, user_id=user_id, method="batch") + except BatchUnavailableError as exc: + raise AssertionError("production must support batch for thread backfill") from exc + result.wait(poll_interval=5.0, timeout=600) + return f"messages=4 status={result.status}" + + return [ + ("document: two chunks share document_id", case_document_two_chunks), + ("thread.add_messages has message_uuids not task_id", case_thread_add_messages_no_task_id), + ("batch: submit 5 episodes then wait once", case_batch_submit_all_wait_once), + ("sequential graph.add: tail episode poll", case_sequential_tail_episode_poll), + ("phased: nodes.wait then episodes.wait", case_phased_nodes_then_episodes), + ("multi-file json: one submit one wait", case_multi_file_one_wait), + ("multi-thread sequential: poll each thread tail", case_multi_thread_sequential_wait), + ("thread batch backfill: batch.get wait", case_thread_batch_backfill), + ] + + +def main() -> int: + run_id = uuid.uuid4().hex[:10] + client = Zep() + graph_id = f"ingest-smoke-{run_id}" + user_id = f"ingest-smoke-user-{run_id}" + client.graph.create(graph_id=graph_id, name=f"ingest smoke {run_id}") + client.user.add( + user_id=user_id, + first_name="Ingest", + last_name="Smoke", + email=f"{user_id}@example.com", + ) + + cases = build_cases(client, graph_id, user_id, run_id) + results: list[CaseResult] = [] + + print(f"zep-ingest smoke run_id={run_id} graph_id={graph_id}") + print("-" * 72) + + # Document case runs first (validates grouping); the rest run in parallel. + first_name, first_fn = cases[0] + results.append(run_case(first_name, first_fn)) + mark = "PASS" if results[0].ok else "FAIL" + print(f"[{mark}] {results[0].name} ({results[0].seconds:.1f}s)") + print(f" {results[0].detail}") + + parallel_cases = cases[1:] + with ThreadPoolExecutor(max_workers=len(parallel_cases)) as pool: + futures = {pool.submit(run_case, name, fn): name for name, fn in parallel_cases} + for future in as_completed(futures): + result = future.result() + results.append(result) + mark = "PASS" if result.ok else "FAIL" + print(f"[{mark}] {result.name} ({result.seconds:.1f}s)") + print(f" {result.detail}") + + print("-" * 72) + passed = sum(1 for r in results if r.ok) + print(f"{passed}/{len(results)} passed") + + try: + client.graph.delete(graph_id) + except Exception: # noqa: BLE001 + print(f"warning: could not delete graph {graph_id}") + + return 0 if passed == len(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ingestion/src/zep_ingest/_graph_api.py b/ingestion/src/zep_ingest/_graph_api.py new file mode 100644 index 00000000..5a1864a8 --- /dev/null +++ b/ingestion/src/zep_ingest/_graph_api.py @@ -0,0 +1,74 @@ +"""Thin wrappers for graph endpoints not yet exposed on the installed zep-cloud SDK.""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import quote + +from zep_cloud.client import Zep +from zep_cloud.core.api_error import ApiError +from zep_cloud.core.pydantic_utilities import parse_obj_as +from zep_cloud.graph.raw_client import OMIT +from zep_cloud.types.episode import Episode as ZepEpisode + +from zep_ingest.types import Destination, Episode, to_graph_add_kwargs + + +def _raise_api_error(response: Any) -> None: + raise ApiError( + status_code=response.status_code, + body=response.text, + headers=dict(response.headers), + ) + + +def graph_add(client: Zep, episode: Episode, destination: Destination) -> ZepEpisode: + """Submit one graph episode, including ``document_id`` when set.""" + kwargs = to_graph_add_kwargs(episode, destination) + document_id = kwargs.pop("document_id", None) + if document_id is None: + return client.graph.add(**kwargs) + + wrapper = client.graph.with_raw_response._client_wrapper + body: dict[str, Any] = { + "data": kwargs["data"], + "type": kwargs["type"], + "created_at": kwargs.get("created_at", OMIT), + "graph_id": kwargs.get("graph_id", OMIT), + "metadata": kwargs.get("metadata", OMIT), + "user_id": kwargs.get("user_id", OMIT), + "document_id": document_id, + } + response = wrapper.httpx_client.request( + "graph", + method="POST", + json=body, + headers={"content-type": "application/json"}, + omit=OMIT, + ) + if 200 <= response.status_code < 300: + return parse_obj_as(ZepEpisode, response.json()) + _raise_api_error(response) + raise AssertionError("unreachable") + + +def get_document_episodes( + client: Zep, + *, + graph_id: str, + document_id: str, +) -> list[dict[str, Any]]: + """List episodes grouped under a document (v3 ``get_episodes_for_document``).""" + wrapper = client.graph.with_raw_response._client_wrapper + encoded = quote(document_id, safe="") + response = wrapper.httpx_client.request( + f"graph/documents/{encoded}/episodes", + method="GET", + params={"graph_id": graph_id}, + ) + if 200 <= response.status_code < 300: + payload = response.json() + episodes = payload.get("episodes") if isinstance(payload, dict) else None + return list(episodes or []) + _raise_api_error(response) + raise AssertionError("unreachable") diff --git a/ingestion/src/zep_ingest/documents.py b/ingestion/src/zep_ingest/documents.py new file mode 100644 index 00000000..4bc7cab3 --- /dev/null +++ b/ingestion/src/zep_ingest/documents.py @@ -0,0 +1,84 @@ +"""Document grouping helpers for Zep's ``document_id`` API field. + +See https://help.getzep.com/documents. Text files, transcripts, emails, and +Slack sources are always documents. Slack is ingested as one episode per +message with a shared ``document_id`` for the Slack thread, plus a second +document of the channel's top-level messages (thread parents and unthreaded +posts) in order. JSON records stay standalone. Zep ``thread_id`` is reserved +for agent–user conversations on a user graph and is never set here. +""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path + +from zep_ingest.exceptions import ConfigurationError + +MAX_DOCUMENT_ID_LENGTH = 100 +_ILLEGAL_DOCUMENT_ID_RE = re.compile(r"[/\?#]|[\x00-\x1f\x7f]") +_MESSAGE_ID = re.compile(r"<([^>]+)>") + + +def normalize_document_id(value: str) -> str: + return value.strip() + + +def validate_document_id(value: str) -> None: + """Validate a caller-supplied document_id before submission.""" + normalized = normalize_document_id(value) + if not normalized: + raise ConfigurationError("document_id must be a non-empty string") + if len(normalized) > MAX_DOCUMENT_ID_LENGTH: + raise ConfigurationError( + f"document_id exceeds {MAX_DOCUMENT_ID_LENGTH} characters (got {len(normalized)})" + ) + if _ILLEGAL_DOCUMENT_ID_RE.search(normalized): + raise ConfigurationError( + "document_id must not contain /, ?, #, or ASCII control characters" + ) + + +def document_id_from_parts(*parts: str, prefix: str) -> str: + """Build a stable, API-safe document_id from logical parts.""" + key = ":".join(str(part) for part in parts if part is not None and str(part) != "") + candidate = f"{prefix}:{key}" if key else prefix + if len(candidate) <= MAX_DOCUMENT_ID_LENGTH and not _ILLEGAL_DOCUMENT_ID_RE.search(candidate): + return candidate + digest = hashlib.sha256(candidate.encode()).hexdigest()[:32] + return f"{prefix}-{digest}" + + +def document_id_for_path(path: Path, *, prefix: str = "file") -> str: + """Stable id for one file (always a document, even as a single episode).""" + return document_id_from_parts(str(path.resolve()), prefix=prefix) + + +def document_id_for_slack_thread(channel: str, thread_ts: str) -> str: + """Shared document_id for every message episode in one Slack thread.""" + return document_id_from_parts(channel, thread_ts, prefix="slack") + + +def document_id_for_slack_channel(channel: str) -> str: + """Shared document_id for a channel's top-level messages (in order).""" + return document_id_from_parts(channel, prefix="slack-channel") + + +def _first_message_id(value: str) -> str | None: + match = _MESSAGE_ID.search(value) + if match: + return match.group(1).strip() + stripped = value.strip() + return stripped or None + + +def document_id_for_email_thread(message: object, path: Path) -> str: + """One document per email thread: References/In-Reply-To root, else Message-ID, else file.""" + for header in ("References", "In-Reply-To", "Message-ID"): + raw = message.get(header) if hasattr(message, "get") else None # type: ignore[union-attr] + if raw: + message_id = _first_message_id(str(raw)) + if message_id: + return document_id_from_parts(message_id, prefix="email") + return document_id_for_path(path, prefix="email") diff --git a/ingestion/src/zep_ingest/loaders/email.py b/ingestion/src/zep_ingest/loaders/email.py index ab9b380a..f6c042e6 100644 --- a/ingestion/src/zep_ingest/loaders/email.py +++ b/ingestion/src/zep_ingest/loaders/email.py @@ -16,6 +16,7 @@ from email.utils import parsedate_to_datetime from zep_ingest._io import resolve_source_files, source_paths_label +from zep_ingest.documents import document_id_for_email_thread from zep_ingest.types import Episode, SourcePaths _HTML_DROP = re.compile(r"<(script|style)\b.*?", re.IGNORECASE | re.DOTALL) @@ -67,6 +68,7 @@ def load(self) -> Iterator[Episode]: data=f"Email from {sender} to {recipient} (subject: {subject}):\n{body}", data_type="text", created_at=created_at, + document_id=document_id_for_email_thread(message, file), metadata={ "source_type": "email", "subject": subject[:100], diff --git a/ingestion/src/zep_ingest/loaders/json_records.py b/ingestion/src/zep_ingest/loaders/json_records.py index 4c402dfc..02c96e8e 100644 --- a/ingestion/src/zep_ingest/loaders/json_records.py +++ b/ingestion/src/zep_ingest/loaders/json_records.py @@ -187,13 +187,14 @@ def _to_episode(self, record: Any, file: Path, index: int) -> tuple[Episode, int created_at: str | None = None timestamp_missing = 0 metadata: dict[str, Any] = {"source_type": "json_record", "file_name": file.name} - if isinstance(record, dict): + source_record: dict[str, Any] | None = record if isinstance(record, dict) else None + if source_record is not None: # every field the caller names is read from the record as they wrote it, # never from a key the loader just wrote: with id_field='sku' and # name_field='id' the name is still the record's own 'id', and the order # of the mappings below carries no meaning - original = record - record = dict(record) + original = source_record + record = dict(source_record) for target, field in ( ("id", self.id_field), ("name", self.name_field), diff --git a/ingestion/src/zep_ingest/loaders/slack.py b/ingestion/src/zep_ingest/loaders/slack.py index 525d1ab3..39c51ae5 100644 --- a/ingestion/src/zep_ingest/loaders/slack.py +++ b/ingestion/src/zep_ingest/loaders/slack.py @@ -15,9 +15,12 @@ raw Slack IDs in the graph, which degrades entity extraction); the same roster labels DMs and group DMs by their members, since Slack names those folders with an opaque id (D01ABC234) or slug (mpdm-alice--bob-1). Normalizes Slack -markup, skips join/leave/bot noise, groups messages by thread (the semantic -unit Zep extracts best from), and stamps every episode with the original -message timestamp so backfilled facts carry the correct valid_at timeline. +markup, skips join/leave/bot noise, emits one episode per message under a Slack-thread ``document_id``, and a second +copy of each top-level message (thread parent or unthreaded post) under a +per-channel ``document_id`` so channel-level conversation is visible as its own +document. Not a Zep ``thread_id`` — those are reserved for agent–user +conversations on a user graph. Stamps every episode with the original message +timestamp so backfilled facts carry the correct valid_at timeline. """ import html @@ -30,6 +33,7 @@ from pathlib import Path from typing import Any, Literal +from zep_ingest.documents import document_id_for_slack_channel, document_id_for_slack_thread from zep_ingest.exceptions import ConfigurationError from zep_ingest.types import Episode @@ -276,7 +280,6 @@ def __init__( *, channels: Sequence[str] | None = None, conversation_types: Sequence[ConversationType] = DEFAULT_CONVERSATION_TYPES, - grouping: Literal["thread", "message"] = "thread", include_bots: bool = False, skip_subtypes: frozenset[str] = DEFAULT_SKIP_SUBTYPES, formatter: Callable[[SlackMessage], str] | None = None, @@ -284,8 +287,6 @@ def __init__( self.path = Path(path) if not self.path.exists(): raise ConfigurationError(f"Slack export not found: {self.path}") - if grouping not in ("thread", "message"): - raise ConfigurationError(f"grouping must be 'thread' or 'message', got {grouping!r}") self.conversation_types = tuple(dict.fromkeys(conversation_types)) if not self.conversation_types or any( kind not in CONVERSATION_FILES for kind in self.conversation_types @@ -296,7 +297,6 @@ def __init__( ) # channels= selects by name *within* the selected conversation types self.channels = list(channels) if channels is not None else None - self.grouping = grouping self.include_bots = include_bots self.skip_subtypes = skip_subtypes self.formatter = formatter or _default_formatter @@ -574,20 +574,22 @@ def _load_conversation( # every episode below carries conversation.label, so the members it # names are now in the graph whether or not they authored anything self._note_label_names(conversation, users) - if self.grouping == "message": - for message in messages: - yield self._episode([message], conversation) - return - threads: dict[str, list[SlackMessage]] = {} - order: list[str] = [] for message in messages: - key = message.thread_ts or message.ts - if key not in threads: - threads[key] = [] - order.append(key) - threads[key].append(message) - for key in sorted(order, key=float): - yield self._episode(threads[key], conversation) + yield self._episode( + message, + conversation, + document_id=document_id_for_slack_thread( + conversation.label, message.thread_ts or message.ts + ), + ) + # Channel-level document: top-level posts in order (thread parents and + # unthreaded messages). Same payload as the thread copy; only document_id + # differs. Replies stay thread-only. + channel_id = document_id_for_slack_channel(conversation.label) + for message in messages: + if not _is_top_level(message): + continue + yield self._episode(message, conversation, document_id=channel_id) def _parse( self, raw: dict[str, Any], conversation: _Conversation, users: dict[str, str] @@ -672,21 +674,26 @@ def _normalize_text(self, text: str, users: dict[str, str]) -> str: text = _LINK_BARE.sub(r"\1", text) return html.unescape(text) - def _episode(self, messages: list[SlackMessage], conversation: _Conversation) -> Episode: - first = messages[0] + def _episode( + self, message: SlackMessage, conversation: _Conversation, *, document_id: str + ) -> Episode: metadata: dict[str, Any] = { "source_type": "slack", "channel": conversation.label, "conversation_type": conversation.kind, } - # a lone message still belongs to a thread when it carries a thread_ts — - # its parent may have been filtered out as a join or bot message — so - # message count alone would leave that episode unfilterable by thread - if len(messages) > 1 or first.thread_ts: - metadata["thread_ts"] = first.thread_ts or first.ts + # Slack thread_ts (not a Zep thread_id) is provenance for replies. + if message.thread_ts: + metadata["thread_ts"] = message.thread_ts return Episode( - data="\n".join(self.formatter(m) for m in messages), + data=self.formatter(message), data_type="text", - created_at=datetime.fromtimestamp(float(first.ts), tz=UTC).isoformat(), + created_at=datetime.fromtimestamp(float(message.ts), tz=UTC).isoformat(), + document_id=document_id, metadata=metadata, ) + + +def _is_top_level(message: SlackMessage) -> bool: + """True for an unthreaded post or the parent of a Slack thread.""" + return message.thread_ts is None or message.thread_ts == message.ts diff --git a/ingestion/src/zep_ingest/loaders/text.py b/ingestion/src/zep_ingest/loaders/text.py index b06f1ef6..e492e038 100644 --- a/ingestion/src/zep_ingest/loaders/text.py +++ b/ingestion/src/zep_ingest/loaders/text.py @@ -10,6 +10,7 @@ from datetime import UTC, datetime from zep_ingest._io import resolve_source_files, source_paths_label +from zep_ingest.documents import document_id_for_path from zep_ingest.types import Episode, SourcePaths @@ -40,5 +41,6 @@ def load(self) -> Iterator[Episode]: data=file.read_text(encoding="utf-8"), data_type="text", created_at=created_at, + document_id=document_id_for_path(file), metadata={"source_type": "document", "file_name": file.name}, ) diff --git a/ingestion/src/zep_ingest/loaders/transcript.py b/ingestion/src/zep_ingest/loaders/transcript.py index 53ad1a3b..555db120 100644 --- a/ingestion/src/zep_ingest/loaders/transcript.py +++ b/ingestion/src/zep_ingest/loaders/transcript.py @@ -12,6 +12,7 @@ from zep_ingest._io import resolve_source_files, source_paths_label from zep_ingest._validation import require_int_range +from zep_ingest.documents import document_id_for_path from zep_ingest.exceptions import ConfigurationError from zep_ingest.types import Episode, SourcePaths @@ -98,6 +99,7 @@ def _load_file(self, file: Path) -> Iterator[Episode]: start = self._resolve_start(file, headers) title = headers.get("MEETING") or headers.get("TITLE") or file.stem chunks = self._chunk(turns) + document_id = document_id_for_path(file, prefix="transcript") for index, chunk in enumerate(chunks, start=1): created_at = None if start is not None: @@ -106,6 +108,7 @@ def _load_file(self, file: Path) -> Iterator[Episode]: data="\n".join(f"{turn.speaker}: {turn.text}" for turn in chunk), data_type="text", created_at=created_at, + document_id=document_id, metadata={ "source_type": "transcript", "meeting": title[:100], diff --git a/ingestion/src/zep_ingest/pipeline.py b/ingestion/src/zep_ingest/pipeline.py index 7c096bab..843517f1 100644 --- a/ingestion/src/zep_ingest/pipeline.py +++ b/ingestion/src/zep_ingest/pipeline.py @@ -77,6 +77,7 @@ def _validated_replay(episodes: Iterable[Episode]) -> Iterator[Iterator[Episode] episode.data_type, episode.created_at, episode.metadata, + episode.document_id, episode.document, ) pickle.dump(record, spool, protocol=pickle.HIGHEST_PROTOCOL) @@ -85,7 +86,9 @@ def _validated_replay(episodes: Iterable[Episode]) -> Iterator[Iterator[Episode] def replay() -> Iterator[Episode]: while True: try: - data, data_type, created_at, metadata, document = pickle.load(spool) + data, data_type, created_at, metadata, document_id, document = pickle.load( + spool + ) except EOFError: return yield Episode( @@ -93,6 +96,7 @@ def replay() -> Iterator[Episode]: data_type=data_type, created_at=created_at, metadata=metadata, + document_id=document_id, document=document, ) @@ -231,7 +235,6 @@ def ingest_slack_export( user_id: str | None = None, channels: Sequence[str] | None = None, conversation_types: Sequence[ConversationType] = DEFAULT_CONVERSATION_TYPES, - grouping: Literal["thread", "message"] = "thread", include_bots: bool = False, skip_subtypes: frozenset[str] = DEFAULT_SKIP_SUBTYPES, formatter: Callable[..., str] | None = None, @@ -241,6 +244,8 @@ def ingest_slack_export( ) -> IngestResult: """One-liner: Slack workspace export (.zip or directory) → Zep graph. + One episode per message under a Slack-thread ``document_id``, plus a + channel-level document of top-level messages in order. Ingests public channels only unless ``conversation_types`` says otherwise; private channels, DMs and group DMs found in the export but not selected are reported in ``result.warnings``. @@ -249,7 +254,6 @@ def ingest_slack_export( path, channels=channels, conversation_types=conversation_types, - grouping=grouping, include_bots=include_bots, skip_subtypes=skip_subtypes, formatter=formatter, diff --git a/ingestion/src/zep_ingest/result.py b/ingestion/src/zep_ingest/result.py index 4d1cd9e4..152eb281 100644 --- a/ingestion/src/zep_ingest/result.py +++ b/ingestion/src/zep_ingest/result.py @@ -12,6 +12,7 @@ IngestTimeoutError, IngestUntrackedError, ) +from zep_ingest.types import Episode if TYPE_CHECKING: from zep_cloud.client import Zep @@ -148,8 +149,15 @@ class IngestResult: # Task-param recovery must not overwrite or reorder that list. _node_uuids_from_submit: bool = field(default=False, repr=False, compare=False) # Plain graph.add (and single-thread backfills) share one per-graph queue, so - # polling the last UUID is enough. Multiple threads are independent sagas. + # polling the last UUID is enough. Multiple threads or document_id sagas are + # independent queues. _single_queue_episode_poll: bool = field(default=True, repr=False, compare=False) + _document_poll_uuids: dict[str, str] = field(default_factory=dict, repr=False, compare=False) + _document_poll_created_at: dict[str, str | None] = field( + default_factory=dict, repr=False, compare=False + ) + _plain_episode_tail: str | None = field(default=None, repr=False, compare=False) + _uses_document_grouping: bool = field(default=False, repr=False, compare=False) @classmethod def from_batch_ids(cls, client: "Zep", batch_ids: "Sequence[str]") -> "IngestResult": @@ -163,6 +171,33 @@ def from_task_ids(cls, client: "Zep", task_ids: "Sequence[str]") -> "IngestResul """Reconstruct a task-backed sequential result from persisted task IDs.""" return cls(method="sequential", task_ids=list(task_ids), client=client) + def record_sequential_episode(self, episode: Episode, uuid: str) -> None: + """Track a sequential graph.add submission for wait() polling.""" + self.episode_uuids.append(uuid) + if episode.document_id: + self._uses_document_grouping = True + prior_at = self._document_poll_created_at.get(episode.document_id) + new_at = episode.created_at + if prior_at is None or new_at is None: + # Missing timestamps: fall back to submission order (last wins). + self._document_poll_uuids[episode.document_id] = uuid + self._document_poll_created_at[episode.document_id] = new_at + elif new_at >= prior_at: + self._document_poll_uuids[episode.document_id] = uuid + self._document_poll_created_at[episode.document_id] = new_at + else: + self._plain_episode_tail = uuid + + def finalize_sequential_episode_poll(self) -> None: + """Collapse sequential episode UUIDs to one tail per independent saga.""" + if not self._uses_document_grouping: + return + poll_uuids = list(self._document_poll_uuids.values()) + if self._plain_episode_tail is not None: + poll_uuids.append(self._plain_episode_tail) + self.episode_uuids = poll_uuids + self._single_queue_episode_poll = len(poll_uuids) <= 1 + def mark_batch_failed(self, batch_id: str, error: str) -> None: """Record a batch whose processing could not be triggered: an AddError is added and the batch's status is pinned to "failed" so status/wait() treat @@ -307,11 +342,11 @@ def combine(self, *others: "IngestResult") -> "IngestResult": "batch" if has_batches and not has_sequential_handles else "sequential" ) combined = IngestResult(method=method, client=client) + uses_document_grouping = False for part in parts: combined.items_submitted += part.items_submitted combined.untracked_items += part.untracked_items combined.batch_ids.extend(part.batch_ids) - combined.episode_uuids.extend(part.episode_uuids) combined.task_ids.extend(part.task_ids) combined.add_errors.extend(part.add_errors) combined.warnings.extend(part.warnings) @@ -322,6 +357,33 @@ def combine(self, *others: "IngestResult") -> "IngestResult": combined._single_queue_episode_poll = ( combined._single_queue_episode_poll and part._single_queue_episode_poll ) + if part._uses_document_grouping: + uses_document_grouping = True + for document_id, uuid in part._document_poll_uuids.items(): + created_at = part._document_poll_created_at.get(document_id) + prior_at = combined._document_poll_created_at.get(document_id) + if prior_at is None or created_at is None: + combined._document_poll_uuids[document_id] = uuid + combined._document_poll_created_at[document_id] = created_at + elif (created_at or "") >= (prior_at or ""): + combined._document_poll_uuids[document_id] = uuid + combined._document_poll_created_at[document_id] = created_at + if part._plain_episode_tail is not None: + combined._plain_episode_tail = part._plain_episode_tail + else: + combined.episode_uuids.extend(part.episode_uuids) + combined._uses_document_grouping = uses_document_grouping + if uses_document_grouping: + poll_uuids = list(combined._document_poll_uuids.values()) + if combined._plain_episode_tail is not None: + poll_uuids.append(combined._plain_episode_tail) + if combined.episode_uuids: + if combined._single_queue_episode_poll: + poll_uuids.append(combined.episode_uuids[-1]) + else: + poll_uuids.extend(combined.episode_uuids) + combined.episode_uuids = list(dict.fromkeys(poll_uuids)) + combined._single_queue_episode_poll = len(combined.episode_uuids) <= 1 return combined def _queued_item_count(self) -> int | None: @@ -363,9 +425,9 @@ def wait( backfills): ``batch.get`` on the last batch id — not per-episode polling. - **Sequential ``graph.add``** (batch fallback): the last-submitted - episode's ``processed`` flag. ``zep-ingest`` does not set - ``document_id``, so submission order matches ingestion order for plain - ``graph.add`` chunks. + episode's ``processed`` flag per document saga (when ``document_id`` is + set) or the last plain episode otherwise. Ingestion order within a + document follows ``created_at``, not submission order. - **Sequential ``thread.add_messages``**: the last message UUID in the last request per thread (``message_uuids[-1]``). Regular ``thread.add_messages`` does not return a ``task_id`` — only diff --git a/ingestion/src/zep_ingest/submitters/sequential.py b/ingestion/src/zep_ingest/submitters/sequential.py index 3af21c44..b6f6155b 100644 --- a/ingestion/src/zep_ingest/submitters/sequential.py +++ b/ingestion/src/zep_ingest/submitters/sequential.py @@ -5,8 +5,8 @@ which correct valid_at sequencing depends on. Sequential means the Batch API is not used — each episode is submitted with graph.add — not that processing must finish before the next episode or file is sent. The HTTP add returns as -soon as the episode is queued; wait() is opt-in and polls the last-submitted -episode (plain graph.add without document_id). +soon as the episode is queued; wait() is opt-in and polls the tail episode per document +saga (or the last plain graph.add episode when no document_id is set). """ import math @@ -22,9 +22,10 @@ from zep_cloud.core.api_error import ApiError from zep_ingest._errors import SubmitError, format_api_error +from zep_ingest._graph_api import graph_add from zep_ingest._validation import require_int_range, require_nonnegative_number from zep_ingest.result import AddError, IngestResult -from zep_ingest.types import Destination, Episode, to_graph_add_kwargs +from zep_ingest.types import Destination, Episode #: Ceiling on any single retry sleep. A server or proxy is free to send #: "Retry-After: 86400"; honoring that verbatim stalls the whole import. @@ -116,13 +117,16 @@ def __init__(self, client: Zep, *, max_retries: int = 5, min_interval: float = 0 def submit(self, episodes: Iterable[Episode], destination: Destination) -> IngestResult: result = IngestResult(method="sequential", client=self.client) for index, episode in enumerate(episodes): - kwargs = to_graph_add_kwargs(episode, destination) - self._add_episode(index, kwargs, result) + self._add_episode(index, episode, destination, result) + result.finalize_sequential_episode_poll() return result - def _add_episode(self, index: int, kwargs: dict, result: IngestResult) -> None: + def _add_episode( + self, index: int, episode: Episode, destination: Destination, result: IngestResult + ) -> None: zep_episode, error = call_with_retries( - lambda: self.client.graph.add(**kwargs), max_retries=self.max_retries + lambda: graph_add(self.client, episode, destination), + max_retries=self.max_retries, ) if error is not None: result.add_errors.append( @@ -133,7 +137,7 @@ def _add_episode(self, index: int, kwargs: dict, result: IngestResult) -> None: ) ) return - result.episode_uuids.append(zep_episode.uuid_) + result.record_sequential_episode(episode, zep_episode.uuid_) result.items_submitted += 1 if self.min_interval > 0: time.sleep(self.min_interval) diff --git a/ingestion/src/zep_ingest/transforms/canonicalizer.py b/ingestion/src/zep_ingest/transforms/canonicalizer.py index e08fd3fb..7baa8791 100644 --- a/ingestion/src/zep_ingest/transforms/canonicalizer.py +++ b/ingestion/src/zep_ingest/transforms/canonicalizer.py @@ -171,6 +171,7 @@ def apply(self, episodes: Iterable[Episode]) -> Iterator[Episode]: data_type=episode.data_type, created_at=episode.created_at, metadata=episode.metadata, + document_id=episode.document_id, document=episode.document, ) self.flush_warnings() diff --git a/ingestion/src/zep_ingest/transforms/chunker.py b/ingestion/src/zep_ingest/transforms/chunker.py index c3ad47d3..7bc388b3 100644 --- a/ingestion/src/zep_ingest/transforms/chunker.py +++ b/ingestion/src/zep_ingest/transforms/chunker.py @@ -78,5 +78,6 @@ def apply(self, episodes: Iterable[Episode]) -> Iterator[Episode]: data_type=episode.data_type, created_at=episode.created_at, metadata=metadata, + document_id=episode.document_id, document=document, ) diff --git a/ingestion/src/zep_ingest/transforms/contextualizer.py b/ingestion/src/zep_ingest/transforms/contextualizer.py index cfced9fd..a67428ea 100644 --- a/ingestion/src/zep_ingest/transforms/contextualizer.py +++ b/ingestion/src/zep_ingest/transforms/contextualizer.py @@ -107,5 +107,6 @@ def _without_document(episode: Episode, data: str) -> Episode: data_type=episode.data_type, created_at=episode.created_at, metadata=episode.metadata, + document_id=episode.document_id, document=None, ) diff --git a/ingestion/src/zep_ingest/transforms/limits.py b/ingestion/src/zep_ingest/transforms/limits.py index 4f2f8cf3..39accd29 100644 --- a/ingestion/src/zep_ingest/transforms/limits.py +++ b/ingestion/src/zep_ingest/transforms/limits.py @@ -72,6 +72,7 @@ def apply(self, episodes: Iterable[Episode]) -> Iterator[Episode]: data_type=output_type, created_at=episode.created_at, metadata=metadata, + document_id=episode.document_id, document=episode.document, ) diff --git a/ingestion/src/zep_ingest/types.py b/ingestion/src/zep_ingest/types.py index 76a13527..d5169faf 100644 --- a/ingestion/src/zep_ingest/types.py +++ b/ingestion/src/zep_ingest/types.py @@ -8,6 +8,7 @@ from zep_cloud.types.batch_add_item import BatchAddItem from zep_ingest._validation import check_scalar_map, check_timestamp +from zep_ingest.documents import validate_document_id from zep_ingest.exceptions import ConfigurationError # Documented Zep API limits (see help.getzep.com/adding-batch-data, @@ -38,12 +39,17 @@ class Episode: ``document`` is internal plumbing: the chunker sets it to the full source document when it splits, and the contextualizer consumes it. It is never sent to the API. + + ``document_id`` is the Zep document grouping key. Text files, transcripts, + and emails always carry one (even as a single episode). Slack messages share + one per Slack thread. JSON records omit it. Never a Zep ``thread_id``. """ data: str data_type: DataType = "text" created_at: str | None = None # RFC3339; loaders populate from source timestamps metadata: dict[str, Any] | None = None + document_id: str | None = None document: str | None = field(default=None, repr=False) def __post_init__(self) -> None: @@ -58,6 +64,11 @@ def __post_init__(self) -> None: check_scalar_map("metadata", self.metadata, errors, max_keys=MAX_METADATA_KEYS) if self.document is not None and not isinstance(self.document, str): errors.append(f"document must be a string, got {type(self.document).__name__}") + if self.document_id is not None: + try: + validate_document_id(self.document_id) + except ConfigurationError as error: + errors.append(str(error)) if errors: raise ConfigurationError("Invalid episode: " + "; ".join(errors)) @@ -80,15 +91,18 @@ def __post_init__(self) -> None: def to_batch_item(episode: Episode, destination: Destination) -> BatchAddItem: """Map a validated Episode to a Batch API item.""" - return BatchAddItem( - type="graph_episode", - data=episode.data, - data_type=episode.data_type, - created_at=episode.created_at, - metadata=episode.metadata, - graph_id=destination.graph_id, - user_id=destination.user_id, - ) + kwargs: dict[str, Any] = { + "type": "graph_episode", + "data": episode.data, + "data_type": episode.data_type, + "created_at": episode.created_at, + "metadata": episode.metadata, + "graph_id": destination.graph_id, + "user_id": destination.user_id, + } + if episode.document_id is not None: + kwargs["document_id"] = episode.document_id + return BatchAddItem(**kwargs) def to_graph_add_kwargs(episode: Episode, destination: Destination) -> dict[str, Any]: @@ -102,4 +116,6 @@ def to_graph_add_kwargs(episode: Episode, destination: Destination) -> dict[str, kwargs["graph_id"] = destination.graph_id if destination.user_id is not None: kwargs["user_id"] = destination.user_id + if episode.document_id is not None: + kwargs["document_id"] = episode.document_id return kwargs diff --git a/ingestion/tests/test_document_poll.py b/ingestion/tests/test_document_poll.py new file mode 100644 index 00000000..56855300 --- /dev/null +++ b/ingestion/tests/test_document_poll.py @@ -0,0 +1,32 @@ +"""Tests for document saga poll tail selection.""" + +from zep_ingest.result import IngestResult +from zep_ingest.types import Episode + + +def test_document_poll_tail_uses_latest_created_at(): + result = IngestResult(method="sequential") + result.record_sequential_episode( + Episode(data="older chunk", document_id="doc-1", created_at="2024-01-01T00:00:00Z"), + "uuid-older", + ) + result.record_sequential_episode( + Episode(data="newer chunk", document_id="doc-1", created_at="2024-06-01T00:00:00Z"), + "uuid-newer", + ) + result.finalize_sequential_episode_poll() + assert result.episode_uuids == ["uuid-newer"] + + +def test_document_poll_tail_submission_order_when_created_at_missing(): + result = IngestResult(method="sequential") + result.record_sequential_episode( + Episode(data="first", document_id="doc-1", created_at=None), + "uuid-first", + ) + result.record_sequential_episode( + Episode(data="second", document_id="doc-1", created_at=None), + "uuid-second", + ) + result.finalize_sequential_episode_poll() + assert result.episode_uuids == ["uuid-second"] diff --git a/ingestion/tests/test_documents.py b/ingestion/tests/test_documents.py new file mode 100644 index 00000000..19a231f9 --- /dev/null +++ b/ingestion/tests/test_documents.py @@ -0,0 +1,87 @@ +"""Tests for document_id helpers and Episode validation.""" + +import pytest + +from zep_ingest.documents import ( + MAX_DOCUMENT_ID_LENGTH, + document_id_for_path, + document_id_for_slack_thread, + document_id_from_parts, + validate_document_id, +) +from zep_ingest.exceptions import ConfigurationError +from zep_ingest.transforms.chunker import TextChunker +from zep_ingest.types import Destination, Episode, to_batch_item, to_graph_add_kwargs + + +def test_validate_document_id_rejects_illegal_chars(): + with pytest.raises(ConfigurationError, match="/"): + validate_document_id("bad/id") + + +def test_document_id_from_parts_hashes_long_keys(): + long_key = "x" * 200 + doc_id = document_id_from_parts(long_key, prefix="file") + assert len(doc_id) <= MAX_DOCUMENT_ID_LENGTH + assert doc_id.startswith("file-") + + +def test_slack_thread_document_id_is_stable(): + assert document_id_for_slack_thread( + "#general", "1710000000.000100" + ) == document_id_for_slack_thread("#general", "1710000000.000100") + + +def test_episode_document_id_validation(): + with pytest.raises(ConfigurationError, match="document_id"): + Episode(data="hello", document_id="") + + +def test_api_mappers_include_document_id(): + episode = Episode(data="chunk", document_id="handbook-v1") + destination = Destination(graph_id="g1") + assert to_graph_add_kwargs(episode, destination)["document_id"] == "handbook-v1" + batch_item = to_batch_item(episode, destination) + assert batch_item.model_dump()["document_id"] == "handbook-v1" + + +def test_chunker_preserves_document_id_when_splitting(): + source = Episode( + data="word " * 300, + document_id=document_id_for_path(__import__("pathlib").Path("notes.md")), + ) + chunks = list(TextChunker(chunk_size=100, overlap=0).apply([source])) + assert len(chunks) > 1 + assert all(chunk.document_id == source.document_id for chunk in chunks) + + +def test_chunker_keeps_document_id_on_pass_through(): + doc_id = document_id_for_path(__import__("pathlib").Path("notes.md")) + source = Episode(data="short doc", document_id=doc_id) + [out] = list(TextChunker().apply([source])) + assert out.document_id == doc_id + + +def test_email_always_gets_document_id(): + from pathlib import Path + + from zep_ingest.documents import document_id_for_email_thread + + class Msg: + def __init__(self, headers: dict[str, str]) -> None: + self._headers = headers + + def get(self, key: str) -> str | None: + return self._headers.get(key) + + standalone = document_id_for_email_thread( + Msg({"Message-ID": ""}), Path("solo.eml") + ) + assert standalone is not None + assert "solo@example.com" in standalone + + reply = document_id_for_email_thread( + Msg({"In-Reply-To": "", "Message-ID": ""}), + Path("reply.eml"), + ) + assert "thread-root@example.com" in reply diff --git a/ingestion/tests/test_pipeline.py b/ingestion/tests/test_pipeline.py index 8e65a2da..4655793e 100644 --- a/ingestion/tests/test_pipeline.py +++ b/ingestion/tests/test_pipeline.py @@ -273,11 +273,12 @@ def test_ingest_slack_export(self, mock_zep): items = mock_zep.batch.add.call_args.kwargs["items"] assert all(i.data_type == "text" for i in items) assert all(i.created_at is not None for i in items) - assert result.items_submitted == len(items) == 4 + assert result.items_submitted == len(items) == 10 + assert all(getattr(i, "document_id", None) for i in items) def test_ingest_slack_export_channel_filter(self, mock_zep): result = ingest_slack_export(mock_zep, FIXTURE, graph_id="g1", channels=["random"]) - assert result.items_submitted == 1 + assert result.items_submitted == 2 def test_ingest_documents_chunks_long_files(self, mock_zep, tmp_path): (tmp_path / "doc.md").write_text("\n\n".join("sentence here. " * 20 for _ in range(10))) @@ -290,7 +291,7 @@ def test_ingest_documents_chunks_long_files(self, mock_zep, tmp_path): def test_ingest_slack_export_skip_subtypes(self, mock_zep): result = ingest_slack_export(mock_zep, FIXTURE, graph_id="g1", skip_subtypes=frozenset()) items = mock_zep.batch.add.call_args.kwargs["items"] - assert result.items_submitted == 5 + assert result.items_submitted == 12 assert any("has joined the channel" in i.data for i in items) def test_ingest_slack_export_risky_words_guard(self, mock_zep): diff --git a/ingestion/tests/test_slack_loader.py b/ingestion/tests/test_slack_loader.py index 22995afa..beff232a 100644 --- a/ingestion/tests/test_slack_loader.py +++ b/ingestion/tests/test_slack_loader.py @@ -23,6 +23,18 @@ def general(episodes: list[Episode]) -> list[Episode]: return [e for e in episodes if e.metadata and e.metadata.get("channel") == "general"] +def thread_pass(episodes: list[Episode]) -> list[Episode]: + """The first copy of each message (Slack-thread document), excluding the + per-channel top-level duplicate.""" + return [ + e + for e in episodes + if e.document_id + and e.document_id.startswith("slack:") + and not e.document_id.startswith("slack-channel:") + ] + + def iso(ts: str) -> str: return datetime.fromtimestamp(float(ts), tz=UTC).isoformat() @@ -136,11 +148,11 @@ def test_all_episodes_are_text_type_with_metadata(self): def test_general_channel_episode_order_and_count(self): eps = general(load()) - # hello, thread (parent + 2 replies), markup message — join/bot/empty skipped - assert len(eps) == 3 + # 5 thread-document episodes, then 3 top-level copies on the channel document + assert len(eps) == 8 assert "Hello world" in eps[0].data assert "Should we deprioritize PROTOTYPE-202?" in eps[1].data - assert "check" in eps[2].data + assert "check" in eps[4].data def test_created_at_is_rfc3339_from_ts(self): eps = general(load()) @@ -150,27 +162,28 @@ def test_missing_export_path_raises_eagerly(self): with pytest.raises(ConfigurationError): SlackExportLoader(FIXTURE / "does-not-exist") - def test_invalid_grouping_is_rejected_eagerly(self): - with pytest.raises(ConfigurationError, match="grouping"): - SlackExportLoader(FIXTURE, grouping="messages") # type: ignore[arg-type] - -class TestThreadGrouping: - def test_thread_grouped_into_one_episode_across_day_files(self): - thread = general(load())[1] - lines = thread.data.split("\n") - assert len(lines) == 3 # parent + reply + cross-day reply (broadcast deduped) - assert "Should we deprioritize PROTOTYPE-202?" in lines[0] - assert "yes, let's do that" in lines[1] - assert "Actually PROTOTYPE-202 stays active" in lines[2] - - def test_thread_created_at_is_parent_ts(self): - thread = general(load())[1] - assert thread.created_at == iso("1718356000.000200") +class TestThreadAsDocument: + def test_one_episode_per_message_shares_document_id(self): + eps = general(load()) + parent = next(e for e in eps if "Should we deprioritize PROTOTYPE-202?" in e.data) + reply = next(e for e in eps if "yes, let's do that" in e.data) + later = next(e for e in eps if "Actually PROTOTYPE-202 stays active" in e.data) + hello = next(e for e in eps if "Hello world" in e.data) + assert parent.document_id == reply.document_id == later.document_id + assert parent.document_id is not None + assert hello.document_id != parent.document_id + + def test_thread_reply_created_at_is_its_own_ts(self): + reply = next(e for e in general(load()) if "yes, let's do that" in e.data) + assert reply.created_at == iso("1718356100.000300") - def test_thread_metadata_has_thread_ts(self): - thread = general(load())[1] - assert thread.metadata["thread_ts"] == "1718356000.000200" + def test_thread_metadata_has_slack_thread_ts_not_zep_thread(self): + parent = next( + e for e in general(load()) if "Should we deprioritize PROTOTYPE-202?" in e.data + ) + assert parent.metadata["thread_ts"] == "1718356000.000200" + assert not hasattr(parent, "thread_id") def test_lone_reply_keeps_thread_ts_when_its_parent_was_filtered(self, tmp_path): """Message count is not the test for "is a thread": a reply whose parent was @@ -195,14 +208,33 @@ def test_lone_reply_keeps_thread_ts_when_its_parent_was_filtered(self, tmp_path) orphan = next(e for e in episodes if "orphan reply" in e.data) standalone = next(e for e in episodes if "standalone" in e.data) assert orphan.metadata["thread_ts"] == "1718400000.1" - assert "thread_ts" not in standalone.metadata # not part of any thread + assert orphan.document_id == next( + e.document_id for e in episodes if e.metadata.get("thread_ts") == "1718400000.1" + ) + assert "thread_ts" not in standalone.metadata # not part of any Slack thread - def test_message_grouping_yields_one_episode_per_message(self): - eps = general(load(grouping="message")) - # hello + 3 thread messages + markup (broadcast deduped, join/bot/empty skipped) - assert len(eps) == 5 + def test_channel_document_duplicates_top_level_only(self): + from zep_ingest.documents import document_id_for_slack_channel, document_id_for_slack_thread + + eps = general(load()) + channel_id = document_id_for_slack_channel("general") + channel_eps = [e for e in eps if e.document_id == channel_id] + assert [e.data for e in channel_eps] == [ + next(e.data for e in eps if "Hello world" in e.data), + next(e.data for e in eps if "Should we deprioritize PROTOTYPE-202?" in e.data), + next(e.data for e in eps if "https://example.com" in e.data), + ] reply = next(e for e in eps if "yes, let's do that" in e.data) - assert reply.created_at == iso("1718356100.000300") + assert reply.document_id != channel_id + parent = next(e for e in eps if "Should we deprioritize PROTOTYPE-202?" in e.data) + parent_channel = next( + e for e in channel_eps if "Should we deprioritize PROTOTYPE-202?" in e.data + ) + assert parent.document_id == document_id_for_slack_thread("general", "1718356000.000200") + assert parent.data == parent_channel.data + assert parent.created_at == parent_channel.created_at + assert parent.metadata == parent_channel.metadata + assert parent.document_id != parent_channel.document_id class TestFiltering: @@ -274,12 +306,12 @@ def test_join_subtype_skipped(self): assert not any("has joined" in e.data for e in load()) def test_empty_messages_skipped(self): - for ep in load(grouping="message"): + for ep in load(): assert ep.data.strip() def test_channel_filter(self): episodes = load(channels=["random"]) - assert len(episodes) == 1 + assert len(episodes) == 2 assert "Random note" in episodes[0].data assert episodes[0].metadata["channel"] == "random" @@ -287,7 +319,7 @@ def test_channel_filter(self): class TestConversationTypes: def test_default_ingests_public_channels_only(self, tmp_path): episodes = load(mixed_export(tmp_path)) - assert [e.metadata["conversation_type"] for e in episodes] == ["public_channel"] + assert {e.metadata["conversation_type"] for e in episodes} == {"public_channel"} assert "public channel message" in episodes[0].data def test_skipped_types_warn_with_counts_and_remedy(self, tmp_path): @@ -312,7 +344,7 @@ def test_no_skip_warning_when_every_type_selected(self, tmp_path): conversation_types=["public_channel", "private_channel", "dm", "group_dm"], ) episodes = list(loader.load()) - assert len(episodes) == 4 + assert len(episodes) == 8 assert not any("conversation_types" in w for w in loader.warnings) def test_dms_only_selection(self, tmp_path): @@ -322,7 +354,7 @@ def test_dms_only_selection(self, tmp_path): def test_private_channels_only_selection(self, tmp_path): episodes = load(mixed_export(tmp_path), conversation_types=["private_channel"]) - assert len(episodes) == 1 + assert len(episodes) == 2 assert episodes[0].metadata["channel"] == "leadership" assert episodes[0].data == ( "Blake Carter (Slack #leadership, 2024-06-14 09:01 UTC): private channel message" @@ -364,7 +396,7 @@ def test_channels_filter_applies_within_selected_types(self, tmp_path): conversation_types=["public_channel", "private_channel"], channels=["leadership"], ) - assert len(episodes) == 1 + assert len(episodes) == 2 assert episodes[0].metadata["channel"] == "leadership" def test_channel_of_unselected_type_raises_with_remedy(self, tmp_path): @@ -378,7 +410,7 @@ def test_dm_selectable_by_id_or_member_label(self, tmp_path): export = mixed_export(tmp_path) by_id = load(export, conversation_types=["dm"], channels=["D01ABC234"]) by_label = load(export, conversation_types=["dm"], channels=["Avery Brown, Blake Carter"]) - assert len(by_id) == 1 + assert len(thread_pass(by_id)) == 1 assert [e.data for e in by_id] == [e.data for e in by_label] def test_unknown_channel_error_lists_dm_ids_with_their_labels(self, tmp_path): @@ -430,10 +462,10 @@ def test_real_name_wins_over_a_display_name_handle(self): assert "avery (" not in eps[0].data def test_name_source_fallbacks(self): - eps = general(load()) + bodies = " ".join(e.data for e in general(load())) # Blake Carter has empty display_name -> real_name; charlie has neither -> name - assert "Blake Carter" in eps[1].data - assert "charlie" in eps[1].data + assert "Blake Carter" in bodies + assert "charlie" in bodies def test_display_name_is_used_when_no_real_name_exists(self, tmp_path): export = tmp_path / "export" @@ -446,7 +478,7 @@ def test_display_name_is_used_when_no_real_name_exists(self, tmp_path): json.dumps([{"type": "message", "user": "U1", "text": "Hi", "ts": "1718355600.000100"}]) ) loader = SlackExportLoader(export) - [episode] = list(loader.load()) + episode = thread_pass(list(loader.load()))[0] assert "Morgan Lee" in episode.data # a full name, handle-shaped or not, is not worth warning about assert not any("no real_name" in w for w in loader.warnings) @@ -468,7 +500,7 @@ def formatter(message): assert None in captured # the fixture's bot post carries a username, not an id def test_markup_normalization(self): - markup = general(load())[2].data + markup = next(e.data for e in general(load()) if "https://example.com" in e.data) assert markup == ( "Blake Carter (Slack #general, 2024-06-15 09:08 UTC): " "@Avery Brown and @U999 check #random @here & see " @@ -560,7 +592,7 @@ def test_a_handle_in_a_mention_is_reported(self, tmp_path): ) ) loader = SlackExportLoader(export) - [episode] = list(loader.load()) + episode = thread_pass(list(loader.load()))[0] assert "@riley" in episode.data assert any("riley" in w for w in loader.warnings if "no real_name" in w) @@ -595,7 +627,7 @@ def test_mention_in_a_message_with_an_unusable_ts_is_not_reported(self, tmp_path ], ) loader = SlackExportLoader(export) - [episode] = list(loader.load()) + episode = thread_pass(list(loader.load()))[0] assert "riley" not in episode.data assert any("not a number" in w for w in loader.warnings) # it really was dropped assert not any("no real_name" in w for w in loader.warnings) @@ -632,7 +664,7 @@ def test_mention_in_a_kept_message_is_still_reported(self, tmp_path): tmp_path, [{"user": "U1", "text": "ask <@U2>", "ts": "1718355600.000100"}] ) loader = SlackExportLoader(export) - [episode] = list(loader.load()) + episode = thread_pass(list(loader.load()))[0] assert "@riley" in episode.data assert any("riley" in w for w in loader.warnings if "no real_name" in w) @@ -684,7 +716,7 @@ def dm_export(root: Path) -> Path: def test_handle_only_dm_member_is_reported(self, tmp_path): loader = SlackExportLoader(self.dm_export(tmp_path), conversation_types=["dm"]) - [episode] = list(loader.load()) + episode = thread_pass(list(loader.load()))[0] # the handle is in the episode text and metadata, so it reaches extraction assert "riley" in episode.data assert "riley" in episode.metadata["channel"] @@ -720,7 +752,7 @@ def test_extracted_wrapper_is_unwrapped(self, tmp_path): loader = SlackExportLoader(root) episodes = list(loader.load()) - assert [e.metadata["channel"] for e in episodes] == ["general"] + assert [e.metadata["channel"] for e in thread_pass(episodes)] == ["general"] assert not any("declares no conversations" in w for w in loader.warnings) def test_extracted_wrapper_still_honors_the_public_only_default(self, tmp_path): @@ -817,8 +849,11 @@ def test_wrapped_grid_archive_ingests_nothing_private_by_default(self, tmp_path) def test_wrapped_grid_archive_typed_correctly_when_requested(self, tmp_path): archive = wrapped_grid_export(tmp_path) episodes = load(archive, conversation_types=["private_channel", "dm"]) - assert [e.metadata["conversation_type"] for e in episodes] == ["private_channel", "dm"] - assert [e.metadata["channel"] for e in episodes] == [ + assert [e.metadata["conversation_type"] for e in thread_pass(episodes)] == [ + "private_channel", + "dm", + ] + assert [e.metadata["channel"] for e in thread_pass(episodes)] == [ "leadership", "Avery Brown, Blake Carter", ] @@ -845,7 +880,7 @@ def test_wrapper_folder_stripped_for_every_index_or_roster_marker(self, tmp_path ), ) # unstripped, the only conversation would be the wrapper folder itself - assert [e.metadata["channel"] for e in load(archive)] == ["general"] + assert [e.metadata["channel"] for e in thread_pass(load(archive))] == ["general"] def test_nested_json_is_not_read_as_a_day_file(self, tmp_path): export = tmp_path / "export" @@ -868,7 +903,7 @@ def test_nested_json_is_not_read_as_a_day_file(self, tmp_path): archive = shutil.make_archive(str(tmp_path / "nested"), "zip", export) for source in (export, Path(archive)): episodes = load(source) - assert len(episodes) == 1 + assert len(episodes) == 2 assert "nested" not in episodes[0].data @@ -879,7 +914,7 @@ class TestIndexLessExport: def test_dm_shaped_folders_are_not_ingested_as_public_channels(self, tmp_path): episodes = load(index_less_export(tmp_path)) - assert [e.metadata["channel"] for e in episodes] == ["general", "leadership"] + assert [e.metadata["channel"] for e in thread_pass(episodes)] == ["general", "leadership"] assert not any("direct message" in e.data or "group dm message" in e.data for e in episodes) def test_undetermined_types_are_warned_about(self, tmp_path): @@ -897,8 +932,11 @@ def test_dm_shaped_folders_reported_as_skipped(self, tmp_path): def test_dm_shaped_folders_ingested_on_explicit_request(self, tmp_path): episodes = load(index_less_export(tmp_path), conversation_types=["dm", "group_dm"]) - assert [e.metadata["conversation_type"] for e in episodes] == ["dm", "group_dm"] - assert [e.metadata["channel"] for e in episodes] == [ + assert [e.metadata["conversation_type"] for e in thread_pass(episodes)] == [ + "dm", + "group_dm", + ] + assert [e.metadata["channel"] for e in thread_pass(episodes)] == [ "D01ABC234", "mpdm-avery--blake--charlie-1", ] diff --git a/ingestion/tests/test_smoke.py b/ingestion/tests/test_smoke.py new file mode 100644 index 00000000..3b194fec --- /dev/null +++ b/ingestion/tests/test_smoke.py @@ -0,0 +1,25 @@ +"""Live integration tests — thin pytest wrapper around scripts/smoke.py.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = [ + pytest.mark.integration, + pytest.mark.smoke, + pytest.mark.skipif(not os.getenv("ZEP_API_KEY"), reason="ZEP_API_KEY not set"), +] + + +def test_smoke_suite(): + root = Path(__file__).resolve().parents[1] + script = root / "scripts" / "smoke.py" + completed = subprocess.run( + [sys.executable, str(script)], + cwd=root, + check=False, + ) + assert completed.returncode == 0, "smoke suite failed; see output above" diff --git a/ingestion/tests/test_types.py b/ingestion/tests/test_types.py index 15b15c57..db61c7a7 100644 --- a/ingestion/tests/test_types.py +++ b/ingestion/tests/test_types.py @@ -40,9 +40,10 @@ def test_defaults(self): assert ep.metadata is None assert ep.document is None - def test_document_excluded_from_repr(self): + def test_internal_document_excluded_from_repr(self): ep = Episode(data="chunk", document="a" * 100) - assert "document" not in repr(ep) + assert "document=" not in repr(ep) + assert "document_id" in repr(ep) class TestMetadataValues: From 77284db6854d794035766818f90e82b93ba6355c Mon Sep 17 00:00:00 2001 From: jackaldenryan Date: Sat, 29 Aug 2026 23:09:01 -0500 Subject: [PATCH 2/2] fix(ingestion): compare document tails as instants, not RFC3339 strings Email Date headers mix timezone offsets; lexicographic created_at order could poll the wrong saga. Post-release smoke now checks out the release SHA. Co-authored-by: Cursor --- .github/workflows/smoke-ingestion.yml | 4 +++ ingestion/src/zep_ingest/result.py | 38 ++++++++++++++++----- ingestion/tests/test_document_poll.py | 48 +++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/.github/workflows/smoke-ingestion.yml b/.github/workflows/smoke-ingestion.yml index e7e053e7..3ae29ce6 100644 --- a/.github/workflows/smoke-ingestion.yml +++ b/.github/workflows/smoke-ingestion.yml @@ -26,6 +26,10 @@ jobs: working-directory: ingestion steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + # workflow_run otherwise checks out current default-branch HEAD, not + # the commit the Release Ingestion Package workflow ran on. + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} - name: Install uv uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 diff --git a/ingestion/src/zep_ingest/result.py b/ingestion/src/zep_ingest/result.py index 152eb281..636629d5 100644 --- a/ingestion/src/zep_ingest/result.py +++ b/ingestion/src/zep_ingest/result.py @@ -3,6 +3,7 @@ import time from collections.abc import Sequence from dataclasses import dataclass, field +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Literal from zep_ingest._validation import require_int_range, require_nonnegative_number @@ -41,6 +42,32 @@ ] +def _parse_created_at(value: str | None) -> datetime | None: + """Parse an episode ``created_at`` into a comparable UTC instant.""" + if not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed + + +def _is_later_or_equal_created_at(new_at: str | None, prior_at: str | None) -> bool: + """Whether ``new_at`` is the later document tail. + + Missing or unparseable timestamps fall back to last-submitted-wins. Offsets + are compared as instants, not lexicographic RFC3339 strings. + """ + new_dt = _parse_created_at(new_at) + prior_dt = _parse_created_at(prior_at) + if new_dt is None or prior_dt is None: + return True + return new_dt >= prior_dt + + def _normalize_task_status(status: str | None) -> str: status = status.lower() if status is not None else None if status is None or status in {"created", "draft", "pending", "queued"}: @@ -178,11 +205,7 @@ def record_sequential_episode(self, episode: Episode, uuid: str) -> None: self._uses_document_grouping = True prior_at = self._document_poll_created_at.get(episode.document_id) new_at = episode.created_at - if prior_at is None or new_at is None: - # Missing timestamps: fall back to submission order (last wins). - self._document_poll_uuids[episode.document_id] = uuid - self._document_poll_created_at[episode.document_id] = new_at - elif new_at >= prior_at: + if _is_later_or_equal_created_at(new_at, prior_at): self._document_poll_uuids[episode.document_id] = uuid self._document_poll_created_at[episode.document_id] = new_at else: @@ -362,10 +385,7 @@ def combine(self, *others: "IngestResult") -> "IngestResult": for document_id, uuid in part._document_poll_uuids.items(): created_at = part._document_poll_created_at.get(document_id) prior_at = combined._document_poll_created_at.get(document_id) - if prior_at is None or created_at is None: - combined._document_poll_uuids[document_id] = uuid - combined._document_poll_created_at[document_id] = created_at - elif (created_at or "") >= (prior_at or ""): + if _is_later_or_equal_created_at(created_at, prior_at): combined._document_poll_uuids[document_id] = uuid combined._document_poll_created_at[document_id] = created_at if part._plain_episode_tail is not None: diff --git a/ingestion/tests/test_document_poll.py b/ingestion/tests/test_document_poll.py index 56855300..63910055 100644 --- a/ingestion/tests/test_document_poll.py +++ b/ingestion/tests/test_document_poll.py @@ -30,3 +30,51 @@ def test_document_poll_tail_submission_order_when_created_at_missing(): ) result.finalize_sequential_episode_poll() assert result.episode_uuids == ["uuid-second"] + + +def test_document_poll_tail_compares_instants_not_rfc3339_strings(): + """Offsets that sort the wrong way lexicographically still pick the later instant.""" + result = IngestResult(method="sequential") + result.record_sequential_episode( + Episode( + data="later Pacific", + document_id="doc-1", + created_at="2024-06-15T18:00:00-07:00", + ), + "uuid-later", + ) + result.record_sequential_episode( + Episode( + data="earlier UTC", + document_id="doc-1", + created_at="2024-06-15T20:00:00+00:00", + ), + "uuid-earlier", + ) + result.finalize_sequential_episode_poll() + assert result.episode_uuids == ["uuid-later"] + + +def test_combine_document_poll_tail_uses_later_instant(): + first = IngestResult(method="sequential") + first.record_sequential_episode( + Episode( + data="later Pacific", + document_id="doc-1", + created_at="2024-06-15T18:00:00-07:00", + ), + "uuid-later", + ) + first.finalize_sequential_episode_poll() + second = IngestResult(method="sequential") + second.record_sequential_episode( + Episode( + data="earlier UTC", + document_id="doc-1", + created_at="2024-06-15T20:00:00+00:00", + ), + "uuid-earlier", + ) + second.finalize_sequential_episode_poll() + combined = first.combine(second) + assert combined.episode_uuids == ["uuid-later"]