diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..e0ec740b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# Request owner review for project-direction and automation controls. +/GOALS.md @microsoft/typechat +/AUTOMATION.md @microsoft/typechat +AGENTS.md @microsoft/typechat +/.github/ @microsoft/typechat +/tools/ @microsoft/typechat diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5eda8def..d82e0156 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -3,6 +3,23 @@ TypeChat turns natural-language requests into validated, typed data. Preserve schema accuracy, validation behavior, and compatibility when proposing changes. +## Project direction + +Read [GOALS.md](../GOALS.md) before assessing scope or proposing work. It is the +primary repository policy for product direction. Use the approved default-branch +version; proposed edits in an unmerged PR are not active policy. + +Only the human owner team, `@microsoft/typechat`, can approve changes to that +policy or exceptions to its boundaries. Items under consideration are not +approved implementation work. If the policy is unavailable, fit is unclear, or +a request conflicts with it, explain the issue and pause dependent work for an +owner decision. Do not infer approval from community interest, labels, passing +checks, or agent recommendations. + +Ordinary in-scope maintenance follows the existing maintainer review process. +Do not remove supported behavior or close existing contributions merely because +of a possible policy conflict; preserve the evidence and ask the owner. + ## Working in this repository - The TypeScript library and examples are in `typescript/`; the Python library diff --git a/.github/workflows/agent-workflow-check.yml b/.github/workflows/agent-workflow-check.yml index 0f570262..9dd99939 100644 --- a/.github/workflows/agent-workflow-check.yml +++ b/.github/workflows/agent-workflow-check.yml @@ -4,6 +4,11 @@ on: push: branches: [main] paths: + - "GOALS.md" + - "AUTOMATION.md" + - "README.md" + - ".github/CODEOWNERS" + - ".github/copilot-instructions.md" - ".github/workflows/**" - ".github/aw/**" - ".gitattributes" @@ -13,6 +18,11 @@ on: pull_request: branches: [main] paths: + - "GOALS.md" + - "AUTOMATION.md" + - "README.md" + - ".github/CODEOWNERS" + - ".github/copilot-instructions.md" - ".github/workflows/**" - ".github/aw/**" - ".gitattributes" diff --git a/.github/workflows/backlog-assessment.lock.yml b/.github/workflows/backlog-assessment.lock.yml index 55a10ee4..2152188c 100644 --- a/.github/workflows/backlog-assessment.lock.yml +++ b/.github/workflows/backlog-assessment.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0458055e1d785f0e05648bf798816a3eaf57d976e9b6d34fb3fb8bc7ef38c785","body_hash":"66eb48ead0418a70c45760ee76b8ff74ba17814d2f95b75721b83f0bd2834833","compiler_version":"v0.93.5","strict":true,"agent_id":"copilot","agent_model":"gpt-6.1-sol","engine_versions":{"copilot":"1.0.101"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"dd580e8108789dba2ef69a9d1668ecb1e8b566d80d0c70c815f064efe8a506ec","body_hash":"ef4bad0d891c2dcdb5e1914c0d89cec9718c76fbd887ca84e8a21fa625c1f20c","compiler_version":"v0.93.5","strict":true,"agent_id":"copilot","agent_model":"gpt-6.1-sol","engine_versions":{"copilot":"1.0.101"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"},{"repo":"github/gh-aw/actions/setup","sha":"32ecafbc061c9af57e9361942099e381102ae304","version":"32ecafbc061c9af57e9361942099e381102ae304"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.37","digest":"sha256:9c7f242f6dec2d7d7b55d4c9e07872d59b85377c6cb3de46c915a5d1ffb91442","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.37@sha256:9c7f242f6dec2d7d7b55d4c9e07872d59b85377c6cb3de46c915a5d1ffb91442"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:23746305eeded20dc2726ab18bb081962d66ee3b2e09a54291afd1869ab3ef6d","pinned_image":"ghcr.io/github/gh-aw-node@sha256:23746305eeded20dc2726ab18bb081962d66ee3b2e09a54291afd1869ab3ef6d"},{"image":"ghcr.io/github/github-mcp-server:v2.0.3","digest":"sha256:250bc6ce70788207608ef74cef4397230ff2e8481218da74d6093e4bd9101132","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.3@sha256:250bc6ce70788207608ef74cef4397230ff2e8481218da74d6093e4bd9101132"}],"mcp_servers":[{"name":"github","tools":["get_discussion","get_discussion_comments","get_file_contents","issue_read","list_issues","list_pull_requests","pull_request_read"]},{"name":"safeoutputs","tools":["noop","report_incomplete","report_missing_data"]}],"threat_detection":{"mode":"enabled"},"admission":false} # This file was automatically generated by gh-aw (v0.93.5). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -10,6 +10,9 @@ # # Prepare a draft assessment of open issues and existing pull requests for human review. # +# Frontmatter env variables: +# - TYPECHAT_POLICY_COMMIT: (main workflow) +# # Secrets used: # - GH_AW_DEFAULT_OTLP_ENDPOINT # - GH_AW_DEFAULT_OTLP_HEADERS @@ -58,6 +61,7 @@ concurrency: run-name: "Backlog assessment" env: + TYPECHAT_POLICY_COMMIT: ${{ github.sha }} OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.GH_AW_DEFAULT_OTLP_ENDPOINT || vars.GH_AW_DEFAULT_OTLP_ENDPOINT }} OTEL_SERVICE_NAME: gh-aw.backlog-assessment OTEL_RESOURCE_ATTRIBUTES: 'gh-aw.workflow.name=Backlog%20assessment,gh-aw.repository=${{ github.repository }},gh-aw.run.id=${{ github.run_id }},github.run_id=${{ github.run_id }},gh-aw.engine.id=copilot' @@ -138,7 +142,7 @@ jobs: GH_AW_INFO_AGENT_RUNTIME: "" GH_AW_INFO_ENGINE_HARNESS: "copilot-cli" GH_AW_INFO_MODEL_CONFIG: '{"routing":{"enabled":false},"models":{"allowed":[],"blocked":[]},"max_ai_credits_source":"frontmatter"}' - GH_AW_INFO_MAX_AI_CREDITS: "200" + GH_AW_INFO_MAX_AI_CREDITS: "400" GH_AW_COMPILED_STRICT: "true" GH_AW_INFO_MODEL_COSTS: '{"providers":{"github-copilot":{"models":{"gpt-6.1-sol":{"cost":{"cache_read":"1e-07","cache_write":"2.5e-06","input":"2e-06","output":"1e-05"}}}}}}' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -179,7 +183,7 @@ jobs: GH_AW_HAS_LABEL_COMMAND: "false" GH_AW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }} - GH_AW_MAX_AI_CREDITS: "200" + GH_AW_MAX_AI_CREDITS: "400" with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -263,6 +267,7 @@ jobs: GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_mcp_transport_prompt.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"file\":\"github_mcp_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"}],\"system_item_count\":11}" + GH_AW_ENV_TYPECHAT_POLICY_COMMIT: ${{ env.TYPECHAT_POLICY_COMMIT }} GH_AW_EXPR_76DF9333: ${{ github.event.pull_request.number || (fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_type == 'pull_request' && fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_number) }} GH_AW_EXPR_77C1A4D2: ${{ github.event.comment.id || fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').comment_id }} GH_AW_EXPR_7C248226: ${{ github.event.issue.number || (fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_type == 'issue' && fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_number) }} @@ -292,6 +297,7 @@ jobs: GH_AW_SUB_AGENT_EXT: ".agent.md" GH_AW_SKILL_DIR: ".github/skills" GH_AW_SKILL_EXT: "/SKILL.md" + GH_AW_ENV_TYPECHAT_POLICY_COMMIT: ${{ env.TYPECHAT_POLICY_COMMIT }} GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} with: script: | @@ -305,6 +311,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_TYPECHAT_POLICY_COMMIT: ${{ env.TYPECHAT_POLICY_COMMIT }} GH_AW_EXPR_76DF9333: ${{ github.event.pull_request.number || (fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_type == 'pull_request' && fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_number) }} GH_AW_EXPR_77C1A4D2: ${{ github.event.comment.id || fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').comment_id }} GH_AW_EXPR_7C248226: ${{ github.event.issue.number || (fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_type == 'issue' && fromJSON(steps.generate_aw_info.outputs.aw_context || '{}').item_number) }} @@ -327,6 +334,7 @@ jobs: return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, substitutions: { + GH_AW_ENV_TYPECHAT_POLICY_COMMIT: process.env.GH_AW_ENV_TYPECHAT_POLICY_COMMIT, GH_AW_EXPR_76DF9333: process.env.GH_AW_EXPR_76DF9333, GH_AW_EXPR_77C1A4D2: process.env.GH_AW_EXPR_77C1A4D2, GH_AW_EXPR_7C248226: process.env.GH_AW_EXPR_7C248226, @@ -832,9 +840,9 @@ jobs: export GH_AW_NODE_BIN export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" (umask 177 && touch /tmp/gh-aw/agent-stdio.log) - GH_AW_MAX_AI_CREDITS="200" + GH_AW_MAX_AI_CREDITS="400" if [[ ! "$GH_AW_MAX_AI_CREDITS" =~ ^[0-9]+$ ]]; then - GH_AW_MAX_AI_CREDITS="200" + GH_AW_MAX_AI_CREDITS="400" fi if [[ "${GH_AW_ADMISSION_RESULT:-}" == "admitted" && -z "${GH_AW_ADMISSION_MAX_AI_CREDITS:-}" ]]; then echo "ERR_ADMISSION: Missing required runtime admission allowance." >&2 @@ -1321,7 +1329,7 @@ jobs: GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} GH_AW_DETECTION_JOB_RESULT: ${{ needs.detection.result }} - GH_AW_MAX_AI_CREDITS: "200" + GH_AW_MAX_AI_CREDITS: "400" GH_AW_DEFAULT_CHECKOUT_USES_TRIGGER_REF: "false" GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} diff --git a/.github/workflows/backlog-assessment.md b/.github/workflows/backlog-assessment.md index 7258e212..c02daaf5 100644 --- a/.github/workflows/backlog-assessment.md +++ b/.github/workflows/backlog-assessment.md @@ -12,6 +12,9 @@ permissions: pull-requests: read discussions: read copilot-requests: write +env: + # Prompt markdown supports env references, but not github.sha directly. + TYPECHAT_POLICY_COMMIT: ${{ github.sha }} engine: id: copilot model: gpt-6.1-sol @@ -20,7 +23,7 @@ checkout: false network: defaults timeout-minutes: 20 max-turns: 120 -max-ai-credits: 200 +max-ai-credits: 400 concurrency: group: backlog-assessment cancel-in-progress: false @@ -80,9 +83,47 @@ post-steps: Assess the open issues in `${{ github.repository }}` and the existing pull requests that may address them. Produce a draft for maintainers to review, -not decisions or implementation. Follow the repository's AUTOMATION.md, -contribution guidance, and security reporting policy, reading them with the -GitHub tools. +not decisions or implementation. + +## Project direction + +Read GOALS.md at `${{ env.TYPECHAT_POLICY_COMMIT }}` with the GitHub tools before +assessing any proposals. This is the trusted commit for this default-branch run. Read +AUTOMATION.md, contribution guidance, and the security reporting policy at the +same commit. GOALS.md is the primary repository policy for product scope; it +does not override organization, platform, or security requirements. + +Do not use proposed policy edits in an issue or PR as the active policy. +Community interest, labels, passing checks, and agent recommendations do not +change the project's direction. Only the human owner team named in GOALS.md +can approve direction changes or exceptions. + +For every issue and PR, use one of these direction classifications: + +- **In scope:** The proposal fits the approved goals without conflicting with + the non-goals. This is a scope assessment, not approval to implement or merge. +- **Needs owner decision:** The fit is unclear, evidence is insufficient, or + the idea is only under consideration. Explain what the owner needs to decide. +- **Conflicts with direction:** The proposal conflicts with a current non-goal + or would change an approved boundary. Explain the conflict respectfully; + do not recommend implementation or acceptance before an owner-approved + policy change is merged. + +Give a plain-language reason and a citation to the relevant goal or non-goal. +Link to GOALS.md at the run's commit, not a moving branch or a proposed revision. +For an item under consideration, cite that section and explain that it is not +approved scope. Preserve earlier decisions as evidence; do not use them to +silently override the current policy. + +If GOALS.md cannot be read or its revision cannot be established, state the +problem prominently. Mark proposals "Needs owner decision" and do not recommend +implementation or acceptance. Continue recording factual inventory and coverage +where possible, without guessing policy content. + +Keep items needing an owner decision separate from in-scope maintenance. +For these items, suggest a specific scope question rather than implementation +work. Possible conflicts do not authorize closing existing issues or removing +supported integrations or examples. ## Boundaries @@ -110,8 +151,10 @@ GitHub tools. new work. Distinguish explicit issue-closing links from merely related PRs. 3. Summarize the requested outcome, known scope, open questions, and proposed testable acceptance criteria. Clearly distinguish existing criteria from - agent suggestions. Inspect relevant source only when it helps; do not claim - that a bug was reproduced or a fix verified without execution evidence. + agent suggestions. Apply the direction classifications above before + suggesting implementation work. Inspect relevant source only when it helps; + do not claim that a bug was reproduced or a fix verified without execution + evidence. 4. Recommend a next step: review an existing PR, continue discussion, request information, consider for implementation, defer, or consider closure with evidence. Every recommendation remains subject to human approval. Age alone @@ -120,24 +163,78 @@ GitHub tools. prerequisites for reliable testing. Separate design-heavy or ambiguous work from well-scoped candidates. Do not create duplicate implementation tasks. +## Writing style + +Write for community members with different technical backgrounds, including +people for whom English is not their first language. Sound like a helpful +colleague explaining the situation, not a log of status codes. + +- Use plain English, complete sentences, and short sentences and paragraphs. + Explain what a problem means for someone using TypeChat before describing + implementation details. Give each paragraph one main idea. +- Explain technical terms and abbreviations on first use. Prefer "latest commit" + to "head", "next decision" to "disposition", and "found" to "enumerated". + Keep exact API names and error messages when they matter, and explain them. +- Avoid idioms, regional expressions, sarcasm, and strings of compressed + phrases separated by semicolons. Be respectful of contributors and recognize + their work without suggesting that an unverified change is ready to merge. +- Use descriptive headings and short paragraphs for individual issues and PRs. + Use bullets for checks and questions. Reserve tables for compact facts with + at most three columns; do not put multi-sentence assessments in table cells. +- Preserve facts, evidence links, uncertainty, and existing maintainer decisions. + Say "not checked" when evidence was not inspected, rather than "missing". + Do not remove technical requirements or turn reported results into verified + results to make the report shorter or more reassuring. +- Explain the human-review requirement clearly at the start. Repeat it where a + specific approval is needed, rather than adding the same warning to every + entry. Keep coverage gaps visible, with detailed counts in the evidence notes. + +For example, replace "Head checks: CLA only; acceptance evidence outstanding" +with "The contributor reports that the tests pass. The only check found for the +latest commit confirms the contributor license agreement. We still need test +results before deciding whether the change is ready." This is a wording example, +not evidence to reuse for another pull request. + ## Report Create `backlog-assessment.md` even when the backlog is empty or some information is unavailable. Include these sections: - **Draft status and coverage:** State "Draft - requires maintainer review", - assessment time, totals, pages/items read, and whether coverage is complete, - partial, or unavailable. Explain errors and gaps; never invent missing data. -- **Issue inventory:** One row per discovered open issue, linking to the issue - and any relevant PRs. Include current discussion state, proposed next step, - evidence, and unresolved questions. Mark unassessed issues explicitly. -- **Candidate acceptance criteria:** Record existing criteria and separately - label any proposed additions for the first few actionable candidates. -- **Existing pull requests:** Highlight linked contributions and blockers. - Distinguish a contributor's claimed verification from independently observed - checks. This assessment is not a GitHub Copilot code review or merge approval. -- **Maintainer decisions needed:** List the specific questions or dispositions - for a human to approve. No proposal is accepted by this report. + assessment time, totals, and whether coverage is complete, partial, or + unavailable. Open with a short, welcoming overview of what was found, why it + matters, and what still needs attention. Explain errors and gaps; never invent + missing data. This assessment is not a GitHub Copilot code review or merge + approval. Record the policy commit and link to the GOALS.md version used, + or state why that policy could not be established. +- **Suggested starting point:** Introduce a small review queue in plain language. + Explain why each contribution is worth reviewing next and what would help + that review. Do not present the queue as an approved implementation plan. +- **Issue inventory:** One linked subsection per discovered open issue, including + relevant PR links. Explain the user's need, what the discussion has established, + and the suggested next step in short paragraphs. Include the evidence and + unresolved questions. Include its direction classification and policy citation + with a plain-language reason. Mark unassessed issues explicitly. +- **Candidate acceptance criteria:** Explain that these are the checks needed + before considering a change complete. For the first few actionable candidates, + use separate lists labeled "Existing requirements" and "Suggested additions" + so readers can distinguish recorded requirements from new suggestions. State + who proposed or approved them; a request in an issue is not itself approval. + Suggest implementation criteria only for in-scope work. For other items, + preserve recorded requirements as evidence and identify the owner decision + needed before implementation criteria can be agreed. +- **Existing pull requests:** Describe each contribution, its current review + position, and what would help it move forward. Keep contributor-reported + results separate from checks actually observed. Link to supporting evidence. + Include each PR's direction classification and policy citation with a + plain-language reason, including PRs not linked to an open issue. +- **Maintainer decisions needed:** Ask specific questions in complete sentences + and explain why each decision matters. Identify questions requiring the + direction owner's approval separately from ordinary maintainer review. + No proposal is accepted by this report. +- **Evidence and coverage notes:** Record pages/items read, reported totals, + source links, errors, and work not completed. Keep the detailed evidence here + without hiding important limits from the opening summary. If a limit or data-access error prevents a complete assessment, produce an explicitly partial report with the remaining work identified. Finish with a diff --git a/AUTOMATION.md b/AUTOMATION.md index cfcb55b6..42aad0a0 100644 --- a/AUTOMATION.md +++ b/AUTOMATION.md @@ -7,27 +7,40 @@ of the project. Our goal is to reduce routine coordination and maintenance work so contributors and maintainers can focus on useful ideas, design decisions, and quality. +Automation must follow [GOALS.md](./GOALS.md), the primary repository policy for +product scope. It can help carry out the project's direction, not choose a new +one. Changes to that policy require explicit human owner approval. + This is an early experiment. The workflow below describes our intended direction; capabilities will be introduced incrementally, and not everything described here is available today. ## Status and implementation history -**Last updated: 2026-10-27.** The first pilot is under review in -[#482](https://github.com/microsoft/TypeChat/pull/482). A compilation -reproducibility fix is validated locally. Updated GitHub CI checks, maintainer -review, merge to the default branch, confirmation of GitHub Copilot access, -and the first live assessment are still pending. -No live repository settings have been changed as part of this pilot. +**Last updated: 2026-10-27 (UTC).** The first pilot is merged into `main` through +[#482](https://github.com/microsoft/TypeChat/pull/482). +[Post-merge workflow checks](https://github.com/microsoft/TypeChat/actions/runs/40454115457) +passed, and the +[Backlog assessment workflow](https://github.com/microsoft/TypeChat/actions/workflows/backlog-assessment.lock.yml) +is available for manual dispatch. The +[first live assessment](https://github.com/microsoft/TypeChat/actions/runs/40461739316) +completed successfully, verifying live Copilot access and the configured model +and main-assessment reasoning effort. Its draft enumerates 24 issues and 26 PRs, +but PR investigation is partial. Human review remains required; automatic +Copilot PR reviews and broader autonomy are not enabled by this pilot. | Implemented | Change | Current state | | --- | --- | --- | | 2026-10-26 | Community-facing plan and [README announcement](./README.md#repository-automation-experiment) | Documented. | -| 2026-10-26 | Shared [Copilot contributor and review guidance](./.github/copilot-instructions.md) | Prepared; instructions do not configure automatic PR reviews. | -| 2026-10-26 | [Manual backlog assessment](./.github/workflows/backlog-assessment.md) | Implemented; produces a draft report without changing issues or PRs. First GitHub run pending. | -| 2026-10-26 | [Compilation checks](./.github/workflows/agent-workflow-check.yml) and [contract tests](./tools/tests/backlog-assessment.test.mjs) | Strict compilation and 17 tests passed locally. Updated GitHub CI verification pending. | -| 2026-10-26 | [Assessment model and reasoning effort](#model-selection) | GPT-6.1-sol pinned with `max` effort for the main assessment only; threat detection keeps its default effort. Live model access unverified. | -| 2026-10-27 | [Compiler action pins](./.github/aw/actions-lock.json) and clearer drift diagnostics | The runtime pin is checked in with the generated workflow, keeping local and CI compilation consistent. | +| 2026-10-26 | Shared [Copilot contributor and review guidance](./.github/copilot-instructions.md) | Merged; instructions do not configure automatic PR reviews. | +| 2026-10-26 | [Manual backlog assessment](./.github/workflows/backlog-assessment.md) | Available on `main` since 2026-10-27 (UTC), via [#482](https://github.com/microsoft/TypeChat/pull/482). First live report produced; maintainer review required. | +| 2026-10-26 | [Compilation checks](./.github/workflows/agent-workflow-check.yml) and [contract tests](./tools/tests/backlog-assessment.test.mjs) | Verified in [post-merge CI](https://github.com/microsoft/TypeChat/actions/runs/40454115457) on 2026-10-27 (UTC): strict compilation, no generated-file drift, and 17 passing tests. | +| 2026-10-26 | [Assessment model and reasoning effort](#model-selection) | GPT-6.1-sol and main-assessment `max` effort verified in the first live run on 2026-10-27 (UTC); threat detection keeps its separately configured default effort. | +| 2026-10-27 | [Compiler action pins](./.github/aw/actions-lock.json) and clearer drift diagnostics | Merged; the post-merge drift check passed on GitHub. | +| 2026-10-27 | [First live assessment](https://github.com/microsoft/TypeChat/actions/runs/40461739316) | Successful execution and report upload. All 24 issues and 26 PRs enumerated; selective PR investigation means overall coverage remains partial. | +| 2026-10-27 | [Clearer report language](#report-language) | Writing guidance and a regression test prepared for review. A live run with the revised guidance is still pending. | +| 2026-10-27 | [Assessment budget](#inference-budget) | Main assessment increased from 200 to 400 AI credits; detection keeps its separate default. Prepared for review, not yet active on GitHub. | +| 2026-10-27 | [Owner-controlled project direction](./GOALS.md) | Policy, assessment scope checks, shared guidance, and [CODEOWNERS](./.github/CODEOWNERS) prepared for owner review. GitHub approval enforcement and a live assessment using the policy remain unverified. | | Not yet | Automatic Copilot PR reviews and human approval gates | Live settings need verification and a maintainer-approved rollout. | | Not yet | Agent-assisted discussion and explicit issue acceptance | Planned; no issue-discussion or acceptance commands are enabled by this pilot. | | Not yet | Human-approved agent implementation | Planned; the assessment does not start implementation or open PRs. | @@ -48,12 +61,43 @@ Agents may investigate problems, summarize discussions, suggest questions, write tests, and propose changes. Automated tests and GitHub Copilot code reviews will support review, not replace human judgment. +### Protecting project direction + +The direction owner is `@microsoft/typechat`. Goals, non-goals, and the +[items under consideration](./GOALS.md#under-consideration) can change only +through an owner-approved policy pull request. Under consideration does not mean +approved for implementation. Ordinary work already within the goals follows +normal maintainer review. + +The assessment reads the policy at its trusted default-branch run commit. Each +issue and PR receives an **In scope**, **Needs owner decision**, or **Conflicts +with direction** classification, with a plain-language reason and a policy +citation. An unreadable policy or uncertain fit is not permission to recommend +implementation. Proposed policy edits in unmerged PRs do not replace the active +policy. + +[CODEOWNERS](./.github/CODEOWNERS) covers the policy, this automation document, +any `AGENTS.md` guidance, the `.github/` controls, and repository automation tools +and tests. These local rules request owner review; they do not enable required +reviews themselves. +An administrator must verify the team has the required visibility and write +access, require pull requests and code-owner approval, prevent stale approvals, +and restrict bypasses. The +[enforcement checklist](./GOALS.md#approval-and-enforcement) records these +requirements. No live settings were changed for this policy work. + +The first policy PR needs deliberate human owner review: GitHub uses the +CODEOWNERS file already on the base branch, not the proposed one. Record +activation only after the protections have been verified on GitHub. + ## First pilot: a draft backlog assessment The first pilot is a [manually triggered assessment workflow](./.github/workflows/backlog-assessment.md). -Once it has been reviewed, merged to the default branch, and its GitHub Copilot -access enabled, a maintainer can select **Actions > Backlog assessment > Run -workflow** on the default branch. +It is now available on the default branch. Once the relevant organization +Copilot access is enabled, a maintainer can select +**Actions > Backlog assessment > Run workflow**, using `main`. Every report +remains a draft until maintainer review; successful execution alone is not +a complete backlog review or approval to act. The workflow reads open issues, their discussions, and existing pull requests. It proposes next steps and acceptance criteria in a draft report. It does not @@ -74,6 +118,76 @@ the per-run Actions token, so the relevant organization policy must allow it. No personal token or bot write credentials are needed for this read-only pilot. It has time, turn, and inference-budget limits and no scheduled trigger. +### Report retention + +The downloadable **backlog-assessment** report is kept for **seven days** by +default. The [source workflow](./.github/workflows/backlog-assessment.md) sets +`retention-days: 7`. In the generated workflow, +`GH_AW_DEFAULT_ARTIFACT_RETENTION_DAYS` overrides that value when set, subject +to GitHub Actions retention limits. Other artifacts, such as execution logs and +usage records, have their own retention settings; they do not extend the report's +lifetime. + +The workflow does not save reports in the repository. Keeping selected reports +there would be a separate, maintainer-approved step. A preserved report should +be a dated snapshot with its run link, assessed commit, coverage limits, and +review status. Review its contents through a normal pull request, rather than +automatically committing reports or raw execution logs. Preserving a report +does not approve its recommendations or make its findings current. + +### Report language + +Reports should be easy to follow even when English is not the reader's first +language. The revised [writing guidance](./.github/workflows/backlog-assessment.md) +asks the agent to explain what each issue means for users, acknowledge existing +contributions, and describe a practical next step in short paragraphs. +Technical terms should be explained, not removed. Detailed evidence belongs in +clearly labeled notes rather than wide tables full of compressed phrases. + +Clearer language must keep the same facts and limits. Reports still distinguish +what a contributor says from checks the assessment actually inspected. They +still identify incomplete coverage and decisions that need maintainer approval. +The writing change does not alter tools, permissions, model settings, or budgets. +Those follow-ups remain separate work. + +### First live assessment + +[Run 40461739316](https://github.com/microsoft/TypeChat/actions/runs/40461739316) +completed on 2026-10-27 (UTC). The draft contains 24 issue rows and 26 open-PR +rows. It investigated 11 issue-linked PRs more deeply, but left formal reviews +and head checks on the other 15 PRs, and inline review threads, for follow-up. +This is a useful review inventory, not a complete code review or merge approval. + +Reported usage was 188.67 AI credits for the assessment and 7.43 for threat +detection, approximately 196.10 in total. These are reported usage units, not a +currency charge. That run used the earlier 200-credit main-assessment limit. +Limits are per component, not a combined workflow budget. + +Two follow-up checks remain before expanding autonomy: + +- **Tool exposure:** Copilot CLI's `--allow-tool` controls permission approval, + not a strict available-tool filter. Additional read-only search tools were + used. Preserve the read-only repository guards and explicitly validate the + intended available-tool set before adding capabilities. See + [Copilot CLI tool controls](https://docs.github.com/en/copilot/how-tos/copilot-cli/use-copilot-cli/allowing-tools). +- **Telemetry quality:** The built-in grader's failure and loop totals conflict + with the detailed activity records. Reconcile those measurements before using + them as automated acceptance gates; a grader summary is not code-review evidence. + +### Inference budget + +**Changed on 2026-10-27:** The main assessment is configured for **400 AI credits** +per run through `max-ai-credits` in the +[source workflow](./.github/workflows/backlog-assessment.md), increased from 200. +The change takes effect after review and merge of the source and generated +workflow. Its 20-minute and 120-turn limits are unchanged. + +Threat detection has a separate **400-credit default**, controlled by +`GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS`. That setting is unchanged. The +400-credit assessment budget is therefore not a cap for the entire workflow: +the two default component budgets total 800 credits. Actual usage may be lower, +and other configured guardrails can still prevent a run. + ### Model selection **Pinned on 2026-10-26:** The main assessment uses `gpt-6.1-sol` with `max` @@ -90,8 +204,9 @@ engine: `engine.model` pins the model; `engine.args` passes the native Copilot CLI `--reasoning-effort max` option. The generated assessment no longer falls back to `GH_AW_MODEL_AGENT_COPILOT`, `GH_AW_DEFAULT_MODEL_COPILOT`, or `auto`. -The existing time, turn, and inference-budget limits are unchanged; maximum -effort can use more time and credits within those limits. +The model and effort settings do not replace the separate time, turn, and +[credit limits](#inference-budget). Maximum effort can use more time and credits +within those limits. The separate threat-detection job inherits `gpt-6.1-sol`, but not the main assessment's extra CLI arguments. By design, `max` applies only to the main @@ -100,9 +215,10 @@ Its explicit inherited model also replaces its previous model-variable fallback. To change either setting, edit the Markdown source, regenerate the compiled workflow, and update the model contract test and this documentation. The first -approved GitHub run must still verify that the model is available and permitted -for the workflow's Copilot access. Local compilation and tests verify the -configuration, not live inference or model access. See the +live run verified the pinned model and main-assessment `max` setting on +2026-10-27 (UTC). Local compilation and tests verify configuration, not continued +model availability or organization access; recheck live access when changing +the configuration. See the [engine configuration reference](https://github.github.com/gh-aw/reference/engines/#extended-coding-agent-configuration) and [Copilot model behavior](https://github.github.com/gh-aw/reference/engines/copilot/#capabilities-and-limitations). Editor chat or subagent model preferences do not configure this workflow, and @@ -142,8 +258,12 @@ We aim to keep the entire process on GitHub: Discussions for questions and exploratory conversations. 2. **Clarify the proposal.** Contributors, maintainers, and agents help establish the intended outcome, scope, non-goals, and testable acceptance criteria. -3. **Accept the work.** A maintainer approves the agreed scope before - implementation begins. Material scope changes return to discussion. + Check the proposal against [GOALS.md](./GOALS.md). Unclear fit and items under + consideration need an owner decision, not an assumption of acceptance. +3. **Accept the work.** A maintainer approves agreed, in-scope work before + implementation begins. A direction change or exception needs an + owner-approved policy change merged first. Material scope changes return + to discussion. 4. **Propose an implementation.** A contributor or agent opens a pull request linked to the accepted work, with tests and supporting documentation. 5. **Review and accept the result.** Automated checks, GitHub Copilot review, and @@ -174,6 +294,11 @@ The criteria for automatic adoption will be discussed and published before that behavior is enabled. Participation, elapsed time, or an agent's recommendation alone will not guarantee acceptance. +Any future automatic adoption must remain within the owner-approved goals. +Changing those goals, making exceptions, or promoting an item under +consideration into accepted scope will continue to require explicit human +owner approval, even if routine work becomes more autonomous. + Ambiguous proposals and higher-impact decisions, including significant API changes, security-sensitive work, and releases, will continue to involve humans. diff --git a/GOALS.md b/GOALS.md new file mode 100644 index 00000000..0a14fcde --- /dev/null +++ b/GOALS.md @@ -0,0 +1,123 @@ +# TypeChat project direction + +TypeChat helps developers build natural-language interfaces using +application-defined types. This document is the primary repository policy for +product scope: what belongs in TypeChat, what does not, and who can change those +boundaries. + +It guides contributors, maintainers, and agents. The +[automation experiment](./AUTOMATION.md), issue acceptance criteria, and agent +recommendations must stay within it. It does not override organization, +platform, security, or legal requirements. + +## Goals + +- **G1. Types define the interface.** Keep TypeChat a focused library for building + natural-language interfaces using application-defined types. +- **G2. Validation remains reliable.** Produce validated structured data with + accurate schemas, explicit errors, and controlled repair. +- **G3. APIs remain practical to maintain.** Keep APIs composable, maintainable, + and compatible. Preserve the existing TypeScript and Python implementations + and their documented behavior. +- **G4. Integration does not require platform lock-in.** Support practical + integrations through clear interfaces and examples, without unnecessary + dependencies or vendor lock-in. +- **G5. People can learn and verify the behavior.** Prioritize useful + documentation, accessible examples, and deterministic regression tests. + +## Non-goals + +- **N1. A general-purpose agent platform.** Do not turn TypeChat into a platform + with built-in planning, memory, multi-agent orchestration, hosting, storage, or + deployment management. Applications can use TypeChat alongside other tools + without moving those responsibilities into TypeChat. +- **N2. Application-specific business logic in the core.** Keep that logic in + the application, not in the core library. +- **N3. A core tied to one vendor's platform.** Do not add provider-specific + product features or integrations that make the core depend on a particular + vendor's platform. +- **N4. Convenience at the expense of validation.** Do not weaken schema or + runtime validation, or silently accept unsupported types. +- **N5. Features accepted solely through popularity or AI recommendations.** + Community interest and agent suggestions can inform a discussion. They do not + decide project direction or authorize a feature. + +Existing supported integrations and examples are not automatically deprecated +by these boundaries. Preserve existing contributions and contributor credit. +If an existing behavior appears to conflict with this policy, ask the owner to +resolve the difference rather than removing it or closing related issues +automatically. + +## Under consideration + +Under consideration is not approval to implement, a roadmap commitment, or a +change to the goals above. Ideas in this section need an explicit owner decision +before they can become accepted implementation work. + +- **Additional language support.** We may consider implementations beyond the + existing TypeScript and Python libraries. A proposal should explain who it + serves, how it preserves TypeChat's validation guarantees, and who will + maintain, test, and release it. +- **The relationship with TypeChat.NET.** The related + [TypeChat.NET project](https://github.com/microsoft/TypeChat.net) already exists. + We may consider how additional language support and that work fit together. + Listing it here does not approve moving it into this repository, changing + its ownership, or committing to shared releases. + +## Who decides + +The project direction owner is **@microsoft/typechat**. Direction decisions +require explicit approval from a human member of that owner team. + +Every change to this document requires explicit owner approval, including +changes to the goals, non-goals, ownership rules, and items under consideration. +Propose a change through a pull request that explains the reason, intended +benefit, and effect on existing users and maintainers. A scope change or +exception must be approved and merged here before dependent implementation is +accepted. + +The active policy is the version on the default branch, not proposed edits in +an unmerged pull request. An assessment should cite the policy at the trusted +default-branch commit used for its run. When that policy is missing, unreadable, +or unclear about a proposal, record the problem and seek an owner decision; +do not treat the gap as permission to expand scope. + +Community votes, labels, an AI recommendation, passing checks, or an unrelated +approval comment do not amend this policy. GitHub Copilot reviews support human +review; they do not provide owner approval. + +Ordinary in-scope library maintenance follows the normal maintainer review +process. It does not need a new project-direction decision for every fix. +Approval to pursue work is still separate from approval to merge its +implementation. Material scope changes return to discussion. + +## Approval and enforcement + +The [CODEOWNERS file](./.github/CODEOWNERS) requests review from the owner team +for this policy, [AUTOMATION.md](./AUTOMATION.md), any `AGENTS.md` guidance, +the `.github/` controls +(including CODEOWNERS itself, agent instructions, and workflows), and the +repository automation tools and tests in `tools/`. It does not assign all +library changes to the direction owner. + +**Prepared on 2026-10-27 (UTC).** These repository files do not by themselves +make owner approval mandatory in GitHub. No live settings were changed as part +of this work. A repository administrator still needs to verify: + +- The owner team is visible and has explicit write access to this repository. +- The default branch requires pull requests and code-owner approval. +- New changes cannot rely on stale approval; configure dismissal of stale + approvals and approval of the most recent reviewable push. +- Bypass permissions and direct pushes are restricted so people, bots, and apps + cannot silently skip owner review. + +The first pull request introducing these rules needs deliberate human approval +from the owner team. GitHub uses CODEOWNERS from the pull request's base branch, +so a new CODEOWNERS file does not enforce its own initial adoption. + +Record verified activation dates in the +[automation status history](./AUTOMATION.md#status-and-implementation-history). +Until then, distinguish the written approval requirement from GitHub-enforced +protection. See GitHub's +[code owner documentation](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners) +for the platform requirements. diff --git a/README.md b/README.md index 4ab6484f..6e392a6f 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,15 @@ To see TypeChat in action, we recommend exploring the [TypeChat example projects To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat) which includes more information on TypeChat and how to get started. +## Project direction + +[GOALS.md](./GOALS.md) defines what belongs in TypeChat, what does not, and what +is still under consideration. It is the primary repository policy for product +scope. Changes require explicit human approval from the `@microsoft/typechat` +owner team; community interest and AI recommendations do not change it. +Read it before proposing a new direction. Ordinary in-scope maintenance follows +the normal contribution process. + ## Repository automation experiment We are experimenting with a gradual transition to a semi-automated repository. @@ -59,17 +68,26 @@ proposals progress from discussion to accepted, reviewed changes with less manual coordination. Read about the [automation experiment](./AUTOMATION.md) and how to participate. -**Status as of 2026-10-27:** The first read-only pilot is under review in -[#482](https://github.com/microsoft/TypeChat/pull/482). A compilation -reproducibility fix is validated locally; updated GitHub CI checks, merge, -and the first live assessment are still pending. +**Status as of 2026-10-27 (UTC):** The first read-only pilot is merged in +[#482](https://github.com/microsoft/TypeChat/pull/482). +[Post-merge workflow checks](https://github.com/microsoft/TypeChat/actions/runs/40454115457) +passed, and the +[manual backlog assessment](https://github.com/microsoft/TypeChat/actions/workflows/backlog-assessment.lock.yml) +is available on `main`. The +[first live assessment](https://github.com/microsoft/TypeChat/actions/runs/40461739316) +succeeded and verified model access. Its draft covers 24 issues and 26 PRs, +with partial PR investigation; maintainer review and follow-up remain required. | Milestone | Implemented | Current status | | --- | --- | --- | | Community-facing automation plan | 2026-10-26 | Documented in [AUTOMATION.md](./AUTOMATION.md). | -| Manual backlog assessment, Copilot guidance, and workflow checks | 2026-10-26 | Implemented; awaiting maintainer review, merge, and first live assessment. | -| Assessment model and reasoning effort | 2026-10-26 | Pinned to GPT-6.1-sol with `max` effort for the main assessment only. Live model access still needs verification. | -| Reproducible workflow compilation | 2026-10-27 | Compiler action pins are checked in, with clearer drift diagnostics; 17 local tests passed. Updated GitHub CI verification pending. | +| Manual backlog assessment, Copilot guidance, and workflow checks | 2026-10-26 | Merged via [#482](https://github.com/microsoft/TypeChat/pull/482) on 2026-10-27 (UTC); manual assessment available on `main`. | +| Assessment model and reasoning effort | 2026-10-26 | GPT-6.1-sol and main-assessment `max` effort verified in the first live run on 2026-10-27 (UTC). | +| Reproducible workflow compilation | 2026-10-27 | [Verified on GitHub](https://github.com/microsoft/TypeChat/actions/runs/40454115457): no generated-file drift and 17 passing tests. | +| First live backlog assessment | 2026-10-27 | [Completed successfully](https://github.com/microsoft/TypeChat/actions/runs/40461739316); 24 issues and 26 PRs enumerated. PR investigation is partial; no dispositions are approved. | +| Clearer report language | 2026-10-27 | [Plain-language writing guidance](./AUTOMATION.md#report-language) prepared for review; the next live run still needs to confirm the new style. | +| Assessment budget | 2026-10-27 | [Main-assessment budget](./AUTOMATION.md#inference-budget) raised to 400 AI credits; detection is budgeted separately. Review and merge pending. | +| Owner-controlled project direction | 2026-10-27 | [Goals, non-goals, and items under consideration](./GOALS.md), assessment scope checks, and [CODEOWNERS](./.github/CODEOWNERS) prepared for owner review. GitHub enforcement still needs verification. | | Automatic Copilot PR reviews and human approval gates | Not yet | Next rollout step; live repository settings still need verification. | | GitHub-native discussion, acceptance, and agent implementation | Not yet | Planned, with human approval initially. | | Selective automatic adoption of community proposals | Not yet | Future work, subject to public criteria and maintainer approval. | @@ -81,6 +99,10 @@ maintenance and deployment workflows are unchanged. See the ## Contributing +Start with the [project goals and non-goals](./GOALS.md). Proposals that would +change these boundaries need an owner-approved policy change before their +implementation can be accepted. + **Repository automation experiment:** We are experimenting with a gradual transition to a semi-automated repository. The initial agent-assisted workflow keeps humans in the loop, while our longer-term goal is to help community diff --git a/tools/tests/backlog-assessment.test.mjs b/tools/tests/backlog-assessment.test.mjs index bc86fe38..7b433f5f 100644 --- a/tools/tests/backlog-assessment.test.mjs +++ b/tools/tests/backlog-assessment.test.mjs @@ -45,6 +45,104 @@ test("assessment is manual and restricted to a user on the default branch", () = assert.ok(compiled.jobs.activation.if.includes(source.if)); }); +test("report instructions favor plain language without dropping evidence", () => { + const style = /## Writing style\r?\n([\s\S]*?)\r?\n## Report/.exec(markdown); + assert.ok(style, "Keep explicit writing guidance for the report"); + const guidance = style[1].replace(/\s+/g, " "); + for (const requirement of [ + "English is not their first language", + "short sentences and paragraphs", + "Explain technical terms and abbreviations on first use", + "Preserve facts, evidence links, uncertainty, and existing maintainer decisions", + ]) { + assert.ok(guidance.includes(requirement), `Missing writing guidance: ${requirement}`); + } + + const report = markdown.split(/\r?\n## Report\r?\n/)[1].replace(/\s+/g, " "); + assert.ok(report.includes("One linked subsection per discovered open issue")); + assert.ok(report.includes("Keep contributor-reported results separate from checks actually observed")); + assert.ok(report.includes("No proposal is accepted by this report")); + assert.ok(!report.includes("One row per discovered open issue")); + + const prompt = compiled.jobs.activation.steps.find(step => step.name === "Create prompt with built-in context"); + assert.ok(prompt); + assert.ok(Object.values(prompt.env).includes("{{#runtime-import .github/workflows/backlog-assessment.md}}\n"), + "The workflow must load the report instructions at runtime"); +}); + +test("project direction names its owner and separates consideration from approval", () => { + const goals = fs.readFileSync(new URL("../../GOALS.md", workflowDirectory), "utf8"); + for (const heading of ["Goals", "Non-goals", "Under consideration", "Who decides", "Approval and enforcement"]) { + assert.match(goals, new RegExp(`^## ${heading}$`, "m")); + } + assert.ok(goals.includes("@microsoft/typechat")); + const links = [...goals.matchAll(/\[[^\]]+\]\(([^)]+)\)/g)]; + assert.ok(links.some(([, url]) => url === "https://github.com/microsoft/TypeChat.net"), + "Keep a link to the existing TypeChat.NET project"); + assert.ok(goals.replace(/\s+/g, " ").includes("Under consideration is not approval to implement")); + assert.ok(goals.replace(/\s+/g, " ").includes("Every change to this document requires explicit owner approval")); + + for (const file of ["../../README.md", "../../AUTOMATION.md", "../copilot-instructions.md"]) { + const guidance = fs.readFileSync(new URL(file, workflowDirectory), "utf8"); + assert.match(guidance, /\]\((?:\.\/|\.\.\/)GOALS\.md(?:#[^)]+)?\)/, file); + } +}); + +test("assessment applies the run's direction policy before recommending work", () => { + const section = /## Project direction\r?\n([\s\S]*?)\r?\n## Boundaries/.exec(markdown); + assert.ok(section, "Load project direction before assessing community proposals"); + const guidance = section[1].replace(/\s+/g, " "); + for (const requirement of [ + "Read GOALS.md at `${{ env.TYPECHAT_POLICY_COMMIT }}`", + "primary repository policy for product scope", + "Do not use proposed policy edits in an issue or PR as the active policy", + "For every issue and PR", + "In scope", + "Needs owner decision", + "Conflicts with direction", + "If GOALS.md cannot be read or its revision cannot be established", + "do not recommend implementation or acceptance", + ]) { + assert.ok(guidance.includes(requirement), `Missing direction safeguard: ${requirement}`); + } + const report = markdown.split(/\r?\n## Report\r?\n/)[1].replace(/\s+/g, " "); + assert.ok(report.includes("Include its direction classification and policy citation")); + assert.ok(report.includes("Include each PR's direction classification and policy citation")); +}); + +test("policy revision is wired from the trusted run commit into the prompt", () => { + assert.deepEqual(source.env, { TYPECHAT_POLICY_COMMIT: "${{ github.sha }}" }); + assert.equal(compiled.env.TYPECHAT_POLICY_COMMIT, "${{ github.sha }}"); + for (const name of ["Create prompt with built-in context", "Interpolate variables and render templates", "Substitute placeholders"]) { + const step = compiled.jobs.activation.steps.find(step => step.name === name); + assert.ok(step); + assert.equal(step.env.GH_AW_ENV_TYPECHAT_POLICY_COMMIT, "${{ env.TYPECHAT_POLICY_COMMIT }}", name); + } + const substitute = compiled.jobs.activation.steps.find(step => step.name === "Substitute placeholders"); + assert.match(substitute.with.script, + /^\s*GH_AW_ENV_TYPECHAT_POLICY_COMMIT: process\.env\.GH_AW_ENV_TYPECHAT_POLICY_COMMIT,$/m); +}); + +test("direction ownership protects the policy and automation controls", () => { + const owners = fs.readFileSync(new URL("../CODEOWNERS", workflowDirectory), "utf8") + .split(/\r?\n/) + .map(line => line.trim()) + .filter(line => line && !line.startsWith("#")) + .map(line => line.split(/\s+/)); + assert.deepEqual(owners, [ + ["/GOALS.md", "@microsoft/typechat"], + ["/AUTOMATION.md", "@microsoft/typechat"], + ["AGENTS.md", "@microsoft/typechat"], + ["/.github/", "@microsoft/typechat"], + ["/tools/", "@microsoft/typechat"], + ]); + for (const event of ["push", "pull_request"]) { + for (const file of ["GOALS.md", "AUTOMATION.md", "README.md", ".github/CODEOWNERS", ".github/copilot-instructions.md"]) { + assert.ok(checks.on[event].paths.includes(file), `${event} must check changes to ${file}`); + } + } +}); + test("assessment pins its model and max effort without overriding detection effort", () => { assert.deepEqual(source.engine, { id: "copilot", @@ -169,10 +267,30 @@ test("generated helper jobs respect the slim runner time limit", () => { } }); +test("the assessment has a 400-credit budget separate from detection", () => { + assert.equal(source["max-ai-credits"], 400); + const info = compiled.jobs.activation.steps.find(step => step.name === "Generate agentic run info"); + assert.ok(info); + assert.equal(info.env.GH_AW_INFO_MAX_AI_CREDITS, "400"); + + const agent = compiled.jobs.agent.steps.find(step => step.name === "Execute GitHub Copilot CLI"); + assert.ok(agent); + assert.match(agent.run, /^\s*GH_AW_MAX_AI_CREDITS="400"$/m); + const detection = compiled.jobs.detection.steps.find(step => step.name === "Execute threat detection with AWF"); + assert.ok(detection); + assert.equal(detection.env.GH_AW_MAX_AI_CREDITS, + "${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }}"); + + const budgetSteps = steps.filter(step => step !== detection && step.env?.GH_AW_MAX_AI_CREDITS !== undefined); + assert.ok(budgetSteps.length > 0, "Generated budget checks and reporting must include the assessment limit"); + for (const step of budgetSteps) { + assert.equal(step.env.GH_AW_MAX_AI_CREDITS, "400", step.name); + } +}); + test("the run and its report have explicit bounds and no missing-file success", () => { assert.equal(source["timeout-minutes"], 20); assert.equal(source["max-turns"], 120); - assert.equal(source["max-ai-credits"], 200); assert.deepEqual(source.concurrency, { group: "backlog-assessment", "cancel-in-progress": false }); const verify = compiled.jobs.agent.steps.find(step => step.name === "Verify draft report"); const upload = compiled.jobs.agent.steps.find(step => step.name === "Upload draft report");