diff --git a/internal/sync/local/adapter_safety_test.go b/internal/sync/local/adapter_safety_test.go new file mode 100644 index 000000000..3ca003b96 --- /dev/null +++ b/internal/sync/local/adapter_safety_test.go @@ -0,0 +1,634 @@ +package local + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncadapter "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +func TestReferenceDescriptorValidationFailures(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + for name, content := range map[string]string{ + "invalid YAML": "formatVersion: [", + "unknown field": "formatVersion: 1\nfile: prompt.md\nformat: markdown\nunknown: true\n", + "multiple docs": "formatVersion: 1\nfile: prompt.md\nformat: markdown\n---\nextra: true\n", + "unsupported format": "formatVersion: 99\nfile: prompt.md\nformat: markdown\n", + "missing file": "formatVersion: 1\nformat: markdown\n", + } { + t.Run(name, func(t *testing.T) { + fsys := fstest.MapFS{"value.ref.yaml": {Data: []byte(content)}} + _, err := ReadDescriptor(fsys, "value.ref.yaml", id) + require.Error(t, err) + }) + } + _, err := ReadDescriptor(fstest.MapFS{}, "missing.ref.yaml", id) + require.Error(t, err) +} + +func TestReferencePathValidationFailures(t *testing.T) { + for name, reference := range map[string]Reference{ + "missing file": {Format: "markdown"}, + "missing format": {File: "prompt.md"}, + "absolute": {File: "/prompt.md", Format: "markdown"}, + "managed": {File: ".launchdarkly/prompt.md", Format: "markdown"}, + "unclean": {File: "dir/../prompt.md", Format: "markdown"}, + } { + t.Run(name, func(t *testing.T) { + require.Error(t, validateReference(reference)) + }) + } + + root := t.TempDir() + writeTestFile(t, root, "directory/file.md", "body") + _, err := resolveSourcePath(root, "missing.md") + require.Error(t, err) + _, err = resolveSourcePath(root, "directory") + require.Error(t, err) + + outside := filepath.Join(t.TempDir(), "outside.md") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + link := filepath.Join(root, "link.md") + if err := os.Symlink(outside, link); err == nil { + _, err = resolveSourcePath(root, "link.md") + require.Error(t, err) + } + + _, err = readSourceFileFromFS(fstest.MapFS{}, "") + require.Error(t, err) + _, err = readSourceFileFromFS(fstest.MapFS{}, "prompt.md") + require.Error(t, err) + data, err := readSourceFileFromFS(fstest.MapFS{"prompt.md": {Data: []byte("Help")}}, "prompt.md") + require.NoError(t, err) + assert.Equal(t, "Help", string(data.Data)) +} + +func TestNewSourceReferenceValidatesRepositoryBoundaries(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "work"), 0o755)) + writeTestFile(t, root, "work/prompt.md", "Help") + + _, err := NewSourceReference(root, filepath.Join(root, "work"), "", "markdown", "", nil) + require.Error(t, err) + _, err = NewSourceReference(root, filepath.Join(root, "work"), "missing.md", "markdown", "", nil) + require.Error(t, err) + + outside := filepath.Join(t.TempDir(), "outside.md") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + _, err = NewSourceReference(root, root, outside, "markdown", "", nil) + require.Error(t, err) + + reference, err := NewSourceReference(root, filepath.Join(root, "work"), "prompt.md", "markdown", "", nil) + require.NoError(t, err) + assert.Equal(t, "work/prompt.md", reference.File) +} + +func TestSourceDefinitionAndCompileSafety(t *testing.T) { + assert.Empty(t, (SourceDefinition{}).ID()) + variationID := syncdomain.VariationID("project", "config", "default") + variation := syncdomain.SyncedResource{Kind: variationID.Kind, ProjectKey: variationID.ProjectKey, LookupKey: variationID.LookupKey} + assert.Equal(t, variationID, (SourceDefinition{Variation: &variation}).ID()) + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + attachment := syncdomain.SyncedAttachment{ + ProjectKey: "project", + Attachment: syncdomain.Attachment{Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{Key: "tool"}}, + } + assert.Equal(t, toolID, (SourceDefinition{Attachment: &attachment}).ID()) + + root := t.TempDir() + writeTestFile(t, root, "value.json", `{"description":"Tool"}`) + definition, err := CompileSource(root, toolID, Reference{ + File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}, + }) + require.NoError(t, err) + require.NotNil(t, definition.Attachment) + + _, err = CompileSource(root, syncdomain.ResourceID{Kind: "unknown"}, Reference{ + File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}, + }) + require.Error(t, err) + + generated := syncadapter.SourceBundle{Primary: syncadapter.Document{ + Path: "generated.py", Data: []byte("# Code generated. DO NOT EDIT.\n"), + }} + _, err = compileSource(variationID, Reference{File: "generated.py", Format: "python"}, generated) + require.Error(t, err) + _, err = compileSource(toolID, Reference{File: "generated.py", Format: "python"}, generated) + require.Error(t, err) + _, err = compileSource( + syncdomain.ResourceID{Kind: "unknown", ProjectKey: "project", LookupKey: "value"}, + Reference{File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}}, + syncadapter.SourceBundle{Primary: syncadapter.Document{Path: "value.json", Data: []byte(`{"description":"value"}`)}}, + ) + require.Error(t, err) +} + +func TestSourceBundleHandlesMissingAndFailedDependencies(t *testing.T) { + reference := Reference{File: "app/main.py", Format: "python"} + content := []byte("from .values import PROMPT\n") + bundle, err := sourceBundle(reference, content, func(string) (syncadapter.Document, error) { + return syncadapter.Document{}, fs.ErrNotExist + }) + require.NoError(t, err) + assert.Empty(t, bundle.Dependencies) + + expected := errors.New("read failed") + _, err = sourceBundle(reference, content, func(string) (syncadapter.Document, error) { + return syncadapter.Document{}, expected + }) + require.ErrorIs(t, err, expected) + + _, err = sourceBundle(Reference{File: "value", Format: "missing"}, nil, func(string) (syncadapter.Document, error) { + return syncadapter.Document{}, nil + }) + require.Error(t, err) + + _, err = sourceBundle(reference, content, func(dependency string) (syncadapter.Document, error) { + if strings.HasSuffix(dependency, "values.py") { + return syncadapter.Document{Data: make([]byte, 8<<20)}, nil + } + return syncadapter.Document{}, fs.ErrNotExist + }) + require.ErrorContains(t, err, "8 MiB") + + // The limit covers the primary file alone, with no dependency. + _, err = sourceBundle(Reference{File: "prompt.md", Format: "markdown"}, make([]byte, 8<<20+1), func(string) (syncadapter.Document, error) { + return syncadapter.Document{}, fs.ErrNotExist + }) + require.ErrorContains(t, err, "8 MiB") +} + +// A generated module beside an authored source is read, not written. An +// import of it that no mapped field uses does not block the resource. +func TestCompileWorkspaceIgnoresAnUnrelatedGeneratedImport(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/agent.py", "from agents import Agent\nfrom .schema_pb2 import SCHEMA\n"+ + "support = Agent(instructions=\"Help with orders.\")\n") + writeTestFile(t, root, "app/schema_pb2.py", "# Code generated by protoc. DO NOT EDIT.\nSCHEMA = {}\n") + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.ref.yaml", + "formatVersion: 1\nfile: app/agent.py\nformat: python\nselect: 'agents[id=\"support\"]'\n"+ + "map:\n prompt: instructions\n") + + resources, err := CompileWorkspace(root) + + require.NoError(t, err) + require.Len(t, resources, 1) + assert.Equal(t, "Help with orders.", resources[0].Variation.Instructions) +} + +// A mapped value that lives in a generated or vendored module is read-only, +// because sync never writes such a file. The resolved path of a link counts, +// as it does for the primary file. +func TestCompileSourceMarksAValueInAGeneratedDependencyReadOnly(t *testing.T) { + id := syncdomain.VariationID("project", "support", "default") + reference := Reference{ + File: "app/agent.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + agent := "from agents import Agent\nfrom .prompt import TEXT\nsupport = Agent(instructions=TEXT)\n" + + t.Run("generated header", func(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/agent.py", agent) + writeTestFile(t, root, "app/prompt.py", "# Code generated by protoc. DO NOT EDIT.\nTEXT = \"Help with orders.\"\n") + + definition, err := CompileSource(root, id, reference) + + require.NoError(t, err, "a generated dependency does not block the resource") + assert.Equal(t, "Help with orders.", definition.Variation.Variation.Instructions, "the value still reads") + state := definition.Variation.Projection.States()["prompt"] + assert.Equal(t, syncdomain.CapabilityReadOnly, state.Capability) + assert.Equal(t, syncdomain.ErrorSourceNotWritable, state.Code) + assert.Equal(t, "app/prompt.py", state.File) + assert.Contains(t, state.Reason, "app/prompt.py is generated, vendored, or minified") + }) + + t.Run("link into a vendored tree", func(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/agent.py", agent) + writeTestFile(t, root, "vendor/prompt.py", "TEXT = \"Help with orders.\"\n") + require.NoError(t, os.Symlink(filepath.Join(root, "vendor", "prompt.py"), filepath.Join(root, "app", "prompt.py"))) + + definition, err := CompileSource(root, id, reference) + + require.NoError(t, err) + state := definition.Variation.Projection.States()["prompt"] + assert.Equal(t, syncdomain.CapabilityReadOnly, state.Capability) + assert.Equal(t, "app/prompt.py", state.File) + }) + + t.Run("authored dependency stays direct", func(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/agent.py", agent) + writeTestFile(t, root, "app/prompt.py", "TEXT = \"Help with orders.\"\n") + + definition, err := CompileSource(root, id, reference) + + require.NoError(t, err) + assert.Equal(t, syncdomain.CapabilityDirect, definition.Variation.Projection.States()["prompt"].Capability) + }) +} + +func TestSourceEditHelpersEnforceAtomicSafety(t *testing.T) { + problem := publicationProblem("value.py", "unsafe", "fix it", assert.AnError) + typed, ok := syncdomain.ProblemFrom(problem) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorSourceNotWritable, typed.Code) + + original := []byte("abcdef") + updated, err := applySourceEdits("value.txt", original, []syncadapter.TextEdit{ + {File: "value.txt", Start: 4, End: 6, Expected: []byte("ef"), Replacement: []byte("EF")}, + {File: "value.txt", Start: 0, End: 2, Expected: []byte("ab"), Replacement: []byte("AB")}, + }) + require.NoError(t, err) + assert.Equal(t, "ABcdEF", string(updated)) + + for name, edits := range map[string][]syncadapter.TextEdit{ + "invalid range": {{File: "value.txt", Start: -1}}, + "stale bytes": {{File: "value.txt", Start: 0, End: 1, Expected: []byte("x")}}, + "overlap": { + {File: "value.txt", Start: 0, End: 3, Expected: []byte("abc")}, + {File: "value.txt", Start: 2, End: 4, Expected: []byte("cd")}, + }, + "two inserts at one offset": { + {File: "value.txt", Start: 2, End: 2, Replacement: []byte("X")}, + {File: "value.txt", Start: 2, End: 2, Replacement: []byte("Y")}, + }, + "insert at the start of a replacement": { + {File: "value.txt", Start: 2, End: 4, Expected: []byte("cd"), Replacement: []byte("CD")}, + {File: "value.txt", Start: 2, End: 2, Replacement: []byte("Y")}, + }, + } { + t.Run(name, func(t *testing.T) { + _, err := applySourceEdits("value.txt", original, edits) + require.Error(t, err) + if strings.Contains(name, "insert") { + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorOverlappingEdits, problem.Code) + } + }) + } + + id := syncdomain.VariationID("project", "config", "default") + prompt := "Help" + projection := syncdomain.VariationProjection{Instructions: &prompt} + assert.True(t, sameProjection(id, projection, projection)) + other := "Other" + assert.False(t, sameProjection(id, projection, syncdomain.VariationProjection{Instructions: &other})) + + assert.True(t, generatedSource("vendor/value.go", nil)) + assert.True(t, generatedSource("Vendor/value.go", nil), "a case-insensitive volume serves the vendored tree under another case") + assert.True(t, generatedSource("value.min.js", nil)) + assert.True(t, generatedSource("value.Min.JS", nil)) + assert.True(t, generatedSource("value.go", []byte("// Code generated by tool. DO NOT EDIT."))) + assert.False(t, generatedSource("value.go", []byte("package value"))) + + root := t.TempDir() + for _, name := range []string{"main.py", "dependency.py", "other.py"} { + writeTestFile(t, root, name, "") + } + snapshot := newSourceSnapshot(root) + key := func(name string) string { + resolved, err := snapshot.key(name) + require.NoError(t, err) + return resolved + } + bundle := syncadapter.SourceBundle{ + Primary: syncadapter.Document{Path: "main.py"}, + Dependencies: map[string]syncadapter.Document{"dependency.py": {Path: "dependency.py"}}, + } + assert.True(t, snapshot.bundleTouches(bundle, map[string]syncadapter.Document{key("dependency.py"): {}})) + assert.True(t, snapshot.bundleTouches(bundle, map[string]syncadapter.Document{key("main.py"): {}})) + assert.False(t, snapshot.bundleTouches(bundle, map[string]syncadapter.Document{key("other.py"): {}})) +} + +func TestCheckSourcesDoesNotWrite(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Old prompt\n") + id := syncdomain.VariationID("project", "config", "default") + reference := Reference{File: "prompt.md", Format: "markdown"} + writeDescriptorForTest(t, root, id, reference) + prompt := "New prompt" + + err := NewStore(root).CheckSources([]SourceReplacement{{ + ID: id, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + require.NoError(t, err) + content, err := os.ReadFile(filepath.Join(root, "prompt.md")) + require.NoError(t, err) + assert.Equal(t, "Old prompt\n", string(content)) +} + +func TestIndexedSourceErrorsDoNotWrite(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + prompt := "New prompt" + for name, selectPath := range map[string]string{ + "malformed": "items[-1]", + "out of range": "items[2]", + } { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + original := []byte("{\n \"items\": [{\"prompt\": \"Old prompt\"}]\n}\n") + writeTestFile(t, root, "values.json", string(original)) + reference := Reference{ + File: "values.json", Format: "json", Select: selectPath, + Map: map[string]string{"prompt": "prompt"}, + } + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + + require.Error(t, err) + actual, readErr := os.ReadFile(filepath.Join(root, "values.json")) + require.NoError(t, readErr) + assert.Equal(t, original, actual) + }) + } +} + +func TestIndexedSourceChangeAfterReviewDoesNotWrite(t *testing.T) { + root := t.TempDir() + original := []byte(`{"items":[{"name":"first","prompt":"Keep"},{"name":"second","prompt":"Old"}]}` + "\n") + writeTestFile(t, root, "values.json", string(original)) + id := syncdomain.VariationID("project", "config", "default") + reference := Reference{ + File: "values.json", Format: "json", Select: "items[1]", + Map: map[string]string{"prompt": "prompt"}, + } + writeDescriptorForTest(t, root, id, reference) + prompt := "New" + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + ExpectedIndexedSelections: []syncdomain.IndexedSelection{{ + Path: "select#0", CollectionLength: 2, + Fingerprint: "sha256:" + strings.Repeat("0", 64), + }}, + }}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorIndexedSelectionChanged, problem.Code) + actual, readErr := os.ReadFile(filepath.Join(root, "values.json")) + require.NoError(t, readErr) + assert.Equal(t, original, actual) +} + +func TestDescriptorStoreRejectsConflictsAndInvalidTargets(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + invalidID := syncdomain.ResourceID{Kind: "unknown", ProjectKey: "project", LookupKey: "value"} + require.Error(t, store.ValidateDescriptorTarget(invalidID)) + _, err := store.WriteDescriptor(invalidID, Reference{File: "prompt.md", Format: "markdown"}) + require.Error(t, err) + + id := syncdomain.VariationID("project", "config", "default") + writeTestFile(t, root, ".launchdarkly/project/configs/config/default.prompt.md", "managed") + require.Error(t, store.ValidateDescriptorTarget(id)) + + _, err = RenderDescriptor(id, Reference{File: "prompt.md", Format: "missing"}) + require.Error(t, err) + + require.NoError(t, os.Remove(filepath.Join(root, ".launchdarkly/project/configs/config/default.prompt.md"))) + writeTestFile(t, root, "prompt.md", "Help") + _, err = store.WriteDescriptor(id, Reference{File: "prompt.md", Format: "missing"}) + require.Error(t, err) + + _, err = NewSourceReference(filepath.Join(root, "missing-root"), root, filepath.Join(root, "prompt.md"), "markdown", "", nil) + require.Error(t, err) + _, err = resolveSourcePath(filepath.Join(root, "missing-root"), "prompt.md") + require.Error(t, err) +} + +func TestSourceEditPreparationFailureBranches(t *testing.T) { + missingStore := NewStore(filepath.Join(t.TempDir(), "missing")) + _, err := missingStore.ReplaceSources(nil) + require.Error(t, err) + + root := t.TempDir() + store := NewStore(root) + paths, err := store.ReplaceSources(nil) + require.NoError(t, err) + assert.Empty(t, paths) + + writeTestFile(t, root, "bad.py", "def broken(") + id := syncdomain.VariationID("project", "config", "default") + reference := Reference{ + File: "bad.py", Format: "python", Select: "$", + Map: map[string]string{"prompt": "$"}, + } + prompt := "Help" + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + require.Error(t, err) + + writeTestFile(t, root, "agents.py", `from agents import Agent +agent = Agent(tools=["search"]) +`) + toolReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="agent"]`, + Map: map[string]string{"prompt": "instructions", "tools": "tools"}, + } + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Reference: toolReference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + require.Error(t, err) + + longGenerated := append(make([]byte, 600), []byte("code generated do not edit")...) + assert.False(t, generatedSource("source.go", longGenerated)) + + // An unreadable descriptor of another resource has no projection to + // keep, so it does not block a check of other sources. + badRoot := t.TempDir() + writeTestFile(t, badRoot, ".launchdarkly/project/configs/config/default.ref.yaml", "invalid: [") + err = NewStore(badRoot).CheckSources(nil) + require.NoError(t, err) + + writeTestFile(t, root, "agent.py", `from agents import Agent +agent = Agent(instructions="Help", tools=["search"]) +`) + agentReference := Reference{ + File: "agent.py", Format: "python", Select: `agents[id="agent"]`, + Map: map[string]string{"prompt": "instructions", "tools": "tools"}, + } + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Reference: agentReference, + Projection: syncdomain.ToolProjection{}, + }}) + require.Error(t, err) + + changedTools := []string{"search", "read"} + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Reference: agentReference, + Projection: syncdomain.VariationProjection{Instructions: &prompt, Tools: &changedTools}, + }}) + require.Error(t, err) +} + +func TestStageSourceDocumentRejectsUnsafeFiles(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + writeTestFile(t, root, "generated.go", "// Code generated by tool. DO NOT EDIT.\n") + _, err := store.stageSourceDocument( + []string{"generated.go"}, + []byte("// Code generated by tool. DO NOT EDIT.\n"), + []byte("updated"), + ) + require.Error(t, err) + + _, err = store.stageSourceDocument([]string{"missing.go"}, nil, []byte("updated")) + require.Error(t, err) + + writeTestFile(t, root, "original.go", "package original\n") + hardlink := filepath.Join(root, "hardlink.go") + if err := os.Link(filepath.Join(root, "original.go"), hardlink); err == nil { + _, err = store.stageSourceDocument([]string{"original.go"}, []byte("package original\n"), []byte("package changed\n")) + require.Error(t, err) + } + + outside := filepath.Join(t.TempDir(), "outside.go") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + if err := os.Symlink(outside, filepath.Join(root, "link.go")); err == nil { + _, err = store.stageSourceDocument([]string{"link.go"}, []byte("outside"), []byte("changed")) + require.Error(t, err) + } +} + +func TestSourceBundleDependencyLimit(t *testing.T) { + var source strings.Builder + for index := 0; index < 33; index++ { + source.WriteString("import value") + source.WriteString(string(rune('A' + index%26))) + source.WriteString(" from './dependency") + source.WriteString(strings.Repeat("x", index+1)) + source.WriteString("';\n") + } + _, err := sourceBundle( + Reference{File: "main.ts", Format: "typescript"}, + []byte(source.String()), + func(dependency string) (syncadapter.Document, error) { + if strings.HasSuffix(dependency, ".ts") && !strings.HasSuffix(dependency, "/index.ts") { + return syncadapter.Document{Data: []byte("export const value = 1;\n")}, nil + } + return syncadapter.Document{}, fs.ErrNotExist + }, + ) + require.ErrorContains(t, err, "more than 32 dependencies") +} + +// A format reports several candidate paths for one import, and only the file +// that exists counts as a dependency. +func TestSourceBundleCountsOnlyTheDependenciesThatExist(t *testing.T) { + source := "import a from './a';\nimport b from './b';\nimport c from './c';\n" + bundle, err := sourceBundle( + Reference{File: "main.ts", Format: "typescript"}, + []byte(source), + func(dependency string) (syncadapter.Document, error) { + if dependency == "a.ts" { + return syncadapter.Document{Data: []byte("export default \"A\";\n")}, nil + } + return syncadapter.Document{}, fs.ErrNotExist + }, + ) + require.NoError(t, err, "three imports resolve to many candidates and one file") + require.Len(t, bundle.Dependencies, 1) + require.Contains(t, bundle.Dependencies, "a.ts") +} + +func TestCompileHelpersReturnReadAndParseErrors(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + fsys := fstest.MapFS{"value.ref.yaml": {Data: []byte("invalid: [")}} + _, err := compileDescriptor(fsys, "value.ref.yaml", id, func(string) (syncadapter.Document, error) { + return syncadapter.Document{}, nil + }, nil) + require.Error(t, err) + + fsys = fstest.MapFS{"value.ref.yaml": {Data: []byte(`formatVersion: 1 +file: prompt.md +format: markdown +`)}} + expectedRead := errors.New("primary read failed") + _, err = compileDescriptor(fsys, "value.ref.yaml", id, func(string) (syncadapter.Document, error) { + return syncadapter.Document{}, expectedRead + }, nil) + require.ErrorIs(t, err, expectedRead) + + fsys = fstest.MapFS{"value.ref.yaml": {Data: []byte(`formatVersion: 1 +file: app/main.py +format: python +select: 'agents[id="agent"]' +map: + prompt: instructions +`)}} + calls := 0 + dependencyError := errors.New("dependency read failed") + _, err = compileDescriptor(fsys, "value.ref.yaml", id, func(string) (syncadapter.Document, error) { + calls++ + if calls == 1 { + return syncadapter.Document{Data: []byte("from .prompt import TEXT\n")}, nil + } + return syncadapter.Document{}, dependencyError + }, nil) + require.ErrorIs(t, err, dependencyError) + + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + _, err = compileDescriptor( + fstest.MapFS{"bad.ref.yaml": {Data: []byte("invalid: [")}}, + "bad.ref.yaml", + toolID, + func(string) (syncadapter.Document, error) { return syncadapter.Document{}, nil }, + nil, + ) + require.Error(t, err) + + fsys = fstest.MapFS{"value.ref.yaml": {Data: []byte(`formatVersion: 1 +file: tool.json +format: json +select: $ +map: + description: description +`)}} + expected := errors.New("read failed") + _, err = compileDescriptor(fsys, "value.ref.yaml", toolID, func(string) (syncadapter.Document, error) { + return syncadapter.Document{}, expected + }, nil) + require.ErrorIs(t, err, expected) + + _, err = compileSource( + id, + Reference{File: "value.json", Format: "json", Select: "$", Map: map[string]string{"prompt": "prompt"}}, + syncadapter.SourceBundle{Primary: syncadapter.Document{Path: "value.json", Data: []byte(`{"prompt":`)}}, + ) + require.Error(t, err) + _, err = compileSource( + toolID, + Reference{File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}}, + syncadapter.SourceBundle{Primary: syncadapter.Document{Path: "value.json", Data: []byte(`{"description":`)}}, + ) + require.Error(t, err) +} + +func TestRejectGeneratedTargetFailsClosedWhenTheRootDoesNotResolve(t *testing.T) { + missing := filepath.Join(t.TempDir(), "missing") + + err := rejectGeneratedTarget(missing, "prompt.md", filepath.Join(missing, "prompt.md"), nil) + + require.ErrorContains(t, err, "resolve repository root") +} diff --git a/internal/sync/local/adapters.go b/internal/sync/local/adapters.go new file mode 100644 index 000000000..3780c70b2 --- /dev/null +++ b/internal/sync/local/adapters.go @@ -0,0 +1,5 @@ +package local + +// The store compiles and publishes repository sources through the registered +// source formats. This import registers every shipped format. +import _ "github.com/launchdarkly/ldcli/internal/sync/adapter/all" diff --git a/internal/sync/local/attachment.go b/internal/sync/local/attachment.go index 7521ecd85..688fbaabd 100644 --- a/internal/sync/local/attachment.go +++ b/internal/sync/local/attachment.go @@ -8,12 +8,14 @@ import ( "io" "io/fs" "maps" + "os" "path" "path/filepath" "slices" "strings" syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncadapter "github.com/launchdarkly/ldcli/internal/sync/adapter" ) // A tool file is JSON. A skill file is Markdown with YAML front matter that @@ -38,7 +40,7 @@ type skillFrontMatter struct { // CompileAttachments reads each tool and skill file that include accepts. It // does not read the other files, so an invalid file outside include does not // stop the read. Like CompileWorkspace, it rejects a symbolic link in a -// managed path. +// managed path and follows a link inside the repository to a source file. func CompileAttachments(repositoryRoot string, include func(syncdomain.ResourceID) bool) ([]syncdomain.SyncedAttachment, error) { root, err := filepath.EvalSymlinks(repositoryRoot) if err != nil { @@ -51,16 +53,49 @@ func CompileAttachments(repositoryRoot string, include func(syncdomain.ResourceI fsys := workspaceFS{root: root} var attachments []syncdomain.SyncedAttachment + definitions := make(map[syncdomain.ResourceID]string) for _, file := range files { id, ok := ParseManagedPath(file) + if !ok { + id, ok = ParseReferencePath(file) + } if !ok || id.Kind == syncdomain.KindVariation || !include(id) { continue } - attachment, err := readAttachment(fsys, id.ProjectKey, syncdomain.AttachmentKind(id.Kind), id.LookupKey) + if info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(file))); err == nil && info.Mode()&fs.ModeSymlink != 0 { + // A leftover link beside a descriptor is not a definition, so + // the descriptor defines the resource. A link alone fails below. + if skip, err := leftoverManagedLink(fsys, file, id); err != nil { + return nil, err + } else if skip { + continue + } + } + if existing, duplicate := definitions[id]; duplicate { + return nil, ParseError{Path: file, Err: definitionConflict( + id, fmt.Sprintf("%s is also defined by %s", id.Label(), existing), + )} + } + definitions[id] = file + + var attachment syncdomain.SyncedAttachment + if strings.HasSuffix(file, referenceFileSuffix) { + var definition SourceDefinition + definition, err = compileDescriptor(fsys, file, id, func(file string) (syncadapter.Document, error) { + return readSourceDocument(root, file) + }, nil) + if err == nil { + attachment = *definition.Attachment + } + } else { + var value syncdomain.Attachment + value, err = readAttachment(fsys, id.ProjectKey, syncdomain.AttachmentKind(id.Kind), id.LookupKey) + attachment = syncdomain.SyncedAttachment{ProjectKey: id.ProjectKey, Attachment: value} + } if err != nil { return nil, ParseError{Path: file, Err: err} } - attachments = append(attachments, syncdomain.SyncedAttachment{ProjectKey: id.ProjectKey, Attachment: attachment}) + attachments = append(attachments, attachment) } return attachments, nil } diff --git a/internal/sync/local/attachment_test.go b/internal/sync/local/attachment_test.go index 2d23151a0..68407b504 100644 --- a/internal/sync/local/attachment_test.go +++ b/internal/sync/local/attachment_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" "testing/fstest" @@ -376,6 +377,93 @@ Help the customer. require.ErrorContains(t, err, "symbolic links are not supported") } +func TestCompileWorkspaceIgnoresASymlinkAtAManagedPathBesideADescriptor(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "config", "default") + require.NoError(t, os.WriteFile(filepath.Join(root, "prompt.md"), []byte("Help the customer.\n"), 0o644)) + _, err := NewStore(root).WriteDescriptor(id, Reference{File: "prompt.md", Format: "markdown"}) + require.NoError(t, err) + external := filepath.Join(root, "external.md") + require.NoError(t, os.WriteFile(external, []byte("---\nformatVersion: 1\nmode: agent\nkey: default\n---\n\nOther.\n"), 0o644)) + managedPath, err := ResourcePath(id) + require.NoError(t, err) + require.NoError(t, os.Symlink(external, filepath.Join(root, filepath.FromSlash(managedPath)))) + + variations, err := CompileWorkspace(root) + + require.NoError(t, err, "a link is not a definition, so the descriptor defines the variation") + require.Len(t, variations, 1) + assert.Contains(t, variations[0].Variation.Instructions, "Help the customer.") + assert.NotNil(t, variations[0].Source) +} + +func TestCompileWorkspaceRejectsASymlinkOnlyVariation(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "config", "default") + external := filepath.Join(root, "external.md") + require.NoError(t, os.WriteFile(external, []byte("---\nformatVersion: 1\nmode: agent\nkey: default\n---\n\nOther.\n"), 0o644)) + managedPath, err := ResourcePath(id) + require.NoError(t, err) + target := filepath.Join(root, filepath.FromSlash(managedPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.Symlink(external, target)) + + _, err = CompileWorkspace(root) + + require.ErrorContains(t, err, "symbolic links are not supported", "a link without a descriptor is an error, not an absent variation") +} + +func TestCompileAttachmentsIgnoresASymlinkAtAManagedPathBesideADescriptor(t *testing.T) { + root := t.TempDir() + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search"} + require.NoError(t, os.WriteFile(filepath.Join(root, "tool.json"), []byte(`{"description":"Search","schema":{"type":"object"}}`), 0o644)) + _, err := NewStore(root).WriteDescriptor(id, Reference{ + File: "tool.json", Format: "json", Select: "$", Map: map[string]string{"description": "description", "schema": "schema"}, + }) + require.NoError(t, err) + external := filepath.Join(root, "external.json") + require.NoError(t, os.WriteFile(external, []byte(`{"formatVersion":1,"key":"search","schema":{}}`), 0o644)) + managedPath, err := ResourcePath(id) + require.NoError(t, err) + require.NoError(t, os.Symlink(external, filepath.Join(root, filepath.FromSlash(managedPath)))) + + attachments, err := CompileAttachments(root, func(syncdomain.ResourceID) bool { return true }) + + require.NoError(t, err, "a link is not a definition, so the descriptor defines the tool") + require.Len(t, attachments, 1) + assert.Equal(t, "Search", *attachments[0].Attachment.Tool.Description) + assert.NotNil(t, attachments[0].Source) +} + +func TestExistsAndDeleteTreatALeftoverLinkBesideADescriptorAsAbsent(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "config", "default") + require.NoError(t, os.WriteFile(filepath.Join(root, "prompt.md"), []byte("Help the customer.\n"), 0o644)) + store := NewStore(root) + descriptorPath, err := store.WriteDescriptor(id, Reference{File: "prompt.md", Format: "markdown"}) + require.NoError(t, err) + external := filepath.Join(root, "external.md") + require.NoError(t, os.WriteFile(external, []byte("Other.\n"), 0o644)) + managedPath, err := ResourcePath(id) + require.NoError(t, err) + require.NoError(t, os.Symlink(external, filepath.Join(root, filepath.FromSlash(managedPath)))) + + exists, err := store.ResourceExists(id) + require.NoError(t, err, "a link is not a second definition") + assert.True(t, exists) + + deleted, err := store.DeleteDefinitions([]syncdomain.ResourceID{id}) + require.NoError(t, err) + assert.Equal(t, []string{ + strings.TrimPrefix(descriptorPath, syncdomain.RootDir+"/"), + strings.TrimPrefix(managedPath, syncdomain.RootDir+"/"), + }, deleted, "detach deletes the descriptor and the leftover link beside it") + assert.NoFileExists(t, filepath.Join(root, filepath.FromSlash(descriptorPath))) + _, err = os.Lstat(filepath.Join(root, filepath.FromSlash(managedPath))) + require.ErrorIs(t, err, os.ErrNotExist, "the link does not stand alone, where compile would reject it") + assert.FileExists(t, external, "the target of the link is kept") +} + func TestCompileWorkspaceRejectsAttachmentParentSymlink(t *testing.T) { root := t.TempDir() wrapperPath := filepath.Join(root, ".launchdarkly", "project", "configs", "config", "default.prompt.md") @@ -411,3 +499,45 @@ Help the customer. _, err = NewStore(root).ReplaceVariations([]VariationReplacement{{ProjectKey: "project", ConfigKey: "config", Variation: variation}}) require.ErrorContains(t, err, "symbolic links are not supported") } + +func TestCompileAttachmentsFollowsARepositoryLinkToALinkedSource(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "lib", "tools"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(root, "lib", "tools", "search.json"), + []byte(`{"description":"Linked","schema":{"type":"object"}}`), 0o644, + )) + require.NoError(t, os.Symlink(filepath.Join(root, "lib"), filepath.Join(root, "linked"))) + writeManagedFile(t, root, "project/tools/search.ref.yaml", + "formatVersion: 1\nfile: linked/tools/search.json\nformat: json\nselect: $\n"+ + "map:\n description: description\n schema: schema\n") + + attachments, err := CompileAttachments(root, func(syncdomain.ResourceID) bool { return true }) + + require.NoError(t, err) + require.Len(t, attachments, 1) + assert.Equal(t, "Linked", *attachments[0].Attachment.Tool.Description) + variations, err := CompileWorkspace(root) + require.NoError(t, err) + assert.Empty(t, variations, "the same read rule applies to both compile paths") +} + +func TestDeleteDefinitionsRemovesALoneLeftoverLink(t *testing.T) { + root := t.TempDir() + external := filepath.Join(root, "external.md") + require.NoError(t, os.WriteFile(external, []byte("Other.\n"), 0o644)) + id := syncdomain.VariationID("project", "config", "linked") + managedPath, err := ResourcePath(id) + require.NoError(t, err) + target := filepath.Join(root, filepath.FromSlash(managedPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.Symlink(external, target)) + + deleted, err := NewStore(root).DeleteDefinitions([]syncdomain.ResourceID{id}) + + require.NoError(t, err) + assert.Len(t, deleted, 1) + _, err = os.Lstat(target) + assert.ErrorIs(t, err, os.ErrNotExist, "the link itself is removed") + assert.FileExists(t, external, "the target of the link stays") +} diff --git a/internal/sync/local/compile.go b/internal/sync/local/compile.go index 6c92875d1..05d02407b 100644 --- a/internal/sync/local/compile.go +++ b/internal/sync/local/compile.go @@ -11,6 +11,7 @@ import ( "strings" syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncadapter "github.com/launchdarkly/ldcli/internal/sync/adapter" ) // ErrNoDirectory reports that the repository has no managed directory. @@ -28,17 +29,29 @@ func (e ParseError) Unwrap() error { return e.Err } // Compile reads every variation in a file system that contains a managed // directory. Tests use it with an in-memory file system. func Compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { - return compile(fsys) + return compile(fsys, func(file string) (syncadapter.Document, error) { + return readSourceFileFromFS(fsys, file) + }, nil) } // CompileWorkspace reads every variation in a repository. It rejects a // symbolic link in a managed path, and a linked file outside the repository. func CompileWorkspace(repositoryRoot string) ([]syncdomain.SyncedResource, error) { + return CompileWorkspaceExcept(repositoryRoot, nil) +} + +// CompileWorkspaceExcept reads every variation in a repository except the +// ones that exclude accepts, which it does not read at all. A link replaces +// the descriptor of its resource, so the current descriptor, which can fail +// to compile, is not in its way. +func CompileWorkspaceExcept(repositoryRoot string, exclude func(syncdomain.ResourceID) bool) ([]syncdomain.SyncedResource, error) { root, err := filepath.EvalSymlinks(repositoryRoot) if err != nil { return nil, fmt.Errorf("resolve repository root: %w", err) } - return compile(workspaceFS{root: root}) + return compile(workspaceFS{root: root}, func(file string) (syncadapter.Document, error) { + return readSourceDocument(root, file) + }, exclude) } // Compile reads every variation in the repository of the store. @@ -46,6 +59,9 @@ func (store Store) Compile() ([]syncdomain.SyncedResource, error) { return CompileWorkspace(store.repositoryRoot) } +// errSymlinkNotSupported rejects a symbolic link in the managed directory. +var errSymlinkNotSupported = errors.New("symbolic links are not supported") + // workspaceFS opens repository files and rejects a path that goes through a // symbolic link. Sync owns the managed files, so a link is not valid there. type workspaceFS struct { @@ -62,13 +78,17 @@ func (fsys workspaceFS) Open(name string) (fs.File, error) { return nil, err } if filepath.Clean(resolved) != filepath.Clean(target) { - return nil, &fs.PathError{Op: "open", Path: name, Err: errors.New("symbolic links are not supported")} + return nil, &fs.PathError{Op: "open", Path: name, Err: errSymlinkNotSupported} } return os.Open(target) } // compile reads the variation files of each project, in identity order. -func compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { +func compile( + fsys fs.FS, + readSource func(file string) (syncadapter.Document, error), + exclude func(syncdomain.ResourceID) bool, +) ([]syncdomain.SyncedResource, error) { entries, err := fs.ReadDir(fsys, syncdomain.RootDir) if errors.Is(err, fs.ErrNotExist) { return nil, ErrNoDirectory @@ -84,7 +104,7 @@ func compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { if !entry.IsDir() { continue } - projectVariations, err := compileProject(fsys, entry.Name()) + projectVariations, err := compileProject(fsys, entry.Name(), readSource, exclude) if err != nil { return nil, err } @@ -101,13 +121,46 @@ func compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { func compileProject( fsys fs.FS, projectKey string, + readSource func(file string) (syncadapter.Document, error), + exclude func(syncdomain.ResourceID) bool, ) ([]syncdomain.SyncedResource, error) { configsRoot := path.Join(syncdomain.RootDir, projectKey, configsDir) readAttachment := func(kind syncdomain.AttachmentKind, key string) (syncdomain.Attachment, error) { + id := syncdomain.ResourceID{Kind: syncdomain.Kind(kind), ProjectKey: projectKey, LookupKey: key} + descriptorPath, err := ReferencePath(id) + if err != nil { + return syncdomain.Attachment{}, err + } + if _, err := fs.Stat(fsys, descriptorPath); err == nil { + managedPath, _ := ResourcePath(id) + if _, managedErr := fs.Stat(fsys, managedPath); managedErr == nil { + return syncdomain.Attachment{}, definitionConflict( + id, id.Label()+" has a managed file and a descriptor", + ) + } + definition, err := compileDescriptor(fsys, descriptorPath, id, readSource, nil) + if err != nil { + return syncdomain.Attachment{}, err + } + return definition.Attachment.Attachment, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return syncdomain.Attachment{}, err + } + managedPath, err := ResourcePath(id) + if err != nil { + return syncdomain.Attachment{}, err + } + if _, err := fs.Stat(fsys, managedPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return syncdomain.Attachment{}, missingLocalAttachmentDefinition(id) + } + return syncdomain.Attachment{}, err + } return readAttachment(fsys, projectKey, kind, key) } var variations []syncdomain.SyncedResource + definitions := make(map[syncdomain.ResourceID]string) err := fs.WalkDir(fsys, configsRoot, func(name string, entry fs.DirEntry, err error) error { switch { case name == configsRoot && errors.Is(err, fs.ErrNotExist): @@ -118,20 +171,54 @@ func compileProject( case entry.IsDir(): return nil } - if id, ok := ParseManagedPath(name); !ok || id.Kind != syncdomain.KindVariation { + id, managed := ParseManagedPath(name) + if !managed { + id, managed = ParseReferencePath(name) + } + if !managed || id.Kind != syncdomain.KindVariation || exclude != nil && exclude(id) { return nil } - - data, err := fs.ReadFile(fsys, name) + // Info reads the entry itself, so it reports a link on a file system + // whose directory listing has no file types. + info, err := entry.Info() if err != nil { return err } - variation, err := parseVariation(localFile{ - ProjectKey: projectKey, - RelPath: strings.TrimPrefix(name, configsRoot+"/"), - Data: data, - ReadAttachment: readAttachment, - }) + if info.Mode()&fs.ModeSymlink != 0 { + // Sync owns the managed files, so a link is not a definition, as + // it is not for a write. A leftover link beside a descriptor is + // skipped, and the descriptor defines the variation. A link + // alone is an error, not an absent variation. + if skip, err := leftoverManagedLink(fsys, name, id); skip || err != nil { + return err + } + return ParseError{Path: name, Err: errSymlinkNotSupported} + } + if existing, duplicate := definitions[id]; duplicate { + return ParseError{Path: name, Err: definitionConflict( + id, fmt.Sprintf("%s is also defined by %s", id.Label(), existing), + )} + } + definitions[id] = name + + var variation syncdomain.SyncedResource + if strings.HasSuffix(name, referenceFileSuffix) { + var definition SourceDefinition + if definition, err = compileDescriptor(fsys, name, id, readSource, readAttachment); err == nil { + variation = *definition.Variation + } + } else { + var data []byte + data, err = fs.ReadFile(fsys, name) + if err == nil { + variation, err = parseVariation(localFile{ + ProjectKey: projectKey, + RelPath: strings.TrimPrefix(name, configsRoot+"/"), + Data: data, + ReadAttachment: readAttachment, + }) + } + } if err != nil { return ParseError{Path: name, Err: err} } @@ -143,3 +230,35 @@ func compileProject( } return variations, nil } + +// leftoverManagedLink reports whether a symbolic link at a managed path has +// a descriptor beside it, which defines the resource instead of the link. +func leftoverManagedLink(fsys fs.FS, name string, id syncdomain.ResourceID) (bool, error) { + if strings.HasSuffix(name, referenceFileSuffix) { + return false, nil + } + descriptorPath, err := ReferencePath(id) + if err != nil { + return false, err + } + _, err = fs.Stat(fsys, descriptorPath) + switch { + case err == nil: + return true, nil + case errors.Is(err, fs.ErrNotExist): + return false, nil + default: + return false, err + } +} + +func missingLocalAttachmentDefinition(id syncdomain.ResourceID) error { + return syncdomain.NewProblem( + syncdomain.ErrorAttachmentDefinitionMissing, + fmt.Sprintf("%s %q has no local definition", id.Kind, id.LookupKey), + fmt.Sprintf( + "Run ldcli sync %ss add --project %s --key %s, or ldcli sync %ss link --project %s --key %s --file --format .", + id.Kind, id.ProjectKey, id.LookupKey, id.Kind, id.ProjectKey, id.LookupKey, + ), + ).WithResource(id) +} diff --git a/internal/sync/local/delete_test.go b/internal/sync/local/delete_test.go new file mode 100644 index 000000000..8853e6f5e --- /dev/null +++ b/internal/sync/local/delete_test.go @@ -0,0 +1,25 @@ +package local + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestDeleteDefinitionsKeepsExternalSource(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + writeDescriptorForTest(t, root, id, Reference{File: "prompt.md", Format: "markdown"}) + + paths, err := NewStore(root).DeleteDefinitions([]syncdomain.ResourceID{id}) + + require.NoError(t, err) + assert.Equal(t, []string{"project/configs/support/default.ref.yaml"}, paths) + require.FileExists(t, filepath.Join(root, "prompt.md")) + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "configs", "support", "default.ref.yaml")) +} diff --git a/internal/sync/local/descriptor_relationship_test.go b/internal/sync/local/descriptor_relationship_test.go new file mode 100644 index 000000000..9d46d6335 --- /dev/null +++ b/internal/sync/local/descriptor_relationship_test.go @@ -0,0 +1,247 @@ +package local + +import ( + "strings" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestDescriptorRelationshipsPreserveAbsentAndEmpty(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + base := Reference{File: "prompt.md", Format: "markdown"} + + absent, err := RenderDescriptor(id, base) + require.NoError(t, err) + assert.NotContains(t, string(absent), "tools:") + + empty := []syncdomain.AttachmentRef{} + rendered, err := RenderDescriptor(id, Reference{ + File: "prompt.md", Format: "markdown", Tools: &empty, + }) + require.NoError(t, err) + assert.Contains(t, string(rendered), "tools: []") + + reference, err := ReadDescriptor(fstest.MapFS{ + "value.ref.yaml": {Data: rendered}, + }, "value.ref.yaml", id) + require.NoError(t, err) + require.NotNil(t, reference.Tools) + assert.Empty(t, *reference.Tools) + assert.Nil(t, reference.Skills) +} + +func TestDescriptorRelationshipsUseStrictKeyOnlyEntries(t *testing.T) { + variationID := syncdomain.VariationID("project", "config", "default") + toolID := syncdomain.ResourceID{ + Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search", + } + skillID := syncdomain.ResourceID{ + Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "review", + } + tests := []struct { + name string + id syncdomain.ResourceID + content string + want string + }{ + { + name: "version", + id: variationID, + content: "formatVersion: 1\nfile: prompt.md\nformat: markdown\n" + + "tools:\n - key: search\n version: 2\n", + want: "field version not found", + }, + { + name: "custom parameters", + id: variationID, + content: "formatVersion: 1\nfile: prompt.md\nformat: markdown\n" + + "tools:\n - key: search\n customParameters: {}\n", + want: "field customParameters not found", + }, + { + name: "tool descriptor relationships", + id: toolID, + content: "formatVersion: 1\nfile: tool.json\nformat: json\nselect: $\n" + + "map:\n description: description\ntools: []\n", + want: `top-level "tools" is not valid for tool descriptors`, + }, + { + name: "skill descriptor relationships", + id: skillID, + content: "formatVersion: 1\nfile: skill.md\nformat: markdown\n" + + "skills: []\n", + want: `top-level "skills" is not valid for skill descriptors`, + }, + { + name: "mapped tools", + id: variationID, + content: "formatVersion: 1\nfile: prompt.json\nformat: json\nselect: $\n" + + "map:\n prompt: prompt\n tools: tools\n", + want: `map key "tools" is not allowed`, + }, + { + name: "mapped skills", + id: variationID, + content: "formatVersion: 1\nfile: prompt.json\nformat: json\nselect: $\n" + + "map:\n prompt: prompt\n skills: skills\n", + want: `map key "skills" is not allowed`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := ReadDescriptor(fstest.MapFS{ + "value.ref.yaml": {Data: []byte(test.content)}, + }, "value.ref.yaml", test.id) + require.ErrorContains(t, err, test.want) + }) + } +} + +func TestCompileDescriptorHydratesOnlyLocalProjectDefinitions(t *testing.T) { + descriptor := "formatVersion: 1\nfile: prompt.md\nformat: markdown\n" + + "tools:\n - key: Search.Case\n" + tool := `{"formatVersion":1,"key":"Search.Case","schema":{"type":"object"}}` + fsys := fstest.MapFS{ + ".launchdarkly/project/configs/config/default.ref.yaml": {Data: []byte(descriptor)}, + ".launchdarkly/project/tools/Search.Case.json": {Data: []byte(tool)}, + "prompt.md": {Data: []byte("Help")}, + } + + resources, err := Compile(fsys) + require.NoError(t, err) + require.Len(t, resources, 1) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "Search.Case"}}, resources[0].Variation.Tools) + require.Len(t, resources[0].Variation.Attachments, 1) + assert.Equal(t, "Search.Case", resources[0].Variation.Attachments[0].Key()) + + delete(fsys, ".launchdarkly/project/tools/Search.Case.json") + fsys[".launchdarkly/project/tools/Search.Case.ref.yaml"] = &fstest.MapFile{Data: []byte( + "formatVersion: 1\nfile: tool.json\nformat: json\nselect: $\n" + + "map:\n description: description\n schema: schema\n", + )} + fsys["tool.json"] = &fstest.MapFile{Data: []byte( + `{"description":"Linked","schema":{"type":"object"}}`, + )} + resources, err = Compile(fsys) + require.NoError(t, err) + require.Len(t, resources[0].Variation.Attachments, 1) + require.NotNil(t, resources[0].Variation.Attachments[0].Projection) + + delete(fsys, ".launchdarkly/project/tools/Search.Case.ref.yaml") + delete(fsys, "tool.json") + fsys[".launchdarkly/other/tools/Search.Case.json"] = &fstest.MapFile{Data: []byte(tool)} + _, err = Compile(fsys) + require.ErrorContains(t, err, `tool "Search.Case" has no local definition`) + require.ErrorContains(t, err, "ldcli sync tools add --project project --key Search.Case") + require.ErrorContains(t, err, "ldcli sync tools link --project project --key Search.Case") + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorAttachmentDefinitionMissing, problem.Code) +} + +func TestDescriptorRelationshipEditPreservesOtherBytesAndSorts(t *testing.T) { + original := []byte("# file comment\nformatVersion: 1\nfile: prompt.md # keep\n" + + "format: markdown\n# relationship comment\ntools: # replace\n" + + " # keep list comment\n - key: old # keep item comment\n" + + "skills:\n - key: keep\n# tail comment\n") + tools := []syncdomain.AttachmentRef{{Key: "z"}, {Key: "A.Case"}} + skills := []syncdomain.AttachmentRef{} + + updated, err := updateDescriptorRelationships(original, tools, true, skills, true) + require.NoError(t, err) + text := string(updated) + assert.Contains(t, text, "# file comment\nformatVersion: 1\nfile: prompt.md # keep\nformat: markdown\n") + assert.Contains(t, text, "# relationship comment\n") + assert.Contains(t, text, "tools: # replace\n # keep list comment\n # keep item comment\n"+ + " - key: A.Case\n - key: z\n") + assert.Contains(t, text, "skills: []\n# tail comment\n") + assert.False(t, strings.Contains(text, "key: old")) +} + +func TestDescriptorRelationshipHelpersHandleOwnedAndUnownedFields(t *testing.T) { + base := Reference{File: "prompt.md", Format: "markdown"} + unchanged, err := referenceWithProjectedRelationships( + base, + syncdomain.VariationProjection{FieldStates: map[syncdomain.Field]syncdomain.FieldState{}}, + ) + require.NoError(t, err) + assert.Nil(t, unchanged.Tools) + assert.Nil(t, unchanged.Skills) + + tools := []string{"search"} + skills := []string{"review"} + owned, err := referenceWithProjectedRelationships( + base, + syncdomain.VariationProjection{ + Tools: &tools, Skills: &skills, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + syncdomain.FieldSkills: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + ) + require.NoError(t, err) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "search"}}, *owned.Tools) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "review"}}, *owned.Skills) + + _, _, err = projectionRelationship( + syncdomain.ToolProjection{FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }}, + syncdomain.FieldTools, + ) + require.ErrorContains(t, err, `relationship field "tools" has no value`) +} + +func TestDescriptorRelationshipEditHandlesNoChangeAndInvalidDocuments(t *testing.T) { + original := []byte("formatVersion: 1\nfile: prompt.md\nformat: markdown\n") + updated, err := updateDescriptorRelationships(original, nil, false, nil, false) + require.NoError(t, err) + assert.Equal(t, original, updated) + + _, err = updateDescriptorRelationships([]byte("- not-a-mapping\n"), nil, true, nil, false) + require.EqualError(t, err, "descriptor must be a YAML mapping") + + _, err = updateDescriptorRelationships([]byte("invalid: [\n"), nil, true, nil, false) + require.Error(t, err) +} + +func TestDescriptorRelationshipReplacementSkipsIdenticalContent(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + writeTestFile(t, root, descriptorPath, + "formatVersion: 1\nfile: prompt.md\nformat: markdown\ntools:\n - key: search\n") + tools := []syncdomain.AttachmentRef{{Key: "search"}} + projected := []string{"search"} + replacement := SourceReplacement{ + ID: id, + Reference: Reference{ + File: "prompt.md", Format: "markdown", Tools: &tools, + }, + Projection: syncdomain.VariationProjection{ + Tools: &projected, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + } + + _, changed, err := NewStore(root).prepareDescriptorRelationshipReplacement(replacement) + + require.NoError(t, err) + assert.False(t, changed) + + replacement.Projection = syncdomain.VariationProjection{ + FieldStates: map[syncdomain.Field]syncdomain.FieldState{}, + } + _, changed, err = NewStore(root).prepareDescriptorRelationshipReplacement(replacement) + require.NoError(t, err) + assert.False(t, changed) +} diff --git a/internal/sync/local/descriptor_relationships.go b/internal/sync/local/descriptor_relationships.go new file mode 100644 index 000000000..617099156 --- /dev/null +++ b/internal/sync/local/descriptor_relationships.go @@ -0,0 +1,492 @@ +package local + +import ( + "bytes" + "errors" + "fmt" + "maps" + "os" + "path" + "path/filepath" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "gopkg.in/yaml.v3" +) + +// This file keeps the top-level tools and skills lists of a descriptor in +// step with a pulled variation. A pull edits only those list nodes and keeps +// the comments of the descriptor. + +func (store Store) prepareMissingRelationshipAttachments( + replacements []SourceReplacement, +) ([]stagedFile, error) { + available := make(map[syncdomain.ResourceID]syncdomain.Attachment) + fingerprints := make(map[syncdomain.ResourceID]string) + for _, replacement := range replacements { + for _, attachment := range replacement.RelationshipAttachments { + id := attachment.ID(replacement.ID.ProjectKey) + fingerprint, err := syncdomain.FingerprintAttachment(replacement.ID.ProjectKey, attachment) + if err != nil { + return nil, err + } + // Two variations can pull one tool with two bodies, for example + // from two versions. One publish writes one file, so the bodies + // must agree. + if previous, seen := fingerprints[id]; seen && previous != fingerprint { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDefinitionConflict, + fmt.Sprintf("%s %q has two different definitions in this pull", id.Kind, id.LookupKey), + fmt.Sprintf("Run ldcli sync %ss first, so that the variations share one version.", id.Kind), + ).WithResource(id) + } + available[id], fingerprints[id] = attachment, fingerprint + } + } + needed := make(map[syncdomain.ResourceID]struct{}) + for _, replacement := range replacements { + for _, kind := range syncdomain.AttachmentKinds { + field := syncdomain.FieldTools + if kind == syncdomain.AttachmentSkill { + field = syncdomain.FieldSkills + } + refs, changed, err := projectionRelationship(replacement.Projection, field) + if err != nil { + return nil, err + } + if !changed { + continue + } + for _, ref := range refs { + id := syncdomain.ResourceID{ + Kind: syncdomain.Kind(kind), ProjectKey: replacement.ID.ProjectKey, LookupKey: ref.Key, + } + managed, err := ResourcePath(id) + if err != nil { + return nil, err + } + descriptor, err := ReferencePath(id) + if err != nil { + return nil, err + } + // A descriptor defines the resource. A leftover link at the + // managed path is not a definition, as it is not for compile, + // so it neither defines the resource nor stops the pull. A + // link at the descriptor path stops the pull. + descriptorTarget := filepath.Join(store.repositoryRoot, filepath.FromSlash(descriptor)) + if err := rejectSymlinkedPath(store.root, descriptorTarget); err != nil { + return nil, err + } + managedTarget := filepath.Join(store.repositoryRoot, filepath.FromSlash(managed)) + if fileExists(descriptorTarget) || fileExists(managedTarget) { + continue + } + needed[id] = struct{}{} + } + } + } + rendered := make([]RenderedFile, 0, len(needed)) + for _, id := range slices.SortedFunc(maps.Keys(needed), syncdomain.CompareResourceIDs) { + attachment, ok := available[id] + if !ok { + return nil, missingLocalAttachmentDefinition(id) + } + file, err := RenderAttachment(syncdomain.SyncedAttachment{ + ProjectKey: id.ProjectKey, Attachment: attachment, + }) + if err != nil { + return nil, err + } + rendered = append(rendered, file) + } + staged, err := store.prepareAttachmentFiles(rendered) + if err != nil { + return nil, err + } + // A rendered attachment path is relative to the managed directory, and + // every other path that a publish reports is relative to the repository. + for index := range staged { + staged[index].displayPath = path.Join(syncdomain.RootDir, staged[index].displayPath) + } + return staged, nil +} + +// fileExists reports whether a regular file is at path. A symbolic link is +// not a definition, because compile rejects it. +func fileExists(path string) bool { + info, err := os.Lstat(path) + return err == nil && info.Mode().IsRegular() +} + +func referenceWithProjectedRelationships(reference Reference, projection syncdomain.Projection) (Reference, error) { + tools, toolsChanged, err := projectionRelationship(projection, syncdomain.FieldTools) + if err != nil { + return Reference{}, err + } + if toolsChanged { + reference.Tools = &tools + } + skills, skillsChanged, err := projectionRelationship(projection, syncdomain.FieldSkills) + if err != nil { + return Reference{}, err + } + if skillsChanged { + reference.Skills = &skills + } + return reference, nil +} + +// referenceWithRelationships returns a reference with the changed lists. +func referenceWithRelationships( + reference Reference, + tools []syncdomain.AttachmentRef, + toolsChanged bool, + skills []syncdomain.AttachmentRef, + skillsChanged bool, +) Reference { + if toolsChanged { + reference.Tools = &tools + } + if skillsChanged { + reference.Skills = &skills + } + return reference +} + +func (store Store) prepareDescriptorRelationshipReplacement( + replacement SourceReplacement, +) (stagedFile, bool, error) { + tools, toolsChanged, err := projectionRelationship(replacement.Projection, syncdomain.FieldTools) + if err != nil { + return stagedFile{}, false, err + } + skills, skillsChanged, err := projectionRelationship(replacement.Projection, syncdomain.FieldSkills) + if err != nil { + return stagedFile{}, false, err + } + if !toolsChanged && !skillsChanged { + return stagedFile{}, false, nil + } + descriptorPath, err := ReferencePath(replacement.ID) + if err != nil { + return stagedFile{}, false, err + } + target := filepath.Join(store.repositoryRoot, filepath.FromSlash(descriptorPath)) + // Sync owns the managed directory, so the write refuses a path that goes + // through a symbolic link, as every other managed write does. + if err := rejectSymlinkedPath(store.root, target); err != nil { + return stagedFile{}, false, err + } + original, err := os.ReadFile(target) + if errors.Is(err, os.ErrNotExist) { + // A link checks its plan before it writes the descriptor, so there + // is no list to edit yet. A write checked the descriptor first. + return stagedFile{}, false, nil + } + if err != nil { + return stagedFile{}, false, fmt.Errorf("read descriptor %q: %w", descriptorPath, err) + } + // The bytes of this one read serve the checks, the edit, and the + // original that the commit verifies, so a descriptor that another + // process changes between two reads cannot pass the checks on one + // content and stage an edit of another. + current, err := decodeDescriptor(original, replacement.ID) + if err != nil { + return stagedFile{}, false, err + } + mappingChanged, err := descriptorMappingChanged(replacement, current) + if err != nil { + return stagedFile{}, false, err + } + if mappingChanged { + return stagedFile{}, false, syncdomain.NewProblem( + syncdomain.ErrorStateChanged, "the source descriptor changed after review", "Run sync again.", + ).WithResource(replacement.ID).WithFile(descriptorPath, 0, 0) + } + if toolsChanged && !sameRelationshipPointer(current.Tools, replacement.Reference.Tools) || + skillsChanged && !sameRelationshipPointer(current.Skills, replacement.Reference.Skills) { + return stagedFile{}, false, syncdomain.NewProblem( + syncdomain.ErrorStateChanged, + "descriptor relationships changed after review", + "Run sync again.", + ).WithResource(replacement.ID).WithFile(descriptorPath, 0, 0) + } + next := referenceWithRelationships(current, tools, toolsChanged, skills, skillsChanged) + updated, err := updateDescriptorRelationships(original, tools, toolsChanged, skills, skillsChanged) + if err == nil && !descriptorReadsBack(updated, replacement.ID, next) { + // The in-place edit does not read back as the pull, for example + // because an anchor on the list was lost with it. + err = errRelationshipsShareLine + } + if errors.Is(err, errRelationshipsShareLine) { + // The lists cannot be edited in place, so the descriptor is + // rendered again from its content. The comments do not survive. + updated, err = RenderDescriptor(replacement.ID, next) + } + if err != nil { + return stagedFile{}, false, fmt.Errorf("update descriptor %q: %w", descriptorPath, err) + } + if bytes.Equal(original, updated) { + return stagedFile{}, false, nil + } + info, err := os.Stat(target) + if err != nil { + return stagedFile{}, false, fmt.Errorf("inspect descriptor %q: %w", descriptorPath, err) + } + return stagedFile{ + displayPath: descriptorPath, path: target, original: original, originalExists: true, + replacement: updated, mode: info.Mode().Perm(), + }, true, nil +} + +// descriptorReadsBack reports whether edited descriptor bytes decode to the +// file, format, selection, map, tools, and skills of the reference. +func descriptorReadsBack(data []byte, id syncdomain.ResourceID, reference Reference) bool { + parsed, err := decodeDescriptor(data, id) + if err != nil { + return false + } + sortReferenceRelationships(&reference) + return parsed.File == reference.File && parsed.Format == reference.Format && + parsed.Select == reference.Select && maps.Equal(parsed.Map, reference.Map) && + sameRelationshipPointer(parsed.Tools, reference.Tools) && + sameRelationshipPointer(parsed.Skills, reference.Skills) +} + +func projectionRelationship( + projection syncdomain.Projection, + field syncdomain.Field, +) ([]syncdomain.AttachmentRef, bool, error) { + if _, present := projection.States()[field]; !present { + return nil, false, nil + } + raw, ok := projection.Value(field) + if !ok { + return nil, false, fmt.Errorf("relationship field %q has no value", field) + } + keys, ok := raw.([]string) + if !ok { + return nil, false, fmt.Errorf("relationship field %q has type %T", field, raw) + } + refs := make([]syncdomain.AttachmentRef, len(keys)) + for index := range keys { + refs[index] = syncdomain.AttachmentRef{Key: keys[index]} + } + slices.SortFunc(refs, func(left, right syncdomain.AttachmentRef) int { + return strings.Compare(left.Key, right.Key) + }) + return refs, true, nil +} + +func sameRelationshipPointer(left, right *[]syncdomain.AttachmentRef) bool { + if left == nil || right == nil { + return left == right + } + return slices.Equal(*left, *right) +} + +// errRelationshipsShareLine reports that a tools or skills list of a +// descriptor shares its lines with other content, as in a flow mapping, so +// the list cannot be edited in place and the descriptor is rendered again. +var errRelationshipsShareLine = errors.New("descriptor relationship lists share a line") + +func updateDescriptorRelationships( + data []byte, + tools []syncdomain.AttachmentRef, + toolsChanged bool, + skills []syncdomain.AttachmentRef, + skillsChanged bool, +) ([]byte, error) { + var document yaml.Node + if err := yaml.Unmarshal(data, &document); err != nil { + return nil, err + } + if len(document.Content) == 0 || document.Content[0].Kind != yaml.MappingNode { + return nil, errors.New("descriptor must be a YAML mapping") + } + // An in-place edit replaces whole lines, and a flow mapping keeps every + // key on one line. + if document.Content[0].Style&yaml.FlowStyle != 0 { + return nil, errRelationshipsShareLine + } + lineStarts := []int{0} + for index, value := range data { + if value == '\n' { + lineStarts = append(lineStarts, index+1) + } + } + type edit struct { + start, end int + replacement []byte + } + var edits []edit + appendField := func(name string, refs []syncdomain.AttachmentRef, changed bool) error { + if !changed { + return nil + } + replacement := renderRelationshipNode(name, refs, nil) + root := document.Content[0] + for index := 0; index+1 < len(root.Content); index += 2 { + if root.Content[index].Value != name { + continue + } + key, value := root.Content[index], root.Content[index+1] + lastLine := relationshipNodeLastLine(value) + if key.Line < 1 || key.Line > len(lineStarts) || lastLine < key.Line { + return errors.New("descriptor relationship has invalid source positions") + } + replacement = renderRelationshipNode(name, refs, relationshipNodeComments(value)) + comment := strings.TrimSpace(value.LineComment) + if comment == "" { + comment = strings.TrimSpace(key.LineComment) + } + if comment != "" { + lineEnd := bytes.IndexByte(replacement, '\n') + replacement = slices.Concat( + replacement[:lineEnd], + []byte(" "+comment), + replacement[lineEnd:], + ) + } + // Another key on the replaced lines would be lost with them. + for other := 0; other+1 < len(root.Content); other += 2 { + if other != index && root.Content[other].Line >= key.Line && root.Content[other].Line <= lastLine { + return errRelationshipsShareLine + } + } + start := lineStarts[key.Line-1] + end := len(data) + if lastLine < len(lineStarts) { + end = lineStarts[lastLine] + } + // A comment on the lines after the list belongs to it, and the + // rendered list repeats it, so the replaced range takes it in. + end = relationshipCommentLinesEnd(data, end, relationshipNodeComments(value)) + edits = append(edits, edit{start: start, end: end, replacement: replacement}) + return nil + } + start := len(data) + for index := range edits { + if edits[index].start == start && edits[index].end == start { + edits[index].replacement = append(edits[index].replacement, replacement...) + return nil + } + } + prefix := []byte{} + if start != 0 && data[start-1] != '\n' { + prefix = []byte("\n") + } + edits = append(edits, edit{start: start, end: start, replacement: append(prefix, replacement...)}) + return nil + } + if err := appendField("tools", tools, toolsChanged); err != nil { + return nil, err + } + if err := appendField("skills", skills, skillsChanged); err != nil { + return nil, err + } + slices.SortFunc(edits, func(left, right edit) int { return right.start - left.start }) + // Two lists on one line, as in a flow mapping, share a range, and two + // edits of one range cannot both apply. + for index := 1; index < len(edits); index++ { + if edits[index].end > edits[index-1].start { + return nil, errRelationshipsShareLine + } + } + updated := bytes.Clone(data) + for _, item := range edits { + next := make([]byte, 0, len(updated)-item.end+item.start+len(item.replacement)) + next = append(next, updated[:item.start]...) + next = append(next, item.replacement...) + next = append(next, updated[item.end:]...) + updated = next + } + return updated, nil +} + +// relationshipCommentLinesEnd extends an end offset over the comment lines +// that follow it, as long as each one is a comment that the list repeats. +func relationshipCommentLinesEnd(data []byte, end int, comments []string) int { + repeated := make(map[string]struct{}, len(comments)) + for _, comment := range comments { + repeated[comment] = struct{}{} + } + for end < len(data) { + lineEnd := bytes.IndexByte(data[end:], '\n') + next := len(data) + if lineEnd >= 0 { + next = end + lineEnd + 1 + } + line := strings.TrimSpace(string(data[end:next])) + if _, ok := repeated[line]; !ok || !strings.HasPrefix(line, "#") { + return end + } + end = next + } + return end +} + +func relationshipNodeLastLine(node *yaml.Node) int { + line := node.Line + for _, child := range node.Content { + if childLine := relationshipNodeLastLine(child); childLine > line { + line = childLine + } + } + return line +} + +func relationshipNodeComments(node *yaml.Node) []string { + var comments []string + seen := make(map[string]struct{}) + var visit func(*yaml.Node) + visit = func(current *yaml.Node) { + for _, value := range []string{current.HeadComment, current.LineComment, current.FootComment} { + for line := range strings.SplitSeq(value, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if _, duplicate := seen[line]; !duplicate { + seen[line] = struct{}{} + comments = append(comments, line) + } + } + } + for _, child := range current.Content { + visit(child) + } + } + for _, child := range node.Content { + visit(child) + } + return comments +} + +func renderRelationshipNode(name string, refs []syncdomain.AttachmentRef, comments []string) []byte { + refs = slices.Clone(refs) + slices.SortFunc(refs, func(left, right syncdomain.AttachmentRef) int { + return strings.Compare(left.Key, right.Key) + }) + if len(refs) == 0 { + var content strings.Builder + content.WriteString(name + ": []\n") + for _, comment := range comments { + content.WriteString(" " + comment + "\n") + } + return []byte(content.String()) + } + var content strings.Builder + content.WriteString(name + ":\n") + for _, comment := range comments { + content.WriteString(" " + comment + "\n") + } + for _, ref := range refs { + content.WriteString(" - key: ") + value, _ := yaml.Marshal(ref.Key) + content.Write(bytes.TrimSpace(value)) + content.WriteByte('\n') + } + return []byte(content.String()) +} diff --git a/internal/sync/local/source_publish.go b/internal/sync/local/source_publish.go new file mode 100644 index 000000000..c16486518 --- /dev/null +++ b/internal/sync/local/source_publish.go @@ -0,0 +1,563 @@ +package local + +import ( + "bytes" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "syscall" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncadapter "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +// SourceReplacement is one descriptor-backed resource update. +type SourceReplacement struct { + ID syncdomain.ResourceID + Reference Reference + Projection syncdomain.Projection + // ExpectedFingerprint is the mapped source state from the reviewed plan. + ExpectedFingerprint string + // ExpectedProjection supplies reviewed values for unreadable fields. + ExpectedProjection syncdomain.Projection + // ExpectedIndexedSelections identifies the indexed items from the reviewed plan. + ExpectedIndexedSelections []syncdomain.IndexedSelection + // RelationshipAttachments supplies content for a relationship that a pull + // adds when the project has no local definition. + RelationshipAttachments []syncdomain.Attachment +} + +// ReplaceSources applies checked edits to each selected repository source. +// Every replacement in one call is one unit: one byte snapshot per file, one +// edit pass per file, and one atomic replacement of every changed file. +func (store Store) ReplaceSources(replacements []SourceReplacement) ([]string, error) { + // The descriptors that the mapping check read are verified again at the + // commit, so a descriptor that another process retargets after the + // check does not reach a write of the file that it named before. + guards := make(map[string]stagedFile) + for _, replacement := range replacements { + guard, err := store.checkDescriptorMapping(replacement) + if err != nil { + return nil, err + } + guards[guard.path] = guard + } + staged, err := store.prepareSourceReplacements(replacements) + if err != nil { + return nil, err + } + paths := make([]string, len(staged)) + for index := range staged { + paths[index] = staged[index].displayPath + delete(guards, staged[index].path) + } + for _, path := range slices.Sorted(maps.Keys(guards)) { + staged = append(staged, guards[path]) + } + if err := replaceFiles(staged); err != nil { + return nil, err + } + return paths, nil +} + +// CheckSources verifies grouped source edits without writing files. +func (store Store) CheckSources(replacements []SourceReplacement) error { + _, err := store.prepareSourceReplacements(replacements) + return err +} + +// prepareSourceReplacements plans, applies, and verifies every edit in +// memory, and then stages each changed file. +func (store Store) prepareSourceReplacements(replacements []SourceReplacement) ([]stagedFile, error) { + // Two descriptors can name one file two ways, for example through a + // link, so every map here is keyed by the resolved path of the file. + before := newSourceSnapshot(store.repositoryRoot) + edits := make(map[string][]syncadapter.TextEdit) + for _, replacement := range replacements { + bundle, err := before.bundle(replacement.Reference) + if err != nil { + return nil, err + } + planned, err := planSourceEdits(bundle, replacement) + if err != nil { + return nil, err + } + for _, edit := range planned { + key, err := before.key(edit.File) + if err != nil { + return nil, err + } + edits[key] = append(edits[key], edit) + } + } + + updated := make(map[string]syncadapter.Document, len(edits)) + for key, fileEdits := range edits { + document := before.documents[key] + data, err := applySourceEdits(document.Path, document.Data, fileEdits) + if err != nil { + return nil, err + } + updated[key] = syncadapter.Document{Path: document.Path, Data: data} + } + after := before.withUpdates(updated) + if err := verifyTargetProjections(after, replacements); err != nil { + return nil, err + } + if err := store.verifyOtherBindings(before, after, updated, replacements); err != nil { + return nil, err + } + + var staged []stagedFile + for _, key := range slices.Sorted(maps.Keys(updated)) { + original := before.documents[key] + next, err := store.stageSourceDocument(before.names[key], original.Data, updated[key].Data) + if err != nil { + return nil, err + } + if !bytes.Equal(next.original, next.replacement) { + staged = append(staged, next) + } + } + for _, replacement := range replacements { + next, changed, err := store.prepareDescriptorRelationshipReplacement(replacement) + if err != nil { + return nil, err + } + if changed { + staged = append(staged, next) + } + } + missingAttachments, err := store.prepareMissingRelationshipAttachments(replacements) + if err != nil { + return nil, err + } + return append(staged, missingAttachments...), nil +} + +// checkDescriptorMapping proves that the descriptor still names the reviewed +// file, format, selection, and map, because the edits follow the reviewed +// reference. The tools and skills lists have their own check. Only a write +// runs it: CheckSources runs before a link writes its descriptor. It returns +// the descriptor as a verify-only file, so that the commit makes sure that +// the checked content is still there. +func (store Store) checkDescriptorMapping(replacement SourceReplacement) (stagedFile, error) { + descriptorPath, err := ReferencePath(replacement.ID) + if err != nil { + return stagedFile{}, err + } + changed := syncdomain.NewProblem( + syncdomain.ErrorStateChanged, "the source descriptor changed after review", "Run sync again.", + ).WithResource(replacement.ID).WithFile(descriptorPath, 0, 0) + target := filepath.Join(store.repositoryRoot, filepath.FromSlash(descriptorPath)) + data, err := os.ReadFile(target) + if err != nil { + return stagedFile{}, syncdomain.WrapProblem(changed, err) + } + current, err := decodeDescriptor(data, replacement.ID) + if err != nil { + return stagedFile{}, syncdomain.WrapProblem(changed, err) + } + mappingChanged, err := descriptorMappingChanged(replacement, current) + if err != nil { + return stagedFile{}, err + } + if mappingChanged { + return stagedFile{}, changed + } + return stagedFile{ + displayPath: descriptorPath, path: target, original: data, originalExists: true, verifyOnly: true, + }, nil +} + +// descriptorMappingChanged reports whether a descriptor read from disk names +// another file, format, selection, or map than the reviewed reference. +func descriptorMappingChanged(replacement SourceReplacement, current Reference) (bool, error) { + // A reviewed reference can carry the defaults of its format implicitly, + // and the descriptor on disk always reads back normalized. + reviewed, err := syncadapter.NormalizeReference(replacement.ID, replacement.Reference) + if err != nil { + return false, err + } + reviewedDigest, err := syncdomain.ReferenceDigest(reviewed) + if err != nil { + return false, err + } + digest, err := syncdomain.ReferenceDigest(current) + if err != nil { + return false, err + } + return digest != reviewedDigest, nil +} + +// planSourceEdits proves that the source still has the reviewed state, and +// returns the direct edits that give it the desired projection. +func planSourceEdits(bundle syncadapter.SourceBundle, replacement SourceReplacement) ([]syncadapter.TextEdit, error) { + file := replacement.Reference.File + current, indexedSelections, err := syncadapter.ParseSourceWithSelections(bundle, replacement.ID, replacement.Reference) + if err != nil { + return nil, publicationProblem(file, "source cannot be updated safely", "Change the source to a supported static form.", err) + } + if replacement.ExpectedIndexedSelections != nil && + !slices.Equal(replacement.ExpectedIndexedSelections, indexedSelections) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorIndexedSelectionChanged, + "an indexed source selection changed after review", + "Run sync again. Use a keyed selector when the item has a stable key.", + ).WithResource(replacement.ID).WithFile(file, 0, 0) + } + if replacement.ExpectedProjection != nil { + if current, err = alignProjection(current, replacement.ExpectedProjection); err != nil { + return nil, err + } + if current, err = syncdomain.FillReadOnlyFields(current, replacement.ExpectedProjection); err != nil { + return nil, err + } + } + fingerprint, err := syncdomain.FingerprintProjection(replacement.ID, current) + if err != nil { + return nil, err + } + if replacement.ExpectedFingerprint != "" && fingerprint != replacement.ExpectedFingerprint { + return nil, syncdomain.NewProblem( + syncdomain.ErrorStateChanged, "mapped source fields changed after review", "Run sync again.", + ).WithResource(replacement.ID).WithFile(file, 0, 0) + } + result, err := syncadapter.UpdateSource(bundle, replacement.ID, replacement.Reference, replacement.Projection) + if err != nil { + return nil, err + } + var edits []syncadapter.TextEdit + for _, field := range slices.Sorted(maps.Keys(result.Fields)) { + update := result.Fields[field] + if update.Capability != syncadapter.CapabilityDirect { + return nil, syncdomain.NewProblem( + syncdomain.ErrorFieldReadOnly, + fmt.Sprintf("field %q is read-only: %s", field, update.Reason), + "Change the field to a directly writable form.", + ).WithResource(replacement.ID).WithFile(file, 0, 0) + } + edits = append(edits, update.Edits...) + } + return edits, nil +} + +// verifyTargetProjections reparses every updated source and proves that it +// has the desired values. +func verifyTargetProjections(after *sourceSnapshot, replacements []SourceReplacement) error { + for _, replacement := range replacements { + reference, err := referenceWithProjectedRelationships(replacement.Reference, replacement.Projection) + if err != nil { + return err + } + bundle, err := after.bundle(reference) + if err != nil { + return err + } + actual, err := syncadapter.ParseSource(bundle, replacement.ID, reference) + if err != nil { + return fmt.Errorf("verify updated %s: %w", replacement.ID.Label(), err) + } + if actual, err = alignProjection(actual, replacement.Projection); err != nil { + return err + } + // A descriptor keeps its tools and skills sorted, and LaunchDarkly + // lists them in its own order, so the lists compare as sets. + if !syncdomain.ProjectionIncludes(withSortedRelationships(actual), withSortedRelationships(replacement.Projection)) { + return fmt.Errorf("updated projection does not match %s", replacement.ID.Label()) + } + } + return nil +} + +// withSortedRelationships returns a variation projection whose tools and +// skills lists are sorted. Another projection is returned as it is. +func withSortedRelationships(projection syncdomain.Projection) syncdomain.Projection { + variation, ok := projection.(syncdomain.VariationProjection) + if !ok { + return projection + } + if variation.Tools != nil { + tools := slices.Sorted(slices.Values(*variation.Tools)) + variation.Tools = &tools + } + if variation.Skills != nil { + skills := slices.Sorted(slices.Values(*variation.Skills)) + variation.Skills = &skills + } + return variation +} + +// verifyOtherBindings proves that no other descriptor that reads an edited +// file sees a different projection after the edits. +func (store Store) verifyOtherBindings( + before, after *sourceSnapshot, + updated map[string]syncadapter.Document, + replacements []SourceReplacement, +) error { + targets := make(map[syncdomain.ResourceID]struct{}, len(replacements)) + for _, replacement := range replacements { + targets[replacement.ID] = struct{}{} + } + bindings, err := store.sourceBindings() + if err != nil { + return err + } + for _, binding := range bindings { + if _, target := targets[binding.id]; target { + continue + } + bundle, err := before.bundle(binding.reference) + if err != nil || !before.bundleTouches(bundle, updated) { + // A source that sync cannot read has no projection to keep. + continue + } + previous, err := syncadapter.ParseSource(bundle, binding.id, binding.reference) + if err != nil { + continue + } + bundle, err = after.bundle(binding.reference) + if err != nil { + return fmt.Errorf("non-target projection %s changed", binding.id.Label()) + } + next, err := syncadapter.ParseSource(bundle, binding.id, binding.reference) + if err != nil || !sameProjection(binding.id, previous, next) { + return fmt.Errorf("non-target projection %s changed", binding.id.Label()) + } + } + return nil +} + +// bundleTouches reports whether a bundle reads an updated file. updated is +// keyed by resolved path. +func (snapshot *sourceSnapshot) bundleTouches(bundle syncadapter.SourceBundle, updated map[string]syncadapter.Document) bool { + files := append([]string{bundle.Primary.Path}, slices.Collect(maps.Keys(bundle.Dependencies))...) + for _, file := range files { + if key, err := snapshot.key(file); err == nil { + if _, ok := updated[key]; ok { + return true + } + } + } + return false +} + +// sourceSnapshot is one read of each repository file that a publication +// touches. Every replacement reads the same bytes of a file, so two +// replacements cannot stage two versions of one file. The documents are +// keyed by resolved path, so two names of one file share one read. +type sourceSnapshot struct { + root string + documents map[string]syncadapter.Document + // names holds each name that a descriptor gave a file, by resolved + // path, so the commit can check every name of a shared file. + names map[string][]string + // keys holds the resolved path of each name at the time of its first + // read, so that a link retargeted after the snapshot cannot move the + // edits of one file onto another before the commit checks the names. + keys map[string]string +} + +func newSourceSnapshot(root string) *sourceSnapshot { + return &sourceSnapshot{ + root: root, + documents: make(map[string]syncadapter.Document), + names: make(map[string][]string), + keys: make(map[string]string), + } +} + +// key returns the resolved path of a repository file, which identifies the +// file however a descriptor names it. A name resolves once per snapshot. +func (snapshot *sourceSnapshot) key(file string) (string, error) { + if key, ok := snapshot.keys[file]; ok { + return key, nil + } + key, err := resolveSourcePath(snapshot.root, file) + if err != nil { + return "", err + } + snapshot.keys[file] = key + return key, nil +} + +func (snapshot *sourceSnapshot) document(file string) (syncadapter.Document, error) { + key, err := snapshot.key(file) + if err != nil { + return syncadapter.Document{}, err + } + if document, ok := snapshot.documents[key]; ok { + if !slices.Contains(snapshot.names[key], file) { + snapshot.names[key] = append(snapshot.names[key], file) + } + return syncadapter.Document{Path: file, Data: document.Data}, nil + } + data, err := os.ReadFile(key) + if err != nil { + return syncadapter.Document{}, fmt.Errorf("read referenced file %q: %w", file, err) + } + generated, err := generatedTarget(snapshot.root, file, key, data) + if err != nil { + return syncadapter.Document{}, err + } + document := syncadapter.Document{Path: file, Data: data, ReadOnly: generated} + snapshot.documents[key] = document + snapshot.names[key] = []string{file} + return document, nil +} + +// read returns the bytes of a file that sync writes, so a generated file is +// a problem here and read-only as a dependency. +func (snapshot *sourceSnapshot) read(file string) ([]byte, error) { + document, err := snapshot.document(file) + if err != nil { + return nil, err + } + if document.ReadOnly { + return nil, generatedProblem(file) + } + return document.Data, nil +} + +// bundle reads a primary source and its static dependencies from the snapshot. +func (snapshot *sourceSnapshot) bundle(reference Reference) (syncadapter.SourceBundle, error) { + content, err := snapshot.read(reference.File) + if err != nil { + return syncadapter.SourceBundle{}, err + } + return sourceBundle(reference, content, snapshot.document) +} + +// withUpdates returns a snapshot that serves the updated bytes of each edited +// file and the original bytes of every other file. +func (snapshot *sourceSnapshot) withUpdates(updated map[string]syncadapter.Document) *sourceSnapshot { + documents := maps.Clone(snapshot.documents) + maps.Copy(documents, updated) + return &sourceSnapshot{root: snapshot.root, documents: documents, names: snapshot.names, keys: snapshot.keys} +} + +// sourceBinding is one descriptor and the resource that it defines. +type sourceBinding struct { + id syncdomain.ResourceID + reference Reference +} + +// alignProjection converts a parsed variation projection to the mode of the +// projection that it is compared with. +func alignProjection(parsed, like syncdomain.Projection) (syncdomain.Projection, error) { + variation, ok := parsed.(syncdomain.VariationProjection) + expected, likeOK := like.(syncdomain.VariationProjection) + if !ok || !likeOK { + return parsed, nil + } + return variation.ForMode(expected.Mode) +} + +// sourceBindings reads every descriptor in the repository. +func (store Store) sourceBindings() ([]sourceBinding, error) { + files, err := SourceFiles(store.repositoryRoot) + if err != nil { + return nil, err + } + root, err := filepath.EvalSymlinks(store.repositoryRoot) + if err != nil { + return nil, fmt.Errorf("resolve repository root: %w", err) + } + fsys := workspaceFS{root: root} + var bindings []sourceBinding + for _, file := range files { + id, ok := ParseReferencePath(file) + if !ok { + continue + } + reference, err := ReadDescriptor(fsys, file, id) + if err != nil { + // A descriptor that sync cannot read has no projection to keep, + // as a source that it cannot read has none. A compile reports + // the descriptor when it matters. + continue + } + bindings = append(bindings, sourceBinding{id: id, reference: reference}) + } + return bindings, nil +} + +// stageSourceDocument stages one checked replacement of a repository file, +// which the descriptors name by each of names. It writes the regular file +// that the source path resolves to, which is the file that compile read, so a +// link inside the repository works on both paths. +func (store Store) stageSourceDocument(names []string, original, updated []byte) (stagedFile, error) { + file := names[0] + path, err := resolveSourcePath(store.repositoryRoot, file) + if err != nil { + return stagedFile{}, err + } + info, err := os.Lstat(path) + if err != nil { + return stagedFile{}, fmt.Errorf("inspect source file %q: %w", file, err) + } + if stat, ok := info.Sys().(*syscall.Stat_t); ok && stat.Nlink > 1 { + return stagedFile{}, publicationProblem(file, "source has multiple hard links", "Use a file with one link.", nil) + } + // A link can point into a vendored or minified tree, so the resolved + // path counts as well as the name in the descriptor. + root, err := filepath.EvalSymlinks(store.repositoryRoot) + if err != nil { + return stagedFile{}, fmt.Errorf("resolve repository root: %w", err) + } + resolved, err := filepath.Rel(root, path) + if err != nil { + return stagedFile{}, fmt.Errorf("resolve source file %q: %w", file, err) + } + if generatedSource(file, original) || generatedSource(filepath.ToSlash(resolved), original) { + return stagedFile{}, publicationProblem(file, "source is generated, vendored, or minified", "Link the authored source file.", nil) + } + return stagedFile{ + displayPath: file, path: path, original: original, originalExists: true, + replacement: updated, mode: info.Mode().Perm(), sources: names, root: store.repositoryRoot, + }, nil +} + +// applySourceEdits applies the edits of one file from the highest offset to +// the lowest. It rejects an edit whose expected bytes changed, and edits that +// overlap. +func applySourceEdits(file string, original []byte, edits []syncadapter.TextEdit) ([]byte, error) { + for _, edit := range edits { + if edit.Start < 0 || edit.End < edit.Start || edit.End > len(original) { + return nil, fmt.Errorf("invalid edit for %q", file) + } + if !bytes.Equal(original[edit.Start:edit.End], edit.Expected) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorStateChanged, "source bytes changed after review", "Run sync again.", + ).WithFile(file, 0, 0) + } + } + slices.SortStableFunc(edits, func(left, right syncadapter.TextEdit) int { + if left.Start != right.Start { + return right.Start - left.Start + } + return right.End - left.End + }) + // Two edits that start at one offset, such as two inserts, have no + // order, so they overlap as a shared range does. + for index := 1; index < len(edits); index++ { + if edits[index-1].Start < edits[index].End || edits[index-1].Start == edits[index].Start { + return nil, syncdomain.NewProblem( + syncdomain.ErrorOverlappingEdits, "source updates overlap", "Use mappings that own separate source values.", + ).WithFile(file, 0, 0) + } + } + updated := bytes.Clone(original) + for _, edit := range edits { + updated = slices.Concat(updated[:edit.Start], edit.Replacement, updated[edit.End:]) + } + return updated, nil +} + +func sameProjection(id syncdomain.ResourceID, left, right syncdomain.Projection) bool { + leftFingerprint, leftErr := syncdomain.FingerprintProjection(id, left) + rightFingerprint, rightErr := syncdomain.FingerprintProjection(id, right) + return leftErr == nil && rightErr == nil && leftFingerprint == rightFingerprint +} diff --git a/internal/sync/local/source_reference.go b/internal/sync/local/source_reference.go new file mode 100644 index 000000000..f2077da8c --- /dev/null +++ b/internal/sync/local/source_reference.go @@ -0,0 +1,205 @@ +package local + +import ( + "errors" + "fmt" + "io/fs" + "maps" + "slices" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncadapter "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +// SourceDefinition is one compiled descriptor-backed resource. +type SourceDefinition struct { + Variation *syncdomain.SyncedResource + Attachment *syncdomain.SyncedAttachment +} + +// ID returns the resource identity of a source definition. +func (definition SourceDefinition) ID() syncdomain.ResourceID { + if definition.Variation != nil { + return definition.Variation.ID() + } + if definition.Attachment != nil { + return definition.Attachment.ID() + } + return syncdomain.ResourceID{} +} + +// CompileSource compiles a proposed reference without writing its descriptor. +func CompileSource(repositoryRoot string, id syncdomain.ResourceID, reference Reference) (SourceDefinition, error) { + bundle, err := ReadSourceBundle(repositoryRoot, reference) + if err != nil { + return SourceDefinition{}, err + } + return compileSource(id, reference, bundle) +} + +// ReadSourceBundle reads a primary source and its static dependencies. +func ReadSourceBundle(repositoryRoot string, reference Reference) (syncadapter.SourceBundle, error) { + return newSourceSnapshot(repositoryRoot).bundle(reference) +} + +// sourceBundle builds the bundle of one source from its content and the +// dependencies that its format reports. read returns one repository file. A +// dependency that does not exist is left out, and does not count toward the +// limit: a format reports several candidate paths for one import, and only +// the file that exists is a dependency. +func sourceBundle( + reference Reference, + content []byte, + read func(file string) (syncadapter.Document, error), +) (syncadapter.SourceBundle, error) { + bundle := syncadapter.SourceBundle{ + Primary: syncadapter.Document{Path: reference.File, Data: content}, + Dependencies: map[string]syncadapter.Document{}, + } + paths, err := syncadapter.DependencyPaths(reference.Format, bundle.Primary) + if err != nil { + return syncadapter.SourceBundle{}, err + } + total := len(content) + if total > 8<<20 { + return syncadapter.SourceBundle{}, fmt.Errorf("source dependencies exceed 8 MiB") + } + for _, dependencyPath := range paths { + document, err := read(dependencyPath) + problem, isProblem := syncdomain.ProblemFrom(err) + if errors.Is(err, fs.ErrNotExist) || + isProblem && problem.Code == syncdomain.ErrorLinkedSourceMissing { + continue + } + if err != nil { + return syncadapter.SourceBundle{}, err + } + total += len(document.Data) + if total > 8<<20 { + return syncadapter.SourceBundle{}, fmt.Errorf("source dependencies exceed 8 MiB") + } + document.Path = dependencyPath + bundle.Dependencies[dependencyPath] = document + if len(bundle.Dependencies) > 32 { + return syncadapter.SourceBundle{}, fmt.Errorf("source has more than 32 dependencies") + } + } + return bundle, nil +} + +// readSourceFileFromFS reads one repository source file from a file system. +// Tests use it with an in-memory file system, which has no links, so the +// name alone tells a generated file. +func readSourceFileFromFS(fsys fs.FS, file string) (syncadapter.Document, error) { + if err := validateSourcePath(file); err != nil { + return syncadapter.Document{}, err + } + data, err := fs.ReadFile(fsys, file) + if err != nil { + return syncadapter.Document{}, fmt.Errorf("read referenced file %q: %w", file, err) + } + return syncadapter.Document{Path: file, Data: data, ReadOnly: generatedSource(file, data)}, nil +} + +// compileDescriptor compiles the resource that one .ref.yaml file defines. +// readAttachment, when set, hydrates the tools and skills of a variation. +func compileDescriptor( + fsys fs.FS, + descriptorPath string, + id syncdomain.ResourceID, + readSource func(file string) (syncadapter.Document, error), + readAttachment func(syncdomain.AttachmentKind, string) (syncdomain.Attachment, error), +) (SourceDefinition, error) { + reference, err := ReadDescriptor(fsys, descriptorPath, id) + if err != nil { + return SourceDefinition{}, err + } + primary, err := readSource(reference.File) + if err != nil { + return SourceDefinition{}, err + } + // Sync writes the primary file, so a generated one blocks the resource. + // A generated dependency only makes the values in it read-only. + if primary.ReadOnly { + return SourceDefinition{}, generatedProblem(reference.File) + } + bundle, err := sourceBundle(reference, primary.Data, readSource) + if err != nil { + return SourceDefinition{}, err + } + definition, err := compileSource(id, reference, bundle) + if err != nil { + return SourceDefinition{}, err + } + if definition.Variation != nil && readAttachment != nil && (reference.Tools != nil || reference.Skills != nil) { + if err := definition.Variation.Variation.HydrateAttachments(readAttachment); err != nil { + return SourceDefinition{}, err + } + } + return definition, nil +} + +// compileSource reads the mapped projection of a source and builds the local +// resource of the kind from it. The descriptor path gives the identity, so +// the key and the display name come from the id. +func compileSource(id syncdomain.ResourceID, reference Reference, bundle syncadapter.SourceBundle) (SourceDefinition, error) { + if generatedSource(reference.File, bundle.Primary.Data) { + return SourceDefinition{}, publicationProblem( + reference.File, "source is generated, vendored, or minified", "Link the authored source file.", nil, + ) + } + projection, indexedSelections, err := syncadapter.ParseSourceWithSelections(bundle, id, reference) + if err != nil { + return SourceDefinition{}, err + } + dependencies := sortedDependencyPaths(bundle) + switch id.Kind { + case syncdomain.KindVariation: + variationProjection, ok := projection.(syncdomain.VariationProjection) + if !ok { + return SourceDefinition{}, fmt.Errorf("source returned %T for a variation", projection) + } + _, variationKey, _ := id.VariationKeys() + variation := syncdomain.ApplyVariationProjection(syncdomain.Variation{ + Key: variationKey, Name: syncdomain.DisplayName(variationKey), + }, variationProjection) + // The descriptor validates its relationships as an agent, because the + // mode comes from the source. A messages source is a completion + // variation, which cannot have skills. + if err := syncdomain.ValidateAttachmentRefs(variation.Mode, variation.Tools, variation.Skills); err != nil { + return SourceDefinition{}, syncdomain.NewProblem( + syncdomain.ErrorInvalidReference, + fmt.Sprintf("descriptor relationships do not fit the %s source: %s", variation.Mode, err), + "Remove the skills from the descriptor, or map a prompt instead of messages.", + ).WithFile(reference.File, 0, 0) + } + return SourceDefinition{Variation: &syncdomain.SyncedResource{ + Kind: id.Kind, ProjectKey: id.ProjectKey, LookupKey: id.LookupKey, + Source: &reference, Dependencies: dependencies, + IndexedSelections: indexedSelections, Projection: projection, Variation: variation, + }}, nil + case syncdomain.KindTool, syncdomain.KindSkill: + attachment := syncdomain.Attachment{Kind: syncdomain.AttachmentKind(id.Kind)} + if id.Kind == syncdomain.KindTool { + attachment.Tool = &syncdomain.Tool{Key: id.LookupKey} + } else { + attachment.Skill = &syncdomain.Skill{Key: id.LookupKey} + } + if attachment, err = syncdomain.ApplyAttachmentProjection(attachment, projection); err != nil { + return SourceDefinition{}, err + } + attachment.Projection = projection + return SourceDefinition{Attachment: &syncdomain.SyncedAttachment{ + ProjectKey: id.ProjectKey, Attachment: attachment, Source: &reference, + Dependencies: dependencies, IndexedSelections: indexedSelections, Projection: projection, + }}, nil + default: + return SourceDefinition{}, fmt.Errorf("unsupported resource kind %q", id.Kind) + } +} + +func sortedDependencyPaths(bundle syncadapter.SourceBundle) []string { + paths := slices.Collect(maps.Keys(bundle.Dependencies)) + slices.Sort(paths) + return paths +} diff --git a/internal/sync/local/source_reference_test.go b/internal/sync/local/source_reference_test.go new file mode 100644 index 000000000..47bfa92ef --- /dev/null +++ b/internal/sync/local/source_reference_test.go @@ -0,0 +1,1632 @@ +package local + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestCompileWorkspaceReadsVariationDescriptor(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "agents.py", `from agents import Agent +support = Agent(instructions="Help with orders.") +`) + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.ref.yaml", `formatVersion: 1 +file: agents.py +format: python +select: 'agents[id="support"]' +map: + prompt: instructions +`) + + resources, err := CompileWorkspace(root) + require.NoError(t, err) + require.Len(t, resources, 1) + assert.Equal(t, syncdomain.VariationID("project", "support", "default"), resources[0].ID()) + assert.Equal(t, "Help with orders.", resources[0].Variation.Instructions) + require.NotNil(t, resources[0].Source) + assert.Equal(t, "agents.py", resources[0].Source.File) +} + +func TestCompileSourceWithoutWritingDescriptor(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help with orders.\n") + id := syncdomain.VariationID("project", "support", "default") + reference := Reference{File: "prompt.md", Format: "markdown"} + + definition, err := CompileSource(root, id, reference) + + require.NoError(t, err) + assert.Equal(t, id, definition.ID()) + require.NotNil(t, definition.Variation) + assert.Equal(t, "Help with orders.", definition.Variation.Variation.Instructions) + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "configs", "support", "default.ref.yaml")) +} + +func TestImportedSourceEditsDependencyAtomically(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/agent.py", `from agents import Agent +from .prompts import SUPPORT_PROMPT +support = Agent(instructions=SUPPORT_PROMPT, model="gpt-5") +`) + writeTestFile(t, root, "app/prompts.py", `SUPPORT_PROMPT = "Original prompt"`+"\n") + id := syncdomain.VariationID("project", "support", "default") + reference := Reference{ + File: "app/agent.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, id, reference) + prompt := "Updated imported prompt" + + paths, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: reference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &prompt, + }, + }}) + + require.NoError(t, err) + assert.Equal(t, []string{"app/prompts.py"}, paths) + primary, err := os.ReadFile(filepath.Join(root, "app", "agent.py")) + require.NoError(t, err) + assert.Contains(t, string(primary), "instructions=SUPPORT_PROMPT") + dependency, err := os.ReadFile(filepath.Join(root, "app", "prompts.py")) + require.NoError(t, err) + assert.Contains(t, string(dependency), `"Updated imported prompt"`) + files, err := SourceFiles(root) + require.NoError(t, err) + assert.Contains(t, files, "app/prompts.py") +} + +func TestReplaceSourcesEditsOneSharedDependencyOnce(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/a.py", "from agents import Agent\nfrom .prompts import A_PROMPT\na = Agent(instructions=A_PROMPT, model=\"gpt-5\")\n") + writeTestFile(t, root, "app/b.py", "from agents import Agent\nfrom .prompts import B_PROMPT\nb = Agent(instructions=B_PROMPT, model=\"gpt-5\")\n") + writeTestFile(t, root, "app/prompts.py", "A_PROMPT = \"Original A\"\nB_PROMPT = \"Original B\"\n") + idA := syncdomain.VariationID("project", "support", "a") + idB := syncdomain.VariationID("project", "support", "b") + referenceA := Reference{File: "app/a.py", Format: "python", Select: `agents[id="a"]`, Map: map[string]string{"prompt": "instructions"}} + referenceB := Reference{File: "app/b.py", Format: "python", Select: `agents[id="b"]`, Map: map[string]string{"prompt": "instructions"}} + writeDescriptorForTest(t, root, idA, referenceA) + writeDescriptorForTest(t, root, idB, referenceB) + + paths, err := NewStore(root).ReplaceSources([]SourceReplacement{ + {ID: idA, Reference: referenceA, + Projection: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("Updated A")}}, + {ID: idB, Reference: referenceB, + Projection: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("Updated B")}}, + }) + + require.NoError(t, err) + assert.Equal(t, []string{"app/prompts.py"}, paths) + shared, err := os.ReadFile(filepath.Join(root, "app", "prompts.py")) + require.NoError(t, err) + assert.Equal(t, "A_PROMPT = \"Updated A\"\nB_PROMPT = \"Updated B\"\n", string(shared)) +} + +func TestReplaceSourcesKeepsBindingsOfAnEditedDependency(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/agent.py", "from agents import Agent\nfrom .prompts import PROMPT\nagent = Agent(instructions=PROMPT, model=\"gpt-5\")\n") + writeTestFile(t, root, "app/prompts.py", "PROMPT = \"Original\"\n") + variationID := syncdomain.VariationID("project", "support", "default") + variationReference := Reference{File: "app/agent.py", Format: "python", Select: `agents[id="agent"]`, Map: map[string]string{"prompt": "instructions"}} + writeDescriptorForTest(t, root, variationID, variationReference) + // A second descriptor binds the same constant through the dependency file. + otherID := syncdomain.VariationID("project", "support", "other") + writeDescriptorForTest(t, root, otherID, Reference{ + File: "app/prompts.py", Format: "python", Select: `constants[id="PROMPT"]`, Map: map[string]string{"prompt": "$"}, + }) + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: variationID, Reference: variationReference, + Projection: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("Updated")}, + }}) + + require.ErrorContains(t, err, "non-target projection") + shared, err := os.ReadFile(filepath.Join(root, "app", "prompts.py")) + require.NoError(t, err) + assert.Equal(t, "PROMPT = \"Original\"\n", string(shared)) +} + +func TestWriteDescriptorCreatesUpdatesAndKeepsIdenticalContent(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "first.md", "First.\n") + writeTestFile(t, root, "second.md", "Second.\n") + id := syncdomain.VariationID("project", "support", "default") + store := NewStore(root) + + path, err := store.WriteDescriptor(id, Reference{File: "first.md", Format: "markdown"}) + require.NoError(t, err) + first, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + _, err = store.WriteDescriptor(id, Reference{File: "first.md", Format: "markdown"}) + require.NoError(t, err) + same, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + assert.Equal(t, first, same) + + _, err = store.WriteDescriptor(id, Reference{File: "second.md", Format: "markdown"}) + require.NoError(t, err) + updated, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + assert.Contains(t, string(updated), "file: second.md") +} + +func TestCompileSourceReportsMissingFile(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + + _, err := CompileSource(root, id, Reference{File: "missing.md", Format: "markdown"}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorLinkedSourceMissing, problem.Code) + assert.Equal(t, "missing.md", problem.File) +} + +func TestCompileWorkspaceRejectsManagedAndDescriptorDefinitions(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.ref.yaml", `formatVersion: 1 +file: prompt.md +format: markdown +`) + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.prompt.md", `--- +formatVersion: 1 +upsert: false +mode: agent +key: default +name: Default +--- + +Help. +`) + + _, err := CompileWorkspace(root) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorDefinitionConflict, problem.Code) +} + +func TestCompileAttachmentsReadsToolAndSkillDescriptors(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "tools.ts", `import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +const server = new McpServer({ name: "orders", version: "1" }); +server.registerTool("lookup", { + description: "Look up an order.", + inputSchema: { type: "object" }, +}, handler); +`) + writeTestFile(t, root, ".agents/skills/refunds/SKILL.md", `--- +name: refunds +description: Explain refunds. +--- + +# Refunds + +Ask for the order ID. +`) + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup"} + skillID := syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"} + writeDescriptorForTest(t, root, toolID, Reference{ + File: "tools.ts", Format: "typescript", Select: `tools[id="lookup"]`, + Map: map[string]string{"description": "description", "schema": "inputSchema"}, + }) + writeDescriptorForTest(t, root, skillID, Reference{ + File: ".agents/skills/refunds/SKILL.md", Format: "agent-skill", + }) + + attachments, err := CompileAttachments(root, func(syncdomain.ResourceID) bool { return true }) + + require.NoError(t, err) + require.Len(t, attachments, 2) + byID := map[syncdomain.ResourceID]syncdomain.SyncedAttachment{} + for _, attachment := range attachments { + byID[attachment.ID()] = attachment + } + assert.Equal(t, "Look up an order.", *byID[toolID].Attachment.Tool.Description) + assert.Equal(t, "Explain refunds.", byID[skillID].Attachment.Skill.Description) +} + +func TestReplaceSourcesGroupsEditsAndPreservesOtherBytes(t *testing.T) { + root := t.TempDir() + original := `from agents import Agent + +support = Agent(instructions="Help with orders.") +refund = Agent(instructions='Explain refunds.') +` + writeTestFile(t, root, "agents.py", original) + supportID := syncdomain.VariationID("project", "support", "default") + refundID := syncdomain.VariationID("project", "refund", "default") + supportReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + refundReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="refund"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, supportID, supportReference) + writeDescriptorForTest(t, root, refundID, refundReference) + + supportPrompt := "Help with orders and shipping." + refundPrompt := "Explain refunds and escalation." + paths, err := NewStore(root).ReplaceSources([]SourceReplacement{ + { + ID: supportID, Reference: supportReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &supportPrompt, + }, + }, + { + ID: refundID, Reference: refundReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &refundPrompt, + }, + }, + }) + require.NoError(t, err) + assert.Equal(t, []string{"agents.py"}, paths) + content, err := os.ReadFile(filepath.Join(root, "agents.py")) + require.NoError(t, err) + assert.Equal(t, `from agents import Agent + +support = Agent(instructions="Help with orders and shipping.") +refund = Agent(instructions='Explain refunds and escalation.') +`, string(content)) +} + +func TestReplaceSourcesGroupsStructuredFormatEdits(t *testing.T) { + tests := []struct { + name string + file string + format string + source string + firstID syncdomain.ResourceID + secondID syncdomain.ResourceID + firstSelect string + secondSelect string + fieldMap map[string]string + first syncdomain.Projection + second syncdomain.Projection + want []string + }{ + { + name: "JSON tools", file: "tools.json", format: "json", + source: `{"tools":[{"name":"lookup","description":"Old lookup"},{"name":"refund","description":"Old refund"}]}` + "\n", + firstID: syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup"}, + secondID: syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "refund"}, + firstSelect: `tools[name="lookup"]`, secondSelect: `tools[name="refund"]`, + fieldMap: map[string]string{"description": "description"}, + first: syncdomain.ToolProjection{Description: stringPointer("New lookup")}, + second: syncdomain.ToolProjection{Description: stringPointer("New refund")}, + want: []string{`"description":"New lookup"`, `"description":"New refund"`}, + }, + { + name: "YAML agents", file: "agents.yaml", format: "yaml", + source: "agents:\n - name: support\n prompt: Old support\n - name: refunds\n prompt: Old refunds\n", + firstID: syncdomain.VariationID("project", "support", "default"), + secondID: syncdomain.VariationID("project", "refunds", "default"), + firstSelect: `agents[name="support"]`, secondSelect: `agents[name="refunds"]`, + fieldMap: map[string]string{"prompt": "prompt"}, + first: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New support")}, + second: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New refunds")}, + want: []string{"prompt: New support", "prompt: New refunds"}, + }, + { + name: "TOML agents", file: "agents.toml", format: "toml", + source: "[agents.support]\nprompt = \"Old support\"\n[agents.refunds]\nprompt = \"Old refunds\"\n", + firstID: syncdomain.VariationID("project", "support", "default"), + secondID: syncdomain.VariationID("project", "refunds", "default"), + firstSelect: "agents.support", secondSelect: "agents.refunds", + fieldMap: map[string]string{"prompt": "prompt"}, + first: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New support")}, + second: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New refunds")}, + want: []string{`prompt = "New support"`, `prompt = "New refunds"`}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, test.file, test.source) + firstReference := Reference{ + File: test.file, Format: test.format, Select: test.firstSelect, Map: test.fieldMap, + } + secondReference := Reference{ + File: test.file, Format: test.format, Select: test.secondSelect, Map: test.fieldMap, + } + writeDescriptorForTest(t, root, test.firstID, firstReference) + writeDescriptorForTest(t, root, test.secondID, secondReference) + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{ + {ID: test.firstID, Reference: firstReference, Projection: test.first}, + {ID: test.secondID, Reference: secondReference, Projection: test.second}, + }) + + require.NoError(t, err) + content, err := os.ReadFile(filepath.Join(root, test.file)) + require.NoError(t, err) + for _, expected := range test.want { + assert.Contains(t, string(content), expected) + } + }) + } +} + +func TestReplaceSourcesRejectsOverlappingEdits(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "tool.json", `{"description":"Old"}`+"\n") + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + reference := Reference{ + File: "tool.json", Format: "json", Select: "$", + Map: map[string]string{"description": "description"}, + } + writeDescriptorForTest(t, root, id, reference) + description := "New" + replacement := SourceReplacement{ + ID: id, Reference: reference, Projection: syncdomain.ToolProjection{Description: &description}, + } + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{replacement, replacement}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorOverlappingEdits, problem.Code) + content, readErr := os.ReadFile(filepath.Join(root, "tool.json")) + require.NoError(t, readErr) + assert.Equal(t, `{"description":"Old"}`+"\n", string(content)) +} + +func TestReplaceSourcesWritesNothingWhenOneFieldIsReadOnly(t *testing.T) { + root := t.TempDir() + original := `from agents import Agent +support = Agent(instructions="Help.") +dynamic = Agent(instructions=load_prompt()) +` + writeTestFile(t, root, "agents.py", original) + supportID := syncdomain.VariationID("project", "support", "default") + dynamicID := syncdomain.VariationID("project", "dynamic", "default") + supportReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + dynamicReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="dynamic"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, supportID, supportReference) + writeDescriptorForTest(t, root, dynamicID, dynamicReference) + supportPrompt, dynamicPrompt := "Changed.", "Cannot write." + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{ + { + ID: supportID, Reference: supportReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &supportPrompt, + }, + }, + { + ID: dynamicID, Reference: dynamicReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &dynamicPrompt, + }, + }, + }) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorFieldReadOnly, problem.Code) + content, readErr := os.ReadFile(filepath.Join(root, "agents.py")) + require.NoError(t, readErr) + assert.Equal(t, original, string(content)) +} + +func TestReplaceSourcesRejectsMappedChangeAfterReview(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "agents.py", `from agents import Agent +support = Agent(instructions="Changed after review.") +`) + id := syncdomain.VariationID("project", "support", "default") + reference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, id, reference) + desired := "Server value." + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: reference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &desired, + }, + ExpectedFingerprint: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + }}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code) + content, readErr := os.ReadFile(filepath.Join(root, "agents.py")) + require.NoError(t, readErr) + assert.Contains(t, string(content), "Changed after review.") +} + +func TestCommitFilesRollsBackEarlierReplacement(t *testing.T) { + root := t.TempDir() + firstPath, secondPath := filepath.Join(root, "first.txt"), filepath.Join(root, "second.txt") + require.NoError(t, os.WriteFile(firstPath, []byte("first original"), 0o644)) + require.NoError(t, os.WriteFile(secondPath, []byte("second original"), 0o644)) + firstStaged, err := writeTempFile(firstPath, []byte("first updated"), 0o644) + require.NoError(t, err) + secondStaged, err := writeTempFile(secondPath, []byte("second updated"), 0o644) + require.NoError(t, err) + t.Cleanup(func() { + _ = os.Remove(firstStaged) + _ = os.Remove(secondStaged) + }) + files := []stagedFile{ + { + displayPath: "first.txt", path: firstPath, original: []byte("first original"), + originalExists: true, replacement: []byte("first updated"), mode: 0o644, stagedPath: firstStaged, + }, + { + displayPath: "second.txt", path: secondPath, original: []byte("second original"), + originalExists: true, replacement: []byte("second updated"), mode: 0o644, stagedPath: secondStaged, + }, + } + + err = commitFilesWithRename(files, func(source, destination string) error { + if source == secondStaged { + return errors.New("injected rename failure") + } + return os.Rename(source, destination) + }) + + require.ErrorContains(t, err, "injected rename failure") + first, readErr := os.ReadFile(firstPath) + require.NoError(t, readErr) + second, readErr := os.ReadFile(secondPath) + require.NoError(t, readErr) + assert.Equal(t, "first original", string(first)) + assert.Equal(t, "second original", string(second)) +} + +func TestReplaceSourcesPullsRelationshipsAndMissingDefinitionAtomically(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + writeTestFile(t, root, descriptorPath, + "# keep\nformatVersion: 1\nfile: prompt.md\nformat: markdown\n"+ + "tools:\n - key: old\n# tail\n") + old := []syncdomain.AttachmentRef{{Key: "old"}} + next := []string{"new"} + projection := syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, + Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + } + tool := syncdomain.Attachment{ + Kind: syncdomain.AttachmentTool, + Tool: &syncdomain.Tool{Key: "new", Schema: map[string]any{"type": "object"}}, + } + + paths, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &old}, + Projection: projection, RelationshipAttachments: []syncdomain.Attachment{tool}, + }}) + + require.NoError(t, err) + assert.Contains(t, paths, descriptorPath) + assert.Contains(t, paths, ".launchdarkly/project/tools/new.json", "every reported path is relative to the repository") + descriptor, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(descriptorPath))) + require.NoError(t, err) + assert.Contains(t, string(descriptor), "# keep\n") + assert.Contains(t, string(descriptor), "tools:\n - key: new\n# tail\n") + require.FileExists(t, filepath.Join(root, ".launchdarkly/project/tools/new.json")) +} + +func TestReplaceSourcesWritesNothingWhenRelationshipDependencyIsInvalid(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + original := "formatVersion: 1\nfile: prompt.md\nformat: markdown\ntools: []\n" + writeTestFile(t, root, descriptorPath, original) + empty := []syncdomain.AttachmentRef{} + next := []string{"new"} + projection := syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + } + + _, err = NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &empty}, + Projection: projection, + RelationshipAttachments: []syncdomain.Attachment{{ + Kind: syncdomain.AttachmentTool, + Tool: &syncdomain.Tool{Key: "new"}, + }}, + }}) + + require.ErrorContains(t, err, `tool "new" schema is required`) + descriptor, readErr := os.ReadFile(filepath.Join(root, filepath.FromSlash(descriptorPath))) + require.NoError(t, readErr) + assert.Equal(t, original, string(descriptor)) + require.NoFileExists(t, filepath.Join(root, ".launchdarkly/project/tools/new.json")) +} + +func TestReplaceSourcesAddsRelationshipNodesWithoutChangingOtherBytes(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + original := "# keep\nformatVersion: 1\nfile: prompt.md\nformat: markdown" + writeTestFile(t, root, descriptorPath, original) + writeTestFile(t, root, ".launchdarkly/project/tools/search.json", + `{"formatVersion":1,"key":"search","schema":{"type":"object"}}`) + tools := []string{"search"} + skills := []string{} + projection := syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &tools, Skills: &skills, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + syncdomain.FieldSkills: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + } + + _, err = NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown"}, + Projection: projection, + }}) + + require.NoError(t, err) + descriptor, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(descriptorPath))) + require.NoError(t, err) + assert.Equal(t, original+"\ntools:\n - key: search\nskills: []\n", string(descriptor)) +} + +func TestReplaceSourcesRejectsChangedDescriptorRelationships(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + original := "formatVersion: 1\nfile: prompt.md\nformat: markdown\ntools:\n - key: changed\n" + writeTestFile(t, root, descriptorPath, original) + old := []syncdomain.AttachmentRef{{Key: "old"}} + next := []string{} + projection := syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + } + + _, err = NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &old}, + Projection: projection, + }}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code) + descriptor, readErr := os.ReadFile(filepath.Join(root, filepath.FromSlash(descriptorPath))) + require.NoError(t, readErr) + assert.Equal(t, original, string(descriptor)) +} + +func TestReplaceSourcesNeedsContentForNewRelationshipDefinition(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + writeTestFile(t, root, descriptorPath, + "formatVersion: 1\nfile: prompt.md\nformat: markdown\ntools: []\n") + empty := []syncdomain.AttachmentRef{} + next := []string{"missing"} + projection := syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + } + + _, err = NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &empty}, + Projection: projection, + }}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorAttachmentDefinitionMissing, problem.Code) + require.NoFileExists(t, filepath.Join(root, ".launchdarkly/project/tools/missing.json")) +} + +func writeDescriptorForTest(t *testing.T, root string, id syncdomain.ResourceID, reference Reference) { + t.Helper() + path, err := ReferencePath(id) + require.NoError(t, err) + content, err := RenderDescriptor(id, reference) + require.NoError(t, err) + writeTestFile(t, root, path, string(content)) +} + +func writeTestFile(t *testing.T, root, name, content string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(name)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} + +func stringPointer(value string) *string { + return &value +} + +func TestRenderAndWriteDescriptorRejectAnInvalidSourcePath(t *testing.T) { + id := syncdomain.VariationID("project", "support", "default") + for name, file := range map[string]string{ + "managed directory": ".launchdarkly/prompt.md", + "unclean path": "./prompt.md", + "parent path": "../prompt.md", + "empty": "", + } { + t.Run(name, func(t *testing.T) { + _, err := RenderDescriptor(id, Reference{File: file, Format: "markdown"}) + require.Error(t, err) + _, err = NewStore(t.TempDir()).WriteDescriptor(id, Reference{File: file, Format: "markdown"}) + require.Error(t, err) + }) + } +} + +func TestWriteDescriptorRejectsASymlinkedParentOnReplace(t *testing.T) { + root := t.TempDir() + external := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + writeTestFile(t, external, "default.ref.yaml", "formatVersion: 1\nfile: old.md\nformat: markdown\n") + configs := filepath.Join(root, ".launchdarkly", "project", "configs") + require.NoError(t, os.MkdirAll(configs, 0o755)) + require.NoError(t, os.Symlink(external, filepath.Join(configs, "support"))) + + _, err := NewStore(root).WriteDescriptor(id, Reference{File: "prompt.md", Format: "markdown"}) + + require.ErrorContains(t, err, "symbolic links are not supported") + data, err := os.ReadFile(filepath.Join(external, "default.ref.yaml")) + require.NoError(t, err) + assert.Contains(t, string(data), "file: old.md", "the file outside the managed tree is unchanged") +} + +func TestAddKeepsALinkedToolThatTheVariationUses(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "tool.json", `{"description":"Linked","schema":{"type":"object"}}`) + writeTestFile(t, root, ".launchdarkly/project/tools/search.ref.yaml", + "formatVersion: 1\nfile: tool.json\nformat: json\nselect: $\nmap:\n description: description\n schema: schema\n") + search := syncdomain.Tool{Key: "search", Schema: map[string]any{"type": "object"}} + variation := syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default", Instructions: "Help.", + Tools: []syncdomain.AttachmentRef{{Key: "search"}}, + Attachments: []syncdomain.Attachment{{Kind: syncdomain.AttachmentTool, Tool: &search}}, + } + + creation, err := NewStore(root).Add([]VariationFile{{ProjectKey: "project", ConfigKey: "config", Variation: variation}}) + + require.NoError(t, err) + assert.Equal(t, []string{"project/configs/config/default.prompt.md"}, creation.CreatedPaths) + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "tools", "search.json")) + resources, err := CompileWorkspace(root) + require.NoError(t, err) + require.Len(t, resources, 1) + assert.Equal(t, "Linked", *resources[0].Variation.Attachments[0].Tool.Description) +} + +func TestCompileSourceRejectsSkillsOnACompletionSource(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompts.py", "from langchain_core.prompts import ChatPromptTemplate\n"+ + "chat = ChatPromptTemplate.from_messages([(\"human\", \"Hi\")])\n") + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.ref.yaml", + "formatVersion: 1\nfile: prompts.py\nformat: python\nselect: 'prompts[id=\"chat\"]'\n"+ + "map:\n messages: messages\nskills:\n - key: refunds\n") + + _, err := CompileWorkspace(root) + + require.ErrorContains(t, err, "do not fit the completion source") + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorInvalidReference, problem.Code) +} + +func TestStageSourceDocumentFollowsARepositoryLink(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "lib/prompt.md", "Old\n") + require.NoError(t, os.Symlink(filepath.Join(root, "lib"), filepath.Join(root, "linked"))) + + staged, err := NewStore(root).stageSourceDocument([]string{"linked/prompt.md"}, []byte("Old\n"), []byte("New\n")) + + require.NoError(t, err) + assert.Equal(t, "linked/prompt.md", staged.displayPath) + resolved, err := filepath.EvalSymlinks(filepath.Join(root, "lib", "prompt.md")) + require.NoError(t, err) + assert.Equal(t, resolved, staged.path, "the write targets the regular file that compile read") +} + +func TestSourcePathsStayOutsideTheManagedDirectoryAfterResolution(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, ".launchdarkly/sync.lock", "formatVersion: 1\n") + require.NoError(t, os.Symlink(filepath.Join(root, ".launchdarkly"), filepath.Join(root, "managed"))) + + _, err := ReadSourceFile(root, "managed/sync.lock") + require.ErrorContains(t, err, "resolves inside .launchdarkly") + assert.Error(t, validateSourcePath(".LaunchDarkly/sync.lock"), "the check ignores case") + assert.NoError(t, validateSourcePath("launchdarkly/notes.md"), "a similar name outside the directory is fine") +} + +func TestRelationshipPullRejectsASymlinkedDescriptorParent(t *testing.T) { + root := t.TempDir() + external := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help\n") + writeTestFile(t, external, "default.ref.yaml", "formatVersion: 1\nfile: prompt.md\nformat: markdown\ntools:\n - key: old\n") + configs := filepath.Join(root, ".launchdarkly", "project", "configs") + require.NoError(t, os.MkdirAll(configs, 0o755)) + require.NoError(t, os.Symlink(external, filepath.Join(configs, "support"))) + id := syncdomain.VariationID("project", "support", "default") + projection := syncdomain.VariationProjection{ + Tools: &[]string{"new"}, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + } + old := []syncdomain.AttachmentRef{{Key: "old"}} + + _, _, err := NewStore(root).prepareDescriptorRelationshipReplacement(SourceReplacement{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &old}, Projection: projection, + }) + + require.ErrorContains(t, err, "symbolic links are not supported") +} + +func TestPublishSharesOneSnapshotForTwoNamesOfOneFile(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "lib/prompts.json", `{"first": "one", "second": "two"}`) + require.NoError(t, os.Symlink(filepath.Join(root, "lib"), filepath.Join(root, "linked"))) + first := syncdomain.VariationID("project", "support", "first") + second := syncdomain.VariationID("project", "support", "second") + store := NewStore(root) + _, err := store.WriteDescriptor(first, Reference{File: "lib/prompts.json", Format: "json", Select: "$", Map: map[string]string{"prompt": "first"}}) + require.NoError(t, err) + _, err = store.WriteDescriptor(second, Reference{File: "linked/prompts.json", Format: "json", Select: "$", Map: map[string]string{"prompt": "second"}}) + require.NoError(t, err) + resources, err := CompileWorkspace(root) + require.NoError(t, err) + require.Len(t, resources, 2) + replacements := make([]SourceReplacement, 0, 2) + for _, resource := range resources { + projection := resource.Projection.(syncdomain.VariationProjection) + next := projection.Instructions + *next = *next + " updated" + replacements = append(replacements, SourceReplacement{ + ID: resource.ID(), Reference: *resource.Source, Projection: projection, + }) + } + + paths, err := store.ReplaceSources(replacements) + + require.NoError(t, err) + assert.Len(t, paths, 1, "one file is written once") + data, err := os.ReadFile(filepath.Join(root, "lib", "prompts.json")) + require.NoError(t, err) + assert.JSONEq(t, `{"first": "one updated", "second": "two updated"}`, string(data)) +} + +func TestStageSourceDocumentRejectsAResolvedGeneratedPath(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "vendor/prompt.md", "Old\n") + require.NoError(t, os.Symlink(filepath.Join(root, "vendor"), filepath.Join(root, "authored"))) + + _, err := NewStore(root).stageSourceDocument([]string{"authored/prompt.md"}, []byte("Old\n"), []byte("New\n")) + + require.ErrorContains(t, err, "generated, vendored, or minified") +} + +func TestRelationshipPullMovesATrailingCommentWithTheList(t *testing.T) { + original := []byte("formatVersion: 1\nfile: prompt.md\nformat: markdown\ntools:\n - key: old\n # keep this tool list short\nskills: []\n") + + updated, err := updateDescriptorRelationships(original, []syncdomain.AttachmentRef{{Key: "new"}}, true, nil, false) + + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(string(updated), "# keep this tool list short"), "the comment appears once") + assert.Contains(t, string(updated), "key: new") + assert.NotContains(t, string(updated), "key: old") +} + +func TestRestoreDescriptorPutsThePreviousContentBackOrRemovesIt(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + id := syncdomain.VariationID("project", "support", "default") + writeTestFile(t, root, "prompt.md", "Help\n") + path, err := store.WriteDescriptor(id, Reference{File: "prompt.md", Format: "markdown"}) + require.NoError(t, err) + previous, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + + _, err = store.WriteDescriptor(id, Reference{File: "prompt.md", Format: "markdown", Select: "$"}) + require.NoError(t, err) + written, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + require.NoError(t, store.RestoreDescriptor(id, previous, written)) + current, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + assert.Equal(t, previous, current) + + require.NoError(t, store.RestoreDescriptor(id, nil, []byte("other content\n"))) + require.FileExists(t, filepath.Join(root, filepath.FromSlash(path)), "a descriptor with other content than the link wrote stays") + + require.NoError(t, store.RestoreDescriptor(id, nil, previous)) + require.NoFileExists(t, filepath.Join(root, filepath.FromSlash(path))) +} + +func TestSourcePathsRejectWindowsAbsolutePaths(t *testing.T) { + for _, file := range []string{`C:/prompts/prompt.md`, `c:prompt.md`, `prompts\prompt.md`, `\\server\share\prompt.md`} { + assert.Error(t, validateSourcePath(file), file) + } + assert.NoError(t, validateSourcePath("prompts/prompt.md")) +} + +func TestReplaceVariationsKeepsALinkedToolDefinition(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "tool.json", `{"description":"Linked","schema":{"type":"object"}}`) + writeTestFile(t, root, ".launchdarkly/project/tools/search.ref.yaml", + "formatVersion: 1\nfile: tool.json\nformat: json\nselect: $\nmap:\n description: description\n schema: schema\n") + search := syncdomain.Tool{Key: "search", Description: new(string), Schema: map[string]any{"type": "object"}} + *search.Description = "From LaunchDarkly" + variation := syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default", Instructions: "Help.", + Tools: []syncdomain.AttachmentRef{{Key: "search"}}, + Attachments: []syncdomain.Attachment{{Kind: syncdomain.AttachmentTool, Tool: &search}}, + } + + _, err := NewStore(root).ReplaceVariations([]VariationReplacement{{ + ProjectKey: "project", ConfigKey: "config", CreateIfMissing: true, Variation: variation, + }}) + + require.NoError(t, err) + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "tools", "search.json"), "the descriptor stays the one definition") + resources, err := CompileWorkspace(root) + require.NoError(t, err) + require.Len(t, resources, 1) +} + +func TestReplaceVariationsRejectsAManagedFileBesideADescriptor(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help\n") + writeTestFile(t, root, ".launchdarkly/project/configs/config/default.ref.yaml", "formatVersion: 1\nfile: prompt.md\nformat: markdown\n") + variation := syncdomain.Variation{Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default", Instructions: "Help."} + + _, err := NewStore(root).ReplaceVariations([]VariationReplacement{{ + ProjectKey: "project", ConfigKey: "config", CreateIfMissing: true, Variation: variation, + }}) + + require.ErrorContains(t, err, "already has a descriptor") + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "configs", "config", "default.prompt.md")) +} + +func TestReplaceAttachmentsRejectsAManagedFileBesideADescriptor(t *testing.T) { + root := t.TempDir() + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search"} + writeTestFile(t, root, "tool.json", `{"description":"Linked","schema":{"type":"object"}}`) + writeDescriptorForTest(t, root, id, Reference{ + File: "tool.json", Format: "json", Select: "$", Map: map[string]string{"description": "description", "schema": "schema"}, + }) + writeTestFile(t, root, ".launchdarkly/project/tools/search.json", `{"formatVersion": 1, "key": "search", "schema": {}}`) + description := "Server" + search := syncdomain.Attachment{Kind: syncdomain.AttachmentTool, Version: 2, Tool: &syncdomain.Tool{ + Key: "search", Description: &description, Schema: map[string]any{"type": "object"}, + }} + + _, err := NewStore(root).ReplaceAttachments([]syncdomain.SyncedAttachment{{ProjectKey: "project", Attachment: search}}) + + require.ErrorContains(t, err, "has a managed file and a descriptor") + data, readErr := os.ReadFile(filepath.Join(root, ".launchdarkly", "project", "tools", "search.json")) + require.NoError(t, readErr) + assert.NotContains(t, string(data), "Server", "the managed file is not written") +} + +func TestAddRejectsAToolManagedFileBesideADescriptor(t *testing.T) { + root := t.TempDir() + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search"} + writeTestFile(t, root, "tool.json", `{"description":"Linked","schema":{"type":"object"}}`) + writeDescriptorForTest(t, root, id, Reference{ + File: "tool.json", Format: "json", Select: "$", Map: map[string]string{"description": "description", "schema": "schema"}, + }) + writeTestFile(t, root, ".launchdarkly/project/tools/search.json", `{"formatVersion": 1, "key": "search", "schema": {}}`) + variation := syncdomain.Variation{Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default"} + variation.Tools = []syncdomain.AttachmentRef{{Key: "search"}} + variation.Attachments = []syncdomain.Attachment{{Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{ + Key: "search", Schema: map[string]any{"type": "object"}, + }}} + + _, err := NewStore(root).Add([]VariationFile{{ProjectKey: "project", ConfigKey: "config", Variation: variation}}) + + require.ErrorContains(t, err, "has a managed file and a descriptor") + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "configs", "config", "default.prompt.md"), "nothing is created") +} + +func TestReplaceAttachmentsSkipsALeftoverLinkBesideADescriptor(t *testing.T) { + root := t.TempDir() + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search"} + writeTestFile(t, root, "tool.json", `{"description":"Linked","schema":{"type":"object"}}`) + writeDescriptorForTest(t, root, id, Reference{ + File: "tool.json", Format: "json", Select: "$", Map: map[string]string{"description": "description", "schema": "schema"}, + }) + writeTestFile(t, root, "external.json", `{"formatVersion": 1, "key": "search", "schema": {}}`) + managedPath, err := ResourcePath(id) + require.NoError(t, err) + target := filepath.Join(root, filepath.FromSlash(managedPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.Symlink(filepath.Join(root, "external.json"), target)) + description := "Server" + search := syncdomain.Attachment{Kind: syncdomain.AttachmentTool, Version: 2, Tool: &syncdomain.Tool{ + Key: "search", Description: &description, Schema: map[string]any{"type": "object"}, + }} + + written, err := NewStore(root).ReplaceAttachments([]syncdomain.SyncedAttachment{{ProjectKey: "project", Attachment: search}}) + + require.NoError(t, err, "a leftover link is not a definition, so the descriptor keeps the one definition") + assert.Empty(t, written) + info, err := os.Lstat(target) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&os.ModeSymlink, "the link stays") +} + +func TestReplaceVariationsTreatsALeftoverLinkAsAbsent(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + writeTestFile(t, root, "prompt.md", "Help.\n") + writeDescriptorForTest(t, root, id, Reference{File: "prompt.md", Format: "markdown"}) + writeTestFile(t, root, "external.md", "---\nformatVersion: 1\nmode: agent\nkey: default\n---\n\nOther.\n") + managedPath, err := ResourcePath(id) + require.NoError(t, err) + target := filepath.Join(root, filepath.FromSlash(managedPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.Symlink(filepath.Join(root, "external.md"), target)) + + _, err = NewStore(root).ReplaceVariations([]VariationReplacement{{ + ProjectKey: "project", ConfigKey: "support", CreateIfMissing: true, + Variation: syncdomain.Variation{Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default"}, + }}) + + require.ErrorContains(t, err, "already has a descriptor", "the link is absent, so the descriptor is the one definition") +} + +func TestReplaceVariationsWritesOverALeftoverLink(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + writeTestFile(t, root, "external.md", "---\nformatVersion: 1\nmode: agent\nkey: default\n---\n\nOther.\n") + managedPath, err := ResourcePath(id) + require.NoError(t, err) + target := filepath.Join(root, filepath.FromSlash(managedPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.Symlink(filepath.Join(root, "external.md"), target)) + + written, err := NewStore(root).ReplaceVariations([]VariationReplacement{{ + ProjectKey: "project", ConfigKey: "support", CreateIfMissing: true, + Variation: syncdomain.Variation{Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default", Instructions: "Written."}, + }}) + + require.NoError(t, err, "a leftover link without a descriptor is absent, so the create writes the file") + require.Len(t, written, 1) + info, err := os.Lstat(target) + require.NoError(t, err) + assert.True(t, info.Mode().IsRegular(), "the managed file replaces the link") + data, err := os.ReadFile(target) + require.NoError(t, err) + assert.Contains(t, string(data), "Written.") + external, err := os.ReadFile(filepath.Join(root, "external.md")) + require.NoError(t, err) + assert.Contains(t, string(external), "Other.", "the target of the link is not written") +} + +func TestReplaceFilesStopsWhenASourceLinkWasRetargeted(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "lib/old.md", "old") + writeTestFile(t, root, "lib/new.md", "new") + // The snapshot resolved the link to the old file. The link now names + // the new file. + require.NoError(t, os.Symlink(filepath.Join(root, "lib", "new.md"), filepath.Join(root, "prompt.md"))) + + err := replaceFiles([]stagedFile{{ + displayPath: "prompt.md", path: filepath.Join(root, "lib", "old.md"), original: []byte("old"), originalExists: true, + replacement: []byte("edited"), mode: 0o644, sources: []string{"prompt.md"}, root: root, + }}) + + require.ErrorContains(t, err, "resolves to another file") + data, readErr := os.ReadFile(filepath.Join(root, "lib", "old.md")) + require.NoError(t, readErr) + assert.Equal(t, "old", string(data), "the file that the name no longer names is not written") +} + +func TestReplaceFilesVerifiesAFileThatItDoesNotWrite(t *testing.T) { + root := t.TempDir() + written := filepath.Join(root, "written.txt") + guarded := filepath.Join(root, "guarded.txt") + require.NoError(t, os.WriteFile(written, []byte("old"), 0o644)) + require.NoError(t, os.WriteFile(guarded, []byte("current"), 0o644)) + + err := replaceFiles([]stagedFile{ + {displayPath: "written.txt", path: written, original: []byte("old"), originalExists: true, replacement: []byte("new"), mode: 0o644}, + {displayPath: "guarded.txt", path: guarded, original: []byte("checked"), originalExists: true, verifyOnly: true}, + }) + + require.ErrorContains(t, err, "guarded.txt changed while syncing") + data, readErr := os.ReadFile(written) + require.NoError(t, readErr) + assert.Equal(t, "old", string(data), "nothing is written when a guarded file changed") + + err = replaceFiles([]stagedFile{ + {displayPath: "written.txt", path: written, original: []byte("old"), originalExists: true, replacement: []byte("new"), mode: 0o644}, + {displayPath: "guarded.txt", path: guarded, original: []byte("current"), originalExists: true, verifyOnly: true}, + }) + + require.NoError(t, err) + data, readErr = os.ReadFile(written) + require.NoError(t, readErr) + assert.Equal(t, "new", string(data)) + data, readErr = os.ReadFile(guarded) + require.NoError(t, readErr) + assert.Equal(t, "current", string(data), "a guarded file is not written") +} + +func TestWriteDescriptorFromStopsWhenTheDescriptorChanged(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + store := NewStore(root) + id := syncdomain.VariationID("project", "support", "default") + first := Reference{File: "prompt.md", Format: "markdown"} + second := Reference{File: "prompt.md", Format: "markdown", Select: "$"} + + _, err := store.WriteDescriptorFrom(id, first, nil) + require.NoError(t, err, "a create with no previous descriptor") + path, err := ReferencePath(id) + require.NoError(t, err) + written, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + + _, err = store.WriteDescriptorFrom(id, second, nil) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok, "a create over a descriptor that appeared is a problem") + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code) + + _, err = store.WriteDescriptorFrom(id, second, []byte("other content\n")) + problem, ok = syncdomain.ProblemFrom(err) + require.True(t, ok, "a replace of a descriptor with other content is a problem") + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code) + current, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + assert.Equal(t, written, current, "the descriptor is not written") + + _, err = store.WriteDescriptorFrom(id, second, written) + require.NoError(t, err, "a replace of the expected content proceeds") + current, err = os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + assert.Contains(t, string(current), "select: $") +} + +func TestSourceBindingsSkipAnUnreadableDescriptor(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + writeDescriptorForTest(t, root, id, Reference{File: "prompt.md", Format: "markdown"}) + toolPath, err := ReferencePath(syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "broken"}) + require.NoError(t, err) + writeTestFile(t, root, toolPath, "invalid: [\n") + + bindings, err := NewStore(root).sourceBindings() + + require.NoError(t, err, "an unreadable descriptor of another resource does not block a publish") + require.Len(t, bindings, 1) + assert.Equal(t, id, bindings[0].id) +} + +func TestReplaceSourcesRejectsADescriptorChangedAfterReview(t *testing.T) { + root := t.TempDir() + original := `{"first": "one", "second": "two"}` + writeTestFile(t, root, "prompts.json", original) + id := syncdomain.VariationID("project", "support", "default") + reviewed := Reference{File: "prompts.json", Format: "json", Select: "$", Map: map[string]string{"prompt": "first"}} + writeDescriptorForTest(t, root, id, reviewed) + prompt := "changed" + replacement := SourceReplacement{ + ID: id, Reference: reviewed, Projection: syncdomain.VariationProjection{Instructions: &prompt}, + } + + retargeted := reviewed + retargeted.Map = map[string]string{"prompt": "second"} + writeDescriptorForTest(t, root, id, retargeted) + _, err := NewStore(root).ReplaceSources([]SourceReplacement{replacement}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code) + assert.Equal(t, "the source descriptor changed after review", problem.Message) + content, err := os.ReadFile(filepath.Join(root, "prompts.json")) + require.NoError(t, err) + assert.Equal(t, original, string(content), "a retargeted descriptor writes nothing") + + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + require.NoError(t, os.Remove(filepath.Join(root, descriptorPath))) + _, err = NewStore(root).ReplaceSources([]SourceReplacement{replacement}) + problem, ok = syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code, "a removed descriptor writes nothing") +} + +func TestMissingRelationshipAttachmentsReportAnInvalidKey(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + tools := []string{"../escape"} + + _, err := NewStore(root).prepareMissingRelationshipAttachments([]SourceReplacement{{ + ID: id, + Projection: syncdomain.VariationProjection{ + Tools: &tools, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + }}) + + require.Error(t, err, "an invalid key is reported instead of treated as a present definition") +} + +func TestSourceFilesIncludeTheResolvedTargetOfALink(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "lib/prompts.json", `{"prompt": "one"}`) + require.NoError(t, os.Symlink(filepath.Join(root, "lib"), filepath.Join(root, "linked"))) + id := syncdomain.VariationID("project", "support", "default") + writeDescriptorForTest(t, root, id, Reference{ + File: "linked/prompts.json", Format: "json", Select: "$", Map: map[string]string{"prompt": "prompt"}, + }) + + files, err := SourceFiles(root) + + require.NoError(t, err) + assert.Contains(t, files, "linked/prompts.json") + assert.Contains(t, files, "lib/prompts.json", "an edit to the file behind the link changes the plan") +} + +func TestRestoreDescriptorAcceptsAMissingNewDescriptor(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + + err := NewStore(root).RestoreDescriptor(id, nil, nil) + + require.NoError(t, err, "a link that wrote nothing has nothing to remove") +} + +func TestReferencePathRejectsAnUnsafeVariationKey(t *testing.T) { + _, err := ReferencePath(syncdomain.VariationID("project", "..", "default")) + require.ErrorContains(t, err, "invalid config key") + _, err = ReferencePath(syncdomain.VariationID("project", "support", "..")) + require.ErrorContains(t, err, "invalid variation key") +} + +func TestDescriptorRelationshipsOnOneLineAreRenderedAgain(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + writeTestFile(t, root, descriptorPath, + "{formatVersion: 1, file: prompt.md, format: markdown, tools: [{key: old}], skills: [{key: triage}]}\n") + oldTools := []syncdomain.AttachmentRef{{Key: "old"}} + oldSkills := []syncdomain.AttachmentRef{{Key: "triage"}} + tools, skills := []string{"new"}, []string{"summarize"} + direct := syncdomain.FieldState{Capability: syncdomain.CapabilityDirect, Known: true} + + staged, changed, err := NewStore(root).prepareDescriptorRelationshipReplacement(SourceReplacement{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &oldTools, Skills: &oldSkills}, + Projection: syncdomain.VariationProjection{ + Tools: &tools, Skills: &skills, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{syncdomain.FieldTools: direct, syncdomain.FieldSkills: direct}, + }, + }) + + require.NoError(t, err) + require.True(t, changed) + descriptor, err := ReadDescriptor(fstest.MapFS{"d.ref.yaml": {Data: staged.replacement}}, "d.ref.yaml", id) + require.NoError(t, err) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "new"}}, *descriptor.Tools) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "summarize"}}, *descriptor.Skills) + assert.Equal(t, "prompt.md", descriptor.File) +} + +func TestDescriptorRelationshipEditThatDoesNotReadBackIsRenderedAgain(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + // The skills list is an alias of the tools list, so an in-place edit of + // the tools lines loses the anchor and the file does not decode. + writeTestFile(t, root, descriptorPath, + "formatVersion: 1\nfile: prompt.md\nformat: markdown\ntools: &shared\n - key: old\nskills: *shared\n") + oldRefs := []syncdomain.AttachmentRef{{Key: "old"}} + tools := []string{"new"} + direct := syncdomain.FieldState{Capability: syncdomain.CapabilityDirect, Known: true} + + staged, changed, err := NewStore(root).prepareDescriptorRelationshipReplacement(SourceReplacement{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &oldRefs, Skills: &oldRefs}, + Projection: syncdomain.VariationProjection{ + Tools: &tools, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{syncdomain.FieldTools: direct}, + }, + }) + + require.NoError(t, err) + require.True(t, changed) + descriptor, err := ReadDescriptor(fstest.MapFS{"d.ref.yaml": {Data: staged.replacement}}, "d.ref.yaml", id) + require.NoError(t, err, "the staged descriptor reads back") + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "new"}}, *descriptor.Tools) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "old"}}, *descriptor.Skills, "the skills keep the value that the alias had") +} + +func TestReplaceSourcesAcceptsRelationshipsInServerOrder(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + writeDescriptorForTest(t, root, id, Reference{File: "prompt.md", Format: "markdown"}) + next := []string{"zeta", "alpha"} + attachments := []syncdomain.Attachment{ + {Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{Key: "zeta", Schema: map[string]any{"type": "object"}}}, + {Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{Key: "alpha", Schema: map[string]any{"type": "object"}}}, + } + + paths, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown"}, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, + Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + RelationshipAttachments: attachments, + }}) + + require.NoError(t, err, "the order of the tools in LaunchDarkly does not fail the publish") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + assert.Contains(t, paths, descriptorPath) + descriptor, err := ReadDescriptor(os.DirFS(root), descriptorPath, id) + require.NoError(t, err) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "alpha"}, {Key: "zeta"}}, *descriptor.Tools) +} + +func TestMissingRelationshipAttachmentDoesNotTreatASymlinkAsADefinition(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + writeDescriptorForTest(t, root, id, Reference{File: "prompt.md", Format: "markdown"}) + writeTestFile(t, root, "elsewhere.json", `{"formatVersion": 1, "key": "new", "schema": {"type": "object"}}`) + toolPath, err := ResourcePath(syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "new"}) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(root, toolPath)), 0o755)) + require.NoError(t, os.Symlink(filepath.Join(root, "elsewhere.json"), filepath.Join(root, toolPath))) + next := []string{"new"} + + _, err = NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown"}, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + RelationshipAttachments: []syncdomain.Attachment{{ + Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{Key: "new", Schema: map[string]any{"type": "object"}}, + }}, + }}) + + require.ErrorContains(t, err, "symbolic link", "a link at the managed path is not a definition") +} + +func TestDescriptorRelationshipEditChecksTheMappingOfTheDescriptorItReads(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + writeTestFile(t, root, descriptorPath, + "formatVersion: 1\nfile: other.md\nformat: markdown\ntools:\n - key: old\n") + old := []syncdomain.AttachmentRef{{Key: "old"}} + next := []string{"new"} + + _, _, err = NewStore(root).prepareDescriptorRelationshipReplacement(SourceReplacement{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &old}, + Projection: syncdomain.VariationProjection{ + Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + }) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code, "a descriptor that names another file is not edited") +} + +func TestCheckSourcesAcceptsRelationshipsBeforeTheDescriptorExists(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + next := []string{"new"} + + err := NewStore(root).CheckSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown"}, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + RelationshipAttachments: []syncdomain.Attachment{{ + Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{Key: "new", Schema: map[string]any{"type": "object"}}, + }}, + }}) + + require.NoError(t, err, "a link checks its plan before it writes the descriptor") + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "tools", "new.json"), "a check writes nothing") +} + +func TestSourceFilesLeaveOutALinkTargetInTheManagedDirectory(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, ".launchdarkly/sync.lock", "formatVersion: 1\n") + require.NoError(t, os.Symlink(filepath.Join(root, ".launchdarkly", "sync.lock"), filepath.Join(root, "linked.md"))) + id := syncdomain.VariationID("project", "support", "default") + writeDescriptorForTest(t, root, id, Reference{File: "linked.md", Format: "markdown"}) + + files, err := SourceFiles(root) + + require.NoError(t, err) + assert.Contains(t, files, "linked.md") + assert.NotContains(t, files, ".launchdarkly/sync.lock", "a write of the lock must not start a sync again") +} + +func TestReplaceSourcesRejectsTwoBodiesForOneNewAttachment(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "first.md", "Help.\n") + writeTestFile(t, root, "second.md", "Assist.\n") + first := syncdomain.VariationID("project", "support", "first") + second := syncdomain.VariationID("project", "support", "second") + writeDescriptorForTest(t, root, first, Reference{File: "first.md", Format: "markdown"}) + writeDescriptorForTest(t, root, second, Reference{File: "second.md", Format: "markdown"}) + next := []string{"new"} + replacement := func(id syncdomain.ResourceID, file, description string) SourceReplacement { + description = strings.Clone(description) + return SourceReplacement{ + ID: id, Reference: Reference{File: file, Format: "markdown"}, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + RelationshipAttachments: []syncdomain.Attachment{{ + Kind: syncdomain.AttachmentTool, + Tool: &syncdomain.Tool{Key: "new", Description: &description, Schema: map[string]any{"type": "object"}}, + }}, + } + } + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{ + replacement(first, "first.md", "One body"), replacement(second, "second.md", "Another body"), + }) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorDefinitionConflict, problem.Code, "one publish writes one definition") + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "tools", "new.json")) +} + +func TestFlowDescriptorKeepsItsMappingWhenOneListChanges(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + descriptorPath, err := ReferencePath(id) + require.NoError(t, err) + writeTestFile(t, root, descriptorPath, "{formatVersion: 1, file: prompt.md, format: markdown, tools: [{key: old}]}\n") + oldTools := []syncdomain.AttachmentRef{{Key: "old"}} + tools := []string{"new"} + + staged, changed, err := NewStore(root).prepareDescriptorRelationshipReplacement(SourceReplacement{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown", Tools: &oldTools}, + Projection: syncdomain.VariationProjection{ + Tools: &tools, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + }) + + require.NoError(t, err) + require.True(t, changed) + descriptor, err := ReadDescriptor(fstest.MapFS{"d.ref.yaml": {Data: staged.replacement}}, "d.ref.yaml", id) + require.NoError(t, err, "the descriptor is rendered again instead of edited in place: %s", staged.replacement) + assert.Equal(t, "prompt.md", descriptor.File, "the mapping survives a change to one list") + assert.Equal(t, "markdown", descriptor.Format) + assert.Equal(t, []syncdomain.AttachmentRef{{Key: "new"}}, *descriptor.Tools) +} + +func TestMissingRelationshipAttachmentKeepsADescriptorBesideALeftoverLink(t *testing.T) { + root := t.TempDir() + tool := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search"} + writeTestFile(t, root, "tool.json", `{"description":"Linked","schema":{"type":"object"}}`) + writeDescriptorForTest(t, root, tool, Reference{ + File: "tool.json", Format: "json", Select: "$", Map: map[string]string{"description": "description", "schema": "schema"}, + }) + writeTestFile(t, root, "external.json", `{"formatVersion": 1, "key": "search", "schema": {}}`) + managedPath, err := ResourcePath(tool) + require.NoError(t, err) + target := filepath.Join(root, filepath.FromSlash(managedPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.Symlink(filepath.Join(root, "external.json"), target)) + id := syncdomain.VariationID("project", "support", "default") + tools := []string{"search"} + direct := syncdomain.FieldState{Capability: syncdomain.CapabilityDirect, Known: true} + + staged, err := NewStore(root).prepareMissingRelationshipAttachments([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown"}, + Projection: syncdomain.VariationProjection{ + Tools: &tools, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{syncdomain.FieldTools: direct}, + }, + }}) + + require.NoError(t, err, "the descriptor defines the tool, and the leftover link does not stop the pull") + assert.Empty(t, staged) +} + +func TestMissingRelationshipAttachmentRejectsASymlinkAtTheDescriptorPath(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + id := syncdomain.VariationID("project", "support", "default") + writeDescriptorForTest(t, root, id, Reference{File: "prompt.md", Format: "markdown"}) + writeTestFile(t, root, "elsewhere.ref.yaml", "formatVersion: 1\nfile: tool.json\nformat: json\n") + toolDescriptor, err := ReferencePath(syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "new"}) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(root, toolDescriptor)), 0o755)) + require.NoError(t, os.Symlink(filepath.Join(root, "elsewhere.ref.yaml"), filepath.Join(root, toolDescriptor))) + next := []string{"new"} + + _, err = NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Reference: Reference{File: "prompt.md", Format: "markdown"}, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Tools: &next, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldTools: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + RelationshipAttachments: []syncdomain.Attachment{{ + Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{Key: "new", Schema: map[string]any{"type": "object"}}, + }}, + }}) + + require.ErrorContains(t, err, "symbolic link", "a link at the descriptor path stops the pull") + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "tools", "new.json")) +} + +func TestCompileSourceRejectsALinkIntoAVendoredTree(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "vendor/lib/prompt.md", "Vendored.\n") + require.NoError(t, os.Symlink(filepath.Join(root, "vendor", "lib", "prompt.md"), filepath.Join(root, "prompt.md"))) + id := syncdomain.VariationID("project", "support", "default") + + _, err := CompileSource(root, id, Reference{File: "prompt.md", Format: "markdown"}) + + require.ErrorContains(t, err, "generated, vendored, or minified", "the resolved path counts at compile time too") +} + +func TestReplaceFilesChecksEveryNameOfASharedFile(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "lib/prompts.md", "old") + writeTestFile(t, root, "other/prompts.md", "other") + // Two descriptors named one file, once directly and once through a + // link. The link now names another file. + require.NoError(t, os.Symlink(filepath.Join(root, "other"), filepath.Join(root, "linked"))) + path, err := resolveSourcePath(root, "lib/prompts.md") + require.NoError(t, err) + + err = replaceFiles([]stagedFile{{ + displayPath: "lib/prompts.md", path: path, + original: []byte("old"), originalExists: true, replacement: []byte("edited"), mode: 0o644, + sources: []string{"lib/prompts.md", "linked/prompts.md"}, root: root, + }}) + + require.ErrorContains(t, err, "source linked/prompts.md changed while syncing") + data, readErr := os.ReadFile(filepath.Join(root, "lib", "prompts.md")) + require.NoError(t, readErr) + assert.Equal(t, "old", string(data), "the file that the second name no longer names is not written") +} + +func TestPrepareSourceReplacementsKeepsEveryNameOfASharedFile(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "lib/prompts.py", "A = \"Original A\"\nB = \"Original B\"\n") + require.NoError(t, os.Symlink(filepath.Join(root, "lib"), filepath.Join(root, "linked"))) + idA := syncdomain.VariationID("project", "support", "a") + idB := syncdomain.VariationID("project", "support", "b") + referenceA := Reference{File: "lib/prompts.py", Format: "python", Select: `constants[id="A"]`, Map: map[string]string{"prompt": "$"}} + referenceB := Reference{File: "linked/prompts.py", Format: "python", Select: `constants[id="B"]`, Map: map[string]string{"prompt": "$"}} + writeDescriptorForTest(t, root, idA, referenceA) + writeDescriptorForTest(t, root, idB, referenceB) + + staged, err := NewStore(root).prepareSourceReplacements([]SourceReplacement{ + {ID: idA, Reference: referenceA, + Projection: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("Updated A")}}, + {ID: idB, Reference: referenceB, + Projection: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("Updated B")}}, + }) + + require.NoError(t, err) + require.Len(t, staged, 1, "two names of one file stage one write") + assert.ElementsMatch(t, []string{"lib/prompts.py", "linked/prompts.py"}, staged[0].sources, "the commit checks both names") + assert.Equal(t, "A = \"Updated A\"\nB = \"Updated B\"\n", string(staged[0].replacement)) +} + +func TestAddReplacesALeftoverVariationLink(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "external.md", "Other.\n") + id := syncdomain.VariationID("project", "config", "default") + managedPath, err := ResourcePath(id) + require.NoError(t, err) + target := filepath.Join(root, filepath.FromSlash(managedPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.Symlink(filepath.Join(root, "external.md"), target)) + variation := syncdomain.Variation{Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default", Instructions: "Help."} + + creation, err := NewStore(root).Add([]VariationFile{{ProjectKey: "project", ConfigKey: "config", Variation: variation}}) + + require.NoError(t, err, "a leftover link is not a definition, so the add replaces it") + assert.Equal(t, []string{"project/configs/config/default.prompt.md"}, creation.CreatedPaths) + info, err := os.Lstat(target) + require.NoError(t, err) + assert.True(t, info.Mode().IsRegular(), "the managed path holds the file now") + external, err := os.ReadFile(filepath.Join(root, "external.md")) + require.NoError(t, err) + assert.Equal(t, "Other.\n", string(external), "the target of the link stays") +} + +func TestSourceSnapshotResolvesANameOnce(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "lib/old.md", "old") + writeTestFile(t, root, "lib/new.md", "new") + require.NoError(t, os.Symlink(filepath.Join(root, "lib", "old.md"), filepath.Join(root, "prompt.md"))) + snapshot := newSourceSnapshot(root) + data, err := snapshot.read("prompt.md") + require.NoError(t, err) + require.Equal(t, "old", string(data)) + first, err := snapshot.key("prompt.md") + require.NoError(t, err) + + // The link now names another file. + require.NoError(t, os.Remove(filepath.Join(root, "prompt.md"))) + require.NoError(t, os.Symlink(filepath.Join(root, "lib", "new.md"), filepath.Join(root, "prompt.md"))) + + second, err := snapshot.key("prompt.md") + require.NoError(t, err) + assert.Equal(t, first, second, "the snapshot keeps the file that it read, so the edits stay on it until the commit checks the name") + data, err = snapshot.read("prompt.md") + require.NoError(t, err) + assert.Equal(t, "old", string(data)) +} diff --git a/internal/sync/local/sources.go b/internal/sync/local/sources.go index 4bf3716e6..2dc5ca07b 100644 --- a/internal/sync/local/sources.go +++ b/internal/sync/local/sources.go @@ -6,14 +6,15 @@ import ( "os" "path/filepath" "slices" + "strings" syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncadapter "github.com/launchdarkly/ldcli/internal/sync/adapter" ) // SourceFiles returns every file that can change the sync plan, relative to -// the repository root. The list has each file in a project directory and -// each managed file. A file that is not valid stays in the list, so that a -// fix to the file starts a new sync in watch mode. +// the repository root. The list has each managed or referenced file. A file +// that is not valid stays in the list, so a fix starts a new sync in watch mode. func SourceFiles(repositoryRoot string) ([]string, error) { managedRoot := filepath.Join(repositoryRoot, syncdomain.RootDir) var files []string @@ -35,6 +36,21 @@ func SourceFiles(repositoryRoot string) ([]string, error) { if filepath.Dir(filePath) != managedRoot { files = append(files, filepath.ToSlash(relative)) } + relative = filepath.ToSlash(relative) + if id, ok := ParseReferencePath(relative); ok { + if reference, err := ReadDescriptor(os.DirFS(repositoryRoot), relative, id); err == nil { + files = append(files, sourcePaths(repositoryRoot, reference.File)...) + if content, err := ReadSourceFile(repositoryRoot, reference.File); err == nil { + dependencies, _ := syncadapter.DependencyPaths( + reference.Format, + syncadapter.Document{Path: reference.File, Data: content}, + ) + for _, dependency := range dependencies { + files = append(files, sourcePaths(repositoryRoot, dependency)...) + } + } + } + } return nil }) if errors.Is(err, os.ErrNotExist) { @@ -47,3 +63,32 @@ func SourceFiles(repositoryRoot string) ([]string, error) { slices.Sort(files) return slices.Compact(files), nil } + +// sourcePaths returns a repository-relative source path with the path of the +// file that it resolves to, when a symbolic link makes the two differ. A +// compile and a publish read the resolved file, so an edit to it changes +// the plan. +func sourcePaths(repositoryRoot, file string) []string { + paths := []string{file} + resolved, err := filepath.EvalSymlinks(filepath.Join(repositoryRoot, filepath.FromSlash(file))) + if err != nil { + return paths + } + root, err := filepath.EvalSymlinks(repositoryRoot) + if err != nil { + return paths + } + relative, err := filepath.Rel(root, resolved) + if err != nil || relative == "." || strings.HasPrefix(relative, "..") { + return paths + } + // A link into the managed directory is not a source, so its target does + // not go on the list: a write of the lock would start a sync again. + if inManagedDirectory(relative) { + return paths + } + if relative = filepath.ToSlash(relative); relative != file { + paths = append(paths, relative) + } + return paths +} diff --git a/internal/sync/projection.go b/internal/sync/projection.go index b6185d725..656e177f4 100644 --- a/internal/sync/projection.go +++ b/internal/sync/projection.go @@ -606,7 +606,17 @@ func ChangedProjection(local, server Projection) (Projection, error) { // ProjectionIncludes reports whether actual has every known value of expected. func ProjectionIncludes(actual, expected Projection) bool { - for _, field := range sortedFields(expected.States()) { + fields := sortedFields(expected.States()) + if len(fields) == 0 { + // A projection without field states, such as a desired value built + // by a caller, is compared on every field that it has a value for. + for _, field := range comparableFields { + if _, ok := expected.Value(field); ok { + fields = append(fields, field) + } + } + } + for _, field := range fields { want, wantOK := expected.Value(field) have, haveOK := actual.Value(field) if wantOK != haveOK || !sameJSON(want, have) { @@ -616,6 +626,13 @@ func ProjectionIncludes(actual, expected Projection) bool { return true } +// comparableFields are the fields of every kind that a projection can hold +// a value for, without the per-parameter model fields. +var comparableFields = []Field{ + FieldPrompt, FieldName, FieldModel, FieldModelParameters, FieldTools, FieldSkills, + FieldDescription, FieldSchema, FieldCustomParameters, FieldTags, FieldMarkdown, +} + var directState = FieldState{Capability: CapabilityDirect, Known: true} func sortedFields(states map[Field]FieldState) []Field { diff --git a/internal/sync/projection_includes_test.go b/internal/sync/projection_includes_test.go new file mode 100644 index 000000000..9f05091c4 --- /dev/null +++ b/internal/sync/projection_includes_test.go @@ -0,0 +1,17 @@ +package sync + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProjectionIncludesComparesValuesWithoutFieldStates(t *testing.T) { + want, have := "wanted", "other" + expected := VariationProjection{Instructions: &want} + + assert.False(t, ProjectionIncludes(VariationProjection{Instructions: &have}, expected), + "a projection without field states still compares the fields that have a value") + assert.True(t, ProjectionIncludes(VariationProjection{Instructions: &want}, expected)) + assert.True(t, ProjectionIncludes(VariationProjection{}, VariationProjection{})) +}