From 7abc71a68b46728aa1f94a944043df0c8fbf80be Mon Sep 17 00:00:00 2001 From: Clifford Tawiah Date: Wed, 4 Nov 2026 19:33:22 -0500 Subject: [PATCH] feat(sync): add the adapter core and the Markdown adapter Add the adapter contract: a format turns source bytes into a tree of logical nodes, and the core resolves select and map paths, builds projections, and plans exact edits. Add the YAML tree, the Markdown adapter with front matter, sections, and markers, and the Agent Skill adapter. Co-Authored-By: Claude Fable 5.1 --- internal/sync/adapter/adapter.go | 171 ++++++ internal/sync/adapter/adapter_test.go | 496 ++++++++++++++++++ internal/sync/adapter/markdown/markdown.go | 363 +++++++++++++ .../sync/adapter/markdown/markdown_test.go | 128 +++++ .../agent-skill/refunds/references/policy.md | 3 + .../refunds/scripts/check_refund.py | 2 + .../refunds/skill-instructions.md.golden | 12 + .../testdata/agents-instructions.md.golden | 9 + .../profiles/agents-instructions.md.golden | 13 + .../profiles/claude-instructions.md.golden | 10 + .../testdata/profiles/copilot.instructions.md | 7 + .../testdata/profiles/cursor-rule.mdc | 9 + internal/sync/adapter/node.go | 207 ++++++++ internal/sync/adapter/parse.go | 324 ++++++++++++ internal/sync/adapter/path.go | 270 ++++++++++ internal/sync/adapter/reference.go | 141 +++++ internal/sync/adapter/resolve.go | 232 ++++++++ internal/sync/adapter/update.go | 167 ++++++ internal/sync/adapter/yamltree/yamltree.go | 331 ++++++++++++ .../sync/adapter/yamltree/yamltree_test.go | 140 +++++ 20 files changed, 3035 insertions(+) create mode 100644 internal/sync/adapter/adapter.go create mode 100644 internal/sync/adapter/adapter_test.go create mode 100644 internal/sync/adapter/markdown/markdown.go create mode 100644 internal/sync/adapter/markdown/markdown_test.go create mode 100644 internal/sync/adapter/markdown/testdata/agent-skill/refunds/references/policy.md create mode 100644 internal/sync/adapter/markdown/testdata/agent-skill/refunds/scripts/check_refund.py create mode 100644 internal/sync/adapter/markdown/testdata/agent-skill/refunds/skill-instructions.md.golden create mode 100644 internal/sync/adapter/markdown/testdata/agents-instructions.md.golden create mode 100644 internal/sync/adapter/markdown/testdata/profiles/agents-instructions.md.golden create mode 100644 internal/sync/adapter/markdown/testdata/profiles/claude-instructions.md.golden create mode 100644 internal/sync/adapter/markdown/testdata/profiles/copilot.instructions.md create mode 100644 internal/sync/adapter/markdown/testdata/profiles/cursor-rule.mdc create mode 100644 internal/sync/adapter/node.go create mode 100644 internal/sync/adapter/parse.go create mode 100644 internal/sync/adapter/path.go create mode 100644 internal/sync/adapter/reference.go create mode 100644 internal/sync/adapter/resolve.go create mode 100644 internal/sync/adapter/update.go create mode 100644 internal/sync/adapter/yamltree/yamltree.go create mode 100644 internal/sync/adapter/yamltree/yamltree_test.go diff --git a/internal/sync/adapter/adapter.go b/internal/sync/adapter/adapter.go new file mode 100644 index 000000000..d41b749df --- /dev/null +++ b/internal/sync/adapter/adapter.go @@ -0,0 +1,171 @@ +// Package adapter is the contract between the sync engine and one source +// format. An adapter turns the bytes of a file into a tree of logical nodes. +// This package resolves select and map paths on that tree, builds typed +// projections, and plans exact edits. Adapters never read files, run code, or +// use the network, so they cannot get update planning, no-op detection, or +// capability checks wrong. +package adapter + +import ( + "fmt" + "sync" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Adapter inspects one source format. +// +// Format is the name that a descriptor and the --format flag use. Behavior +// names the version of the mapping rules of the adapter, such as python/1. +// It changes when a change to the adapter can change a mapped value, so that +// every binding of the format gets a review. Inspect returns the logical tree +// of a source bundle. Defaults returns the select path and the field map that +// a descriptor of a kind can leave out, or an empty select path when the +// format needs an explicit one. +type Adapter interface { + Format() string + Behavior() string + Inspect(SourceBundle) (*Node, error) + Defaults(syncdomain.Kind) (selectPath string, fieldMap map[string]string) +} + +// DependencyReporter is an optional adapter interface. It lists the +// repository files, relative to the repository root, whose bytes can change +// the projection of a primary source. The host reads them. +type DependencyReporter interface { + Dependencies(Document) ([]string, error) +} + +// Document is one repository file supplied to an adapter. +type Document struct { + Path string + Data []byte +} + +// SourceBundle is a primary source and the dependencies that its format +// reported, by repository-relative path. +type SourceBundle struct { + Primary Document + Dependencies map[string]Document +} + +// Capability is the shared mapped-field capability. +type Capability = syncdomain.Capability + +const ( + CapabilityDirect = syncdomain.CapabilityDirect + CapabilityReadOnly = syncdomain.CapabilityReadOnly +) + +// TextEdit replaces one exact byte range of one file. +type TextEdit struct { + File string + Start int + End int + Expected []byte + Replacement []byte +} + +// FieldUpdate is the capability of one mapped field and the edits that give +// it the desired value. +type FieldUpdate struct { + Capability Capability + Edits []TextEdit + Reason string +} + +// UpdateResult has one entry for each mapped field whose value changes. +type UpdateResult struct { + Fields map[syncdomain.Field]FieldUpdate +} + +type registry struct { + mu sync.RWMutex + adapters map[string]Adapter +} + +var formats = registry{adapters: make(map[string]Adapter)} + +// Register adds one source format. A format name registers once. +func Register(registered Adapter) error { + if registered == nil || registered.Format() == "" { + return fmt.Errorf("adapter format is required") + } + formats.mu.Lock() + defer formats.mu.Unlock() + if _, exists := formats.adapters[registered.Format()]; exists { + return fmt.Errorf("source format %q is already registered", registered.Format()) + } + formats.adapters[registered.Format()] = registered + return nil +} + +// MustRegister registers a format from an init function. +func MustRegister(registered Adapter) { + if err := Register(registered); err != nil { + panic(err) + } +} + +func lookup(format string) (Adapter, error) { + formats.mu.RLock() + registered := formats.adapters[format] + formats.mu.RUnlock() + if registered == nil { + return nil, syncdomain.NewProblem( + syncdomain.ErrorUnsupportedFormat, + fmt.Sprintf("source format %q is not supported", format), + "Use a registered format name.", + ) + } + return registered, nil +} + +// Behavior returns the behavior version of a registered format. +func Behavior(format string) (string, error) { + registered, err := lookup(format) + if err != nil { + return "", err + } + return registered.Behavior(), nil +} + +// DependencyPaths returns the repository files that a format reports for a +// primary source. +func DependencyPaths(format string, primary Document) ([]string, error) { + registered, err := lookup(format) + if err != nil { + return nil, err + } + reporter, ok := registered.(DependencyReporter) + if !ok { + return nil, nil + } + return reporter.Dependencies(primary) +} + +// Parse returns the logical tree of a bundle in the named format. Every node +// without a file gets the path of the primary document. +func Parse(bundle SourceBundle, format string) (*Node, error) { + registered, err := lookup(format) + if err != nil { + return nil, err + } + root, err := registered.Inspect(bundle) + if err != nil { + if _, ok := syncdomain.ProblemFrom(err); !ok { + err = syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + err.Error(), + "Fix the source syntax or use a supported static form.", + ).WithFile(bundle.Primary.Path, 0, 0), err) + } + } + root.SetFile(bundle.Primary.Path) + return root, err +} + +// ParseDocument returns the logical tree of one document without dependencies. +func ParseDocument(document Document, format string) (*Node, error) { + return Parse(SourceBundle{Primary: document}, format) +} diff --git a/internal/sync/adapter/adapter_test.go b/internal/sync/adapter/adapter_test.go new file mode 100644 index 000000000..7274b3132 --- /dev/null +++ b/internal/sync/adapter/adapter_test.go @@ -0,0 +1,496 @@ +package adapter + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestPathParserRejectsEveryUnsupportedShape(t *testing.T) { + paths := []string{ + `[id="one"]`, + `$["key"]`, + `items[id`, + `items[id!`, + `items[id="one"][name="two"]`, + `items[id="one",name=two]`, + `items[id="one",name]`, + `items[id="one",="two"]`, + `items[id="one",name="two"`, + `items[id="\uD800"]`, + `items[id="\uDC00"]`, + `items[id="\uD800\u0041"]`, + `items[id="bad\q"]`, + `["unclosed]`, + `items[id="one"] trailing`, + } + for _, path := range paths { + t.Run(path, func(t *testing.T) { + err := ParsePath(path) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorInvalidPathSyntax, problem.Code) + }) + } + require.NoError(t, ParsePath(`items[id="\uD83D\uDE00"]`)) + assert.Error(t, pathSyntaxError("x", 10, "test")) +} + +func TestSourcePathFailuresUseStableCodes(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "scalar": {Value: "text"}, + "items": {Items: []*Node{ + {Fields: map[string]*Node{"id": {Value: 10}}}, + }}, + }} + tests := []struct { + name, path string + code syncdomain.ErrorCode + }{ + {name: "missing field", path: "missing", code: syncdomain.ErrorPathNotFound}, + {name: "field on scalar", path: "scalar.child", code: syncdomain.ErrorPathTypeMismatch}, + {name: "filter type", path: `items[id="10"]`, code: syncdomain.ErrorFilterTypeMismatch}, + {name: "no filter match", path: `items[name="missing"]`, code: syncdomain.ErrorPathNotFound}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := root.Resolve(test.path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + }) + } +} + +func TestBuildProjectionAcceptsAllMappedFieldTypes(t *testing.T) { + selected := &Node{Fields: map[string]*Node{ + "prompt": {Value: "Help"}, + "messages": {Value: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "Help"}}}, + "name": {Value: "assistant"}, + "model": {Value: "model"}, + "temperature": {Value: int64(1)}, + }} + fieldMap := map[string]string{ + string(syncdomain.FieldPrompt): "prompt", + string(syncdomain.FieldName): "name", + string(syncdomain.FieldModel): "model", + `model.parameters["temperature"]`: "temperature", + } + projection, err := buildProjection(syncdomain.KindVariation, selected, fieldMap) + require.NoError(t, err) + variation := projection.(syncdomain.VariationProjection) + assert.Equal(t, syncdomain.VariationModeAgent, variation.Mode) + assert.Equal(t, 1.0, (*variation.Parameters)["temperature"]) + + projection, err = buildProjection( + syncdomain.KindVariation, selected, + map[string]string{string(syncdomain.FieldMessages): "messages"}, + ) + require.NoError(t, err) + assert.NotNil(t, projection.(syncdomain.VariationProjection).Messages) + + _, err = buildProjection(syncdomain.KindVariation, selected, map[string]string{ + string(syncdomain.FieldPrompt): "prompt", string(syncdomain.FieldMessages): "messages", + }) + require.ErrorContains(t, err, "cannot both be set") + _, err = buildProjection(syncdomain.KindVariation, selected, map[string]string{string(syncdomain.FieldName): "name"}) + require.ErrorContains(t, err, "is required for a variation") + + selected = &Node{Fields: map[string]*Node{ + "description": {Value: "Search"}, + "schema": {Value: map[string]any{"type": "object"}}, + "parameters": {Value: map[string]any{"limit": 10}}, + "tags": {Value: []any{"search", "safe"}}, + }} + projection, err = buildProjection(syncdomain.KindTool, selected, map[string]string{ + string(syncdomain.FieldDescription): "description", + string(syncdomain.FieldSchema): "schema", + string(syncdomain.FieldCustomParameters): "parameters", + string(syncdomain.FieldTags): "tags", + }) + require.NoError(t, err) + assert.Equal(t, []string{"search", "safe"}, *projection.(syncdomain.ToolProjection).Tags) + + selected = &Node{Fields: map[string]*Node{ + "description": {Value: "Review"}, + "markdown": {Value: "# Review"}, + }} + projection, err = buildProjection(syncdomain.KindSkill, selected, map[string]string{ + string(syncdomain.FieldDescription): "description", + string(syncdomain.FieldMarkdown): "markdown", + }) + require.NoError(t, err) + assert.Equal(t, "# Review", *projection.(syncdomain.SkillProjection).Markdown) +} + +func TestBuildProjectionRejectsInvalidMappedFieldTypes(t *testing.T) { + tests := []struct { + name string + kind syncdomain.Kind + mode syncdomain.VariationMode + field syncdomain.Field + value any + }{ + {name: "agent prompt", kind: syncdomain.KindVariation, field: syncdomain.FieldPrompt, value: true}, + {name: "messages", kind: syncdomain.KindVariation, mode: syncdomain.VariationModeCompletion, field: syncdomain.FieldMessages, value: true}, + {name: "message role", kind: syncdomain.KindVariation, mode: syncdomain.VariationModeCompletion, field: syncdomain.FieldMessages, value: []syncdomain.Message{{Role: "invalid"}}}, + {name: "name", kind: syncdomain.KindVariation, field: syncdomain.FieldName, value: true}, + {name: "model", kind: syncdomain.KindVariation, field: syncdomain.FieldModel, value: true}, + {name: "parameters", kind: syncdomain.KindVariation, field: syncdomain.FieldModelParameters, value: "hot"}, + {name: "description", kind: syncdomain.KindTool, field: syncdomain.FieldDescription, value: true}, + {name: "schema", kind: syncdomain.KindTool, field: syncdomain.FieldSchema, value: true}, + {name: "parameters", kind: syncdomain.KindTool, field: syncdomain.FieldCustomParameters, value: true}, + {name: "tags", kind: syncdomain.KindTool, field: syncdomain.FieldTags, value: []any{"ok", 1}}, + {name: "skill", kind: syncdomain.KindSkill, field: syncdomain.FieldMarkdown, value: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + selected := &Node{Fields: map[string]*Node{ + string(test.field): {Value: test.value}, + }} + fieldMap := map[string]string{string(test.field): string(test.field)} + if test.kind == syncdomain.KindVariation && test.field != syncdomain.FieldPrompt && test.field != syncdomain.FieldMessages { + selected.Fields["prompt"] = &Node{Value: "Help"} + fieldMap[string(syncdomain.FieldPrompt)] = "prompt" + } + _, err := buildProjection(test.kind, selected, fieldMap) + require.Error(t, err) + }) + } + + _, err := buildProjection(syncdomain.KindVariation, &Node{}, map[string]string{}) + require.Error(t, err) + _, err = buildProjection(syncdomain.Kind("unknown"), &Node{}, nil) + require.Error(t, err) +} + +func TestProjectionFieldReadsEveryOptionalField(t *testing.T) { + text := "value" + list := []string{"one"} + messages := []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "Help"}} + object := map[string]any{"type": "object"} + parameters := map[string]any{"temperature": 0.5} + projections := []syncdomain.Projection{ + syncdomain.VariationProjection{ + Instructions: &text, Name: &text, Model: &text, Parameters: ¶meters, Tools: &list, Skills: &list, + }, + syncdomain.VariationProjection{Messages: &messages}, + syncdomain.ToolProjection{ + Description: &text, Schema: &object, CustomParameters: &object, Tags: &list, + }, + syncdomain.SkillProjection{Description: &text, Markdown: &text}, + } + fields := [][]syncdomain.Field{ + {syncdomain.FieldPrompt, syncdomain.FieldName, syncdomain.FieldModel, syncdomain.FieldModelParameters, syncdomain.FieldTools, syncdomain.FieldSkills}, + {syncdomain.FieldMessages}, + {syncdomain.FieldDescription, syncdomain.FieldSchema, syncdomain.FieldCustomParameters, syncdomain.FieldTags}, + {syncdomain.FieldDescription, syncdomain.FieldMarkdown}, + } + for index, projection := range projections { + for _, field := range fields[index] { + _, present := projection.Value(field) + assert.True(t, present) + } + } +} + +func TestMessageUpdateSafetyBranches(t *testing.T) { + have := []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "old"}} + tests := []struct { + name string + node *Node + want []syncdomain.Message + }{ + {name: "different count", node: &Node{}, want: nil}, + {name: "different role", node: &Node{}, want: []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "old"}}}, + {name: "missing content", node: &Node{}, want: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}}, + {name: "read only content", node: &Node{Items: []*Node{{Fields: map[string]*Node{"content": {Reason: "dynamic"}}}}}, want: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + update := updateMessageContents(SourceBundle{}, test.node, have, test.want) + assert.Equal(t, CapabilityReadOnly, update.Capability) + }) + } + + content := String("old", 0, 5, func(string) ([]byte, error) { return []byte(`"new"`), nil }) + content.File = "missing.py" + update := updateMessageContents( + SourceBundle{Primary: Document{Path: "main.py"}}, + &Node{Items: []*Node{{Fields: map[string]*Node{"content": content}}}}, + have, []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}, + ) + assert.Equal(t, CapabilityReadOnly, update.Capability) +} + +func TestSourceValueHelpersRejectDynamicValues(t *testing.T) { + assert.False(t, semanticEqual(make(chan int), make(chan int))) + _, ok := AsNumber(math.NaN()) + assert.True(t, ok) + _, ok = AsNumber("1") + assert.False(t, ok) + _, ok = stringList(true) + assert.False(t, ok) + _, ok = objectValue(true) + assert.False(t, ok) + + stringValue := String("old", 0, 3, func(value string) ([]byte, error) { return []byte(value), nil }) + _, err := stringValue.Encode(1) + require.Error(t, err) + numberValue := Number(1, 0, 1, func(float64) ([]byte, error) { return []byte("1"), nil }) + _, err = numberValue.Encode("one") + require.Error(t, err) +} + +func TestRegistryContractAndDependencyErrors(t *testing.T) { + require.Error(t, Register(nil)) + require.Error(t, Register(contractAdapter{})) + require.NoError(t, Register(namedFakeAdapter{format: "contract-test"})) + require.Error(t, Register(namedFakeAdapter{format: "contract-test"})) + + _, err := lookup("missing-format") + require.Error(t, err) + dependencies, err := DependencyPaths("contract-test", Document{}) + require.NoError(t, err) + assert.Nil(t, dependencies) + _, err = DependencyPaths("missing-format", Document{}) + require.Error(t, err) +} + +func TestParsePathGrammar(t *testing.T) { + for _, path := range []string{ + "$", + "field.child", + `items[id="value"]`, + `items[role="system",name="intro"]`, + "items[0]", + "items[0][1]", + "items[2147483647]", + `items[id="value"].messages[0].content`, + `object["literal.key"]`, + `["top level"]`, + } { + require.NoError(t, ParsePath(path), path) + } + for _, path := range []string{ + "", + "$.field", + "items[]", + "items[-1]", + "items[1.0]", + "items[01]", + "items[2147483648]", + `items[0][id="value"]`, + "items[*]", + `items[id=value]`, + `items[ id = "value" ]`, + "items..id", + } { + err := ParsePath(path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok, path) + assert.Equal(t, syncdomain.ErrorInvalidPathSyntax, problem.Code, path) + } +} + +func TestResolvePathSupportsChainedAndMixedIndexes(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "agents": {Items: []*Node{ + {Fields: map[string]*Node{ + "id": {Value: "support"}, + "messages": {Items: []*Node{ + {Fields: map[string]*Node{"content": {Value: "First"}}}, + {Fields: map[string]*Node{"content": {Value: "Second"}}}, + }, Collection: true}, + }}, + }, Collection: true}, + }} + + value, err := root.Resolve(`agents[id="support"].messages[1].content`) + + require.NoError(t, err) + assert.Equal(t, "Second", value.Value) +} + +func TestResolvePathIndexErrorsUseStableCodes(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "items": {Items: []*Node{{Value: "only"}}, Collection: true}, + "value": {Value: "text"}, + }} + tests := []struct { + path string + code syncdomain.ErrorCode + }{ + {path: "items[1]", code: syncdomain.ErrorIndexOutOfRange}, + {path: "value[0]", code: syncdomain.ErrorPathTypeMismatch}, + } + for _, test := range tests { + _, err := root.Resolve(test.path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + } +} + +func TestResolvePathSupportsLiteralKeysAndRejectsAmbiguousSelectors(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "metadata": {Fields: map[string]*Node{"display.name": {Value: "Support"}}}, + "agents": { + Items: []*Node{ + {Fields: map[string]*Node{"id": {Value: "support"}}}, + {Fields: map[string]*Node{"id": {Value: "support"}}}, + }, + }, + }} + value, err := root.Resolve(`metadata["display.name"]`) + require.NoError(t, err) + assert.Equal(t, "Support", value.Value) + + _, err = root.Resolve(`agents[id="support"]`) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorPathAmbiguous, problem.Code) +} + +func TestResolvePathMatchesEveryFilterCondition(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "items": {Items: []*Node{ + {Fields: map[string]*Node{ + "role": {Value: "system"}, "name": {Value: "intro"}, "content": {Value: "First"}, + }}, + {Fields: map[string]*Node{ + "role": {Value: "system"}, "name": {Value: "outro"}, "content": {Value: "Second"}, + }}, + }}, + }} + + value, err := root.Resolve(`items[role="system",name="intro"].content`) + + require.NoError(t, err) + assert.Equal(t, "First", value.Value) +} + +func TestRegisteredAdapterRunsThroughSharedCore(t *testing.T) { + require.NoError(t, Register(fakeAdapter{})) + id := syncdomain.VariationID("project", "config", "default") + + projection, err := ParseSource( + bundle("prompt.fake", []byte("Registered without a core change.")), + id, syncdomain.Reference{File: "prompt.fake", Format: "fake-test"}, + ) + + require.NoError(t, err) + assert.Equal(t, "Registered without a core change.", *projection.(syncdomain.VariationProjection).Instructions) +} + +func TestRegistryRejectsInvalidAndDuplicateAdapters(t *testing.T) { + require.Error(t, Register(nil)) + adapter := namedFakeAdapter{format: "duplicate-test"} + require.NoError(t, Register(adapter)) + require.Error(t, Register(adapter)) + + _, err := lookup("not-registered") + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorUnsupportedFormat, problem.Code) +} + +func TestIndexedSelectionIdentityUsesStableFieldsAndFallback(t *testing.T) { + identified := &Node{Fields: map[string]*Node{ + "id": {Value: "support"}, + "n": {Value: 1}, + }} + assert.Equal(t, map[string]any{"id": "support"}, directSelectionIdentity(identified)) + assert.Nil(t, directSelectionIdentity(nil)) + assert.True(t, scalarValue(uint64(2))) + assert.False(t, scalarValue(nil)) + assert.False(t, (*Node)(nil).IsCollection()) + assert.True(t, (&Node{Value: []string{"one"}}).IsCollection()) + assert.Equal(t, + []any{"one"}, + stableNodeValue(&Node{Items: []*Node{{Value: "one"}}, Collection: true}, map[*Node]bool{}), + ) + assert.Nil(t, stableNodeValue(&Node{}, map[*Node]bool{})) + + cycle := &Node{Fields: map[string]*Node{}} + cycle.Fields["self"] = cycle + fallback := &Node{Fields: map[string]*Node{"prompt": {Value: "Help"}}} + selections, err := indexedSelections( + []indexedResolution{ + {path: "items[0]", collectionLength: 1, item: identified}, + {path: "items[0].children[0]", collectionLength: 1, item: fallback}, + {path: "items[0].cycle[0]", collectionLength: 1, item: cycle}, + }, + fallback, + ) + require.NoError(t, err) + require.Len(t, selections, 3) + for _, selection := range selections { + assert.NotEmpty(t, selection.Fingerprint) + } + + _, err = indexedSelections( + []indexedResolution{{path: "items[0]", collectionLength: 1, item: &Node{Value: make(chan int)}}}, + fallback, + ) + require.ErrorContains(t, err, "encode indexed selection fingerprint") +} + +type fakeAdapter struct{} + +func (fakeAdapter) Format() string { return "fake-test" } +func (fakeAdapter) Behavior() string { return "fake-test/1" } +func (fakeAdapter) Inspect(bundle SourceBundle) (*Node, error) { + document := bundle.Primary + return String(string(document.Data), 0, len(document.Data), func(text string) ([]byte, error) { + return []byte(text), nil + }), nil +} +func (fakeAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + return "$", map[string]string{string(syncdomain.FieldPrompt): "$"} +} + +type namedFakeAdapter struct{ format string } + +func (adapter namedFakeAdapter) Format() string { return adapter.format } +func (namedFakeAdapter) Behavior() string { return "fake-test/1" } +func (namedFakeAdapter) Inspect(bundle SourceBundle) (*Node, error) { + return fakeAdapter{}.Inspect(bundle) +} +func (namedFakeAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + return fakeAdapter{}.Defaults(kind) +} + +type contractAdapter struct{} + +func (contractAdapter) Format() string { return "" } +func (contractAdapter) Behavior() string { return "" } +func (contractAdapter) Inspect(SourceBundle) (*Node, error) { return nil, nil } +func (contractAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func bundle(path string, data []byte) SourceBundle { + return SourceBundle{Primary: Document{Path: path, Data: data}} +} + +func TestCoreHelpersRejectUnknownKindsAndFailedEncoders(t *testing.T) { + assert.False(t, fieldAllowed(syncdomain.Kind("unknown"), syncdomain.FieldPrompt)) + _, err := ParseDocument(Document{}, "missing") + require.Error(t, err) + number, ok := AsNumber(1) + assert.True(t, ok) + assert.Equal(t, 1.0, number) + + content := String("old", 0, 3, func(string) ([]byte, error) { return nil, assert.AnError }) + update := updateMessageContents( + bundle("value.json", []byte("old")), + &Node{Items: []*Node{{Fields: map[string]*Node{"content": content}}}}, + []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "old"}}, + []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}, + ) + assert.Equal(t, CapabilityReadOnly, update.Capability) +} diff --git a/internal/sync/adapter/markdown/markdown.go b/internal/sync/adapter/markdown/markdown.go new file mode 100644 index 000000000..0ee5e91e0 --- /dev/null +++ b/internal/sync/adapter/markdown/markdown.go @@ -0,0 +1,363 @@ +// Package markdown inspects Markdown documents and Agent Skill files. +// +// A Markdown document exposes four logical fields. frontmatter is the YAML +// front matter, or an empty object. body is the text after the front matter. +// sections has one item for each heading, with the heading text and the body +// below it. markers has one item for each LaunchDarkly marker region. The +// same adapter serves plain prompt files, AGENTS.md, CLAUDE.md, Cursor rules, +// and Copilot instructions, because only the mapped value changes. +package markdown + +import ( + "bytes" + "errors" + "fmt" + "strings" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" + "github.com/launchdarkly/ldcli/internal/sync/adapter/yamltree" +) + +type markdownAdapter struct{} + +func (markdownAdapter) Format() string { return "markdown" } +func (markdownAdapter) Behavior() string { return "markdown/1" } + +func (markdownAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parse(bundle.Primary, false) +} + +func (markdownAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindVariation { + return "$", map[string]string{string(syncdomain.FieldPrompt): "body"} + } + return "$", nil +} + +type agentSkillAdapter struct{} + +func (agentSkillAdapter) Format() string { return "agent-skill" } +func (agentSkillAdapter) Behavior() string { return "agent-skill/1" } + +func (agentSkillAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parseAgentSkill(bundle.Primary) +} + +func (agentSkillAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindSkill { + return "$", map[string]string{ + string(syncdomain.FieldDescription): "frontmatter.description", + string(syncdomain.FieldMarkdown): "body", + } + } + return "$", nil +} + +func init() { + adapter.MustRegister(markdownAdapter{}) + adapter.MustRegister(agentSkillAdapter{}) +} + +// parse builds the logical document. With exactBody, the body keeps every +// byte, as a skill needs. Otherwise the body is normalized prompt text. +func parse(document adapter.Document, exactBody bool) (*adapter.Node, error) { + data := document.Data + if !utf8.Valid(data) { + return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path) + } + bodyStart := len(data) - len(bytes.TrimPrefix(data, []byte("\ufeff"))) + frontMatter := &adapter.Node{Value: map[string]any{}, Fields: map[string]*adapter.Node{}} + if hasFrontMatter(data[bodyStart:]) { + region, end, err := splitFrontMatter(data) + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + if frontMatter, err = parseFrontMatter(region); err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + bodyStart = end + } + body := data[bodyStart:] + bodyNode := textNode(body, bodyStart, len(data)) + if exactBody { + bodyNode = exactNode(body, bodyStart, len(data)) + } + markers, err := markerRegions(body, bodyStart) + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + return &adapter.Node{Fields: map[string]*adapter.Node{ + "frontmatter": frontMatter, + "body": bodyNode, + "sections": {Items: sections(body, bodyStart), Collection: true}, + "markers": markers, + }}, nil +} + +// parseAgentSkill builds a SKILL.md document and checks the fields that a +// skill needs: a string description in the front matter and a body. +func parseAgentSkill(document adapter.Document) (*adapter.Node, error) { + if !hasFrontMatter(bytes.TrimPrefix(document.Data, []byte("\ufeff"))) { + return nil, fmt.Errorf("%s: missing YAML front matter", document.Path) + } + root, err := parse(document, true) + if err != nil { + return nil, err + } + description := root.Fields["frontmatter"].Fields["description"] + switch { + case root.Fields["frontmatter"].Fields == nil: + return nil, fmt.Errorf("%s: front matter must be a mapping", document.Path) + case description == nil: + return nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + "front matter field \"description\" was not found", + "Add the field or update the map path.", + ).WithFile(document.Path, 0, 0) + case description.Value == nil: + return nil, fmt.Errorf("%s: front matter description must be a string", document.Path) + } + if _, ok := description.Value.(string); !ok { + return nil, fmt.Errorf("%s: front matter description must be a string", document.Path) + } + if strings.TrimSpace(root.Fields["body"].Value.(string)) == "" { + return nil, fmt.Errorf("%s: skill Markdown is required", document.Path) + } + return root, nil +} + +// hasFrontMatter reports whether data starts with a front matter fence. A +// fence is three dashes at the start of a line and nothing else. +func hasFrontMatter(data []byte) bool { + return isFence(string(data[:nextLineEnd(data, 0)])) +} + +func isFence(line string) bool { + return strings.TrimRight(line, "\r\n") == "---" +} + +// splitFrontMatter returns the YAML region between the front matter fences +// and the offset where the body starts. +func splitFrontMatter(data []byte) (yamltree.Region, int, error) { + offset := len(data) - len(bytes.TrimPrefix(data, []byte("\ufeff"))) + contentStart := nextLineEnd(data, offset) + for lineStart := contentStart; lineStart < len(data); { + lineEnd := nextLineEnd(data, lineStart) + if isFence(string(data[lineStart:lineEnd])) { + bodyStart := lineEnd + if bytes.HasPrefix(data[bodyStart:], []byte("\r\n")) { + bodyStart += 2 + } else if bytes.HasPrefix(data[bodyStart:], []byte("\n")) { + bodyStart++ + } + return yamltree.Region{Data: data[contentStart:lineStart], StartOffset: contentStart}, bodyStart, nil + } + lineStart = lineEnd + } + return yamltree.Region{}, 0, errors.New("unclosed YAML front matter") +} + +// parseFrontMatter builds the logical tree of the front matter. Empty front +// matter is an empty object. +func parseFrontMatter(region yamltree.Region) (*adapter.Node, error) { + var document yaml.Node + if err := yaml.Unmarshal(region.Data, &document); err != nil { + return nil, fmt.Errorf("invalid front matter: %w", err) + } + if len(document.Content) == 0 { + return &adapter.Node{Value: map[string]any{}, Fields: map[string]*adapter.Node{}}, nil + } + return yamltree.Build(region.Data, document.Content[0], region.StartOffset, false) +} + +func nextLineEnd(data []byte, start int) int { + if index := bytes.IndexByte(data[start:], '\n'); index >= 0 { + return start + index + 1 + } + return len(data) +} + +// sections returns one node for each heading. The body of a section runs to +// the next heading of the same or a higher level. Headings inside fenced code +// and indented lines do not count. base is the offset of data in the file. +func sections(data []byte, base int) []*adapter.Node { + type heading struct { + level int + text string + lineStart int + bodyStart int + } + var headings []heading + forEachLine(data, func(line markdownLine) { + if line.fenced || line.indented { + return + } + trimmed := strings.TrimLeft(line.text, " ") + level := 0 + for level < len(trimmed) && trimmed[level] == '#' { + level++ + } + if level > 0 && level <= 6 && level < len(trimmed) && trimmed[level] == ' ' { + headings = append(headings, heading{ + level: level, text: headingText(trimmed[level:]), lineStart: line.start, bodyStart: line.end, + }) + } + }) + + result := make([]*adapter.Node, 0, len(headings)) + for index, current := range headings { + bodyEnd := len(data) + for next := index + 1; next < len(headings); next++ { + if headings[next].level <= current.level { + bodyEnd = headings[next].lineStart + break + } + } + result = append(result, &adapter.Node{Fields: map[string]*adapter.Node{ + "heading": {Value: current.text}, + "body": textNode(data[current.bodyStart:bodyEnd], base+current.bodyStart, base+bodyEnd), + }}) + } + return result +} + +// markdownLine is one line of a document with its byte range, whether it is +// inside or part of a fenced code block, and whether it is indented code. +type markdownLine struct { + start, end int + text string + fenced bool + indented bool +} + +// forEachLine visits each line of data in order and tracks fenced code +// blocks, so that headings and markers inside code do not count. As in +// CommonMark, a closing fence has nothing but spaces after it, and the info +// string of a backtick fence has no backtick. +func forEachLine(data []byte, visit func(markdownLine)) { + var fenceCharacter byte + fenceLength := 0 + for offset := 0; offset < len(data); { + end := nextLineEnd(data, offset) + text := strings.TrimRight(string(data[offset:end]), "\r\n") + trimmed := strings.TrimLeft(text, " ") + line := markdownLine{start: offset, end: end, text: text, fenced: fenceLength != 0} + line.indented = len(text)-len(trimmed) > 3 || strings.HasPrefix(trimmed, "\t") + if !line.indented { + marker, markerLength := fence(trimmed) + rest := trimmed[markerLength:] + switch { + case fenceLength == 0 && markerLength >= 3 && (marker == '~' || !strings.Contains(rest, "`")): + fenceCharacter, fenceLength, line.fenced = marker, markerLength, true + case fenceLength != 0 && marker == fenceCharacter && markerLength >= fenceLength && strings.TrimSpace(rest) == "": + fenceCharacter, fenceLength = 0, 0 + } + } + visit(line) + offset = end + } +} + +func fence(line string) (byte, int) { + if line == "" || line[0] != '`' && line[0] != '~' { + return 0, 0 + } + length := 1 + for length < len(line) && line[length] == line[0] { + length++ + } + return line[0], length +} + +// headingText returns the heading without its closing hashes. +func headingText(text string) string { + text = strings.TrimSpace(text) + lastSpace := strings.LastIndexByte(text, ' ') + if lastSpace < 0 || lastSpace == len(text)-1 || strings.Trim(text[lastSpace+1:], "#") != "" { + return text + } + return strings.TrimSpace(text[:lastSpace]) +} + +// markerRegions returns one node for each region between a begin and an end +// marker. A marker inside fenced or indented code does not count. Nested, +// reversed, and unclosed markers are errors. +func markerRegions(data []byte, base int) (*adapter.Node, error) { + const beginPrefix = ""): + if openID != "" { + err = fmt.Errorf("nested LaunchDarkly marker %q", openID) + return + } + openID = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(text, beginPrefix), "-->")) + bodyStart = line.end + case strings.HasPrefix(text, endPrefix) && strings.HasSuffix(text, "-->"): + id := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(text, endPrefix), "-->")) + if openID == "" || id != openID { + err = fmt.Errorf("LaunchDarkly marker %q has no matching begin marker", id) + return + } + items = append(items, &adapter.Node{Fields: map[string]*adapter.Node{ + "id": {Value: id}, + "body": textNode(data[bodyStart:line.start], base+bodyStart, base+line.start), + }}) + openID = "" + } + }) + if err != nil { + return nil, err + } + if openID != "" { + return nil, fmt.Errorf("LaunchDarkly marker %q is not closed", openID) + } + return &adapter.Node{Items: items, Collection: true}, nil +} + +// textNode is prompt text: normalized line endings and no surrounding white +// space. An update keeps the white space around the text and the line ending +// style of the file. +func textNode(data []byte, start, end int) *adapter.Node { + lineEnding := "\n" + if bytes.Contains(data, []byte("\r\n")) { + lineEnding = "\r\n" + } + trimmed := bytes.TrimFunc(data, isSpace) + prefix := bytes.Clone(data[:bytes.Index(data, trimmed)]) + suffix := bytes.Clone(data[len(prefix)+len(trimmed):]) + return adapter.String(normalize(string(data)), start, end, func(text string) ([]byte, error) { + text = strings.ReplaceAll(normalize(text), "\n", lineEnding) + return bytes.Join([][]byte{prefix, []byte(text), suffix}, nil), nil + }) +} + +// exactNode is text that keeps every byte, as skill Markdown does. An update +// keeps the final newline of the file. +func exactNode(data []byte, start, end int) *adapter.Node { + finalNewline := bytes.HasSuffix(data, []byte("\n")) + return adapter.String(string(data), start, end, func(text string) ([]byte, error) { + if finalNewline && text != "" && !strings.HasSuffix(text, "\n") { + text += "\n" + } + return []byte(text), nil + }) +} + +func normalize(text string) string { + return syncdomain.NormalizePromptText(text) +} + +func isSpace(character rune) bool { + return character == ' ' || character == '\t' || character == '\r' || character == '\n' +} diff --git a/internal/sync/adapter/markdown/markdown_test.go b/internal/sync/adapter/markdown/markdown_test.go new file mode 100644 index 000000000..5af6b029b --- /dev/null +++ b/internal/sync/adapter/markdown/markdown_test.go @@ -0,0 +1,128 @@ +package markdown + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +func TestMarkdownRejectsUnsafeMarkersAndFrontMatter(t *testing.T) { + inputs := []string{ + "\n", + "", + "", + "---\ninvalid: [\n---\nbody", + "---\ndescription: value", + } + for _, input := range inputs { + _, err := parse(adapter.Document{Path: "AGENTS.md", Data: []byte(input)}, false) + require.Error(t, err, input) + } + + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: []byte("\ufeff# Instructions")}, false) + require.NoError(t, err) + assert.Equal(t, "# Instructions", root.Fields["body"].Value) + assert.Empty(t, root.Fields["frontmatter"].Fields) + assert.Empty(t, root.Fields["markers"].Items) +} + +func TestMarkdownAndAgentSkillSafetyForms(t *testing.T) { + selectPath, fieldMap := markdownAdapter{}.Defaults(syncdomain.KindTool) + assert.Equal(t, "$", selectPath) + assert.Nil(t, fieldMap) + selectPath, fieldMap = agentSkillAdapter{}.Defaults(syncdomain.KindVariation) + assert.Equal(t, "$", selectPath) + assert.Nil(t, fieldMap) + + _, err := parse(adapter.Document{Path: "README.md", Data: []byte{0xff}}, false) + require.Error(t, err) + _, err = parseAgentSkill(adapter.Document{Path: "SKILL.md", Data: []byte{0xff}}) + require.Error(t, err) + + for name, input := range map[string]string{ + "missing front matter": "# Body", + "unclosed front matter": "---\ndescription: value", + "invalid YAML": "---\ndescription: [\n---\nBody", + "front matter scalar": "---\nvalue\n---\nBody", + "missing description": "---\nname: skill\n---\nBody", + "description type": "---\ndescription: [value]\n---\nBody", + "null description": "---\ndescription: null\n---\nBody", + "tilde description": "---\ndescription: ~\n---\nBody", + "blank description": "---\ndescription:\n---\nBody", + "empty body": "---\ndescription: value\n---\n", + } { + t.Run(name, func(t *testing.T) { + _, err := parseAgentSkill(adapter.Document{Path: "SKILL.md", Data: []byte(input)}) + require.Error(t, err) + }) + } + + root, err := parseAgentSkill(adapter.Document{Path: "SKILL.md", Data: []byte("---\ndescription: |\n long text\n---\nBody\n")}) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields["frontmatter"].Fields["description"].Capability) + + headings := sections([]byte(" # indented\n````\n# hidden\n````\n## Visible ##\nbody\n"), 0) + require.Len(t, headings, 1) + assert.Equal(t, "Visible", headings[0].Fields["heading"].Value) + + exact := exactNode([]byte("body\n"), 0, 5) + encoded, err := exact.Encode("next") + require.NoError(t, err) + assert.Equal(t, "next\n", string(encoded)) + + region, bodyStart, err := splitFrontMatter([]byte("\ufeff---\r\ndescription: value\r\n---\r\n\r\nBody")) + require.NoError(t, err) + assert.NotEmpty(t, region.Data) + assert.Greater(t, bodyStart, 0) +} + +func TestFrontMatterFenceMustStartTheLine(t *testing.T) { + source := []byte("---\ndescription: |\n ---\n not a fence\nname: support\n---\nBody\n") + root, err := parse(adapter.Document{Path: "rule.md", Data: source}, false) + require.NoError(t, err) + assert.Equal(t, "---\nnot a fence\n", root.Fields["frontmatter"].Fields["description"].Value) + assert.Equal(t, "support", root.Fields["frontmatter"].Fields["name"].Value) + assert.Equal(t, "Body", root.Fields["body"].Value) +} + +func TestMarkersInsideFencedCodeDoNotCount(t *testing.T) { + source := []byte("# Guide\n\n```markdown\n\nshown as an example\n\n```\n\n\nReal region.\n\n") + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + markers := root.Fields["markers"].Items + require.Len(t, markers, 1) + assert.Equal(t, "real", markers[0].Fields["id"].Value) + assert.Equal(t, "Real region.", markers[0].Fields["body"].Value) +} + +func TestClosingFenceHasNothingAfterIt(t *testing.T) { + source := []byte("# Guide\n\n```\n# Code heading\n```js\n\nstill code\n\n``` \n\n## Real\n\n\nReal region.\n\n") + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + markers := root.Fields["markers"].Items + require.Len(t, markers, 1) + assert.Equal(t, "real", markers[0].Fields["id"].Value) + headings := make([]string, 0, 2) + for _, section := range root.Fields["sections"].Items { + headings = append(headings, section.Fields["heading"].Value.(string)) + } + assert.Equal(t, []string{"Guide", "Real"}, headings) + + source = []byte("``` a ` b\n# Heading\n") + root, err = parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + require.Len(t, root.Fields["sections"].Items, 1) +} + +func TestMarkersInsideIndentedCodeDoNotCount(t *testing.T) { + source := []byte("# Guide\n\nExample:\n\n \n shown as an example\n \n\n\nReal region.\n\n") + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + markers := root.Fields["markers"].Items + require.Len(t, markers, 1) + assert.Equal(t, "real", markers[0].Fields["id"].Value) +} diff --git a/internal/sync/adapter/markdown/testdata/agent-skill/refunds/references/policy.md b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/references/policy.md new file mode 100644 index 000000000..52b962517 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/references/policy.md @@ -0,0 +1,3 @@ +# Refund policy + +Escalate refunds above $500. diff --git a/internal/sync/adapter/markdown/testdata/agent-skill/refunds/scripts/check_refund.py b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/scripts/check_refund.py new file mode 100644 index 000000000..e28fe6cd2 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/scripts/check_refund.py @@ -0,0 +1,2 @@ +def eligible(total: float) -> bool: + return total <= 500 diff --git a/internal/sync/adapter/markdown/testdata/agent-skill/refunds/skill-instructions.md.golden b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/skill-instructions.md.golden new file mode 100644 index 000000000..95237b040 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/skill-instructions.md.golden @@ -0,0 +1,12 @@ +--- +name: refunds +description: Explain the refund policy and escalation rules. +license: Apache-2.0 +metadata: + owner: support-platform +--- + +# Refund handling + +Confirm the order ID first. +Escalate refunds above $500. diff --git a/internal/sync/adapter/markdown/testdata/agents-instructions.md.golden b/internal/sync/adapter/markdown/testdata/agents-instructions.md.golden new file mode 100644 index 000000000..16f2d0c47 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agents-instructions.md.golden @@ -0,0 +1,9 @@ +# Repository guide + +## Support agent + +Ask for an order ID before discussing an order. + +## Build + +Run `make test`. diff --git a/internal/sync/adapter/markdown/testdata/profiles/agents-instructions.md.golden b/internal/sync/adapter/markdown/testdata/profiles/agents-instructions.md.golden new file mode 100644 index 000000000..28d9f4674 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/agents-instructions.md.golden @@ -0,0 +1,13 @@ +# Repository guide + +## Support + +Ask for an order ID. + + +Explain the refund policy. + + +## Build + +Run `make test`. diff --git a/internal/sync/adapter/markdown/testdata/profiles/claude-instructions.md.golden b/internal/sync/adapter/markdown/testdata/profiles/claude-instructions.md.golden new file mode 100644 index 000000000..f4a10263c --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/claude-instructions.md.golden @@ -0,0 +1,10 @@ +--- +name: support +description: Support subagent instructions +tools: Read, Grep +model: sonnet +--- + +# Support subagent + +Ask for an order ID and follow the support policy. diff --git a/internal/sync/adapter/markdown/testdata/profiles/copilot.instructions.md b/internal/sync/adapter/markdown/testdata/profiles/copilot.instructions.md new file mode 100644 index 000000000..024f8f369 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/copilot.instructions.md @@ -0,0 +1,7 @@ +--- +applyTo: "src/support/**/*.ts" +--- + +# Support + +Use the support policy. diff --git a/internal/sync/adapter/markdown/testdata/profiles/cursor-rule.mdc b/internal/sync/adapter/markdown/testdata/profiles/cursor-rule.mdc new file mode 100644 index 000000000..6eef5da26 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/cursor-rule.mdc @@ -0,0 +1,9 @@ +--- +description: Support instructions +globs: "src/support/**/*.ts" +alwaysApply: false +--- + +# Support + +Ask for an order ID before discussing an order. diff --git a/internal/sync/adapter/node.go b/internal/sync/adapter/node.go new file mode 100644 index 000000000..24cd82812 --- /dev/null +++ b/internal/sync/adapter/node.go @@ -0,0 +1,207 @@ +package adapter + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strconv" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Node is one logical value that an adapter exposes to select and map paths. +// +// A leaf has a Value, the byte span that holds it, and an Encode function that +// writes a new value in the syntax of the source. An object has Fields. A +// collection has Items. A node whose Value is nil is unknown: the adapter saw +// it but cannot read it statically. A read-only node has a Reason, and an +// optional Code, that explain why sync cannot write it. +type Node struct { + Value any + Fields map[string]*Node + Items []*Node + Collection bool + Start int + End int + File string + Capability Capability + Reason string + Code syncdomain.ErrorCode + Encode func(any) ([]byte, error) +} + +// String returns a writable string leaf. encode writes a new string in the +// syntax of the source. +func String(value string, start, end int, encode func(string) ([]byte, error)) *Node { + return &Node{ + Value: value, Start: start, End: end, Capability: CapabilityDirect, + Encode: func(value any) ([]byte, error) { + text, ok := value.(string) + if !ok { + return nil, TypeMismatch("source", "a string") + } + return encode(text) + }, + } +} + +// Number returns a writable number leaf. +func Number(value float64, start, end int, encode func(float64) ([]byte, error)) *Node { + return &Node{ + Value: value, Start: start, End: end, Capability: CapabilityDirect, + Encode: func(value any) ([]byte, error) { + number, ok := AsNumber(value) + if !ok { + return nil, TypeMismatch("source", "a number") + } + return encode(number) + }, + } +} + +// Bool returns a writable boolean leaf. encode writes a new boolean in the +// syntax of the source. The default writes true or false. +func Bool(value bool, start, end int, encode func(bool) ([]byte, error)) *Node { + if encode == nil { + encode = func(value bool) ([]byte, error) { return []byte(strconv.FormatBool(value)), nil } + } + return &Node{ + Value: value, Start: start, End: end, Capability: CapabilityDirect, + Encode: func(value any) ([]byte, error) { + boolean, ok := value.(bool) + if !ok { + return nil, TypeMismatch("source", "a boolean") + } + return encode(boolean) + }, + } +} + +// ReadOnly returns a node that sync can see but cannot write. +func ReadOnly(start, end int, reason string) *Node { + return &Node{Start: start, End: end, Capability: CapabilityReadOnly, Reason: reason} +} + +// Collections returns the root of a code source: one named collection of +// resources for each logical kind, such as agents, prompts, tools, and +// constants. +func Collections(collections map[string][]*Node) *Node { + fields := make(map[string]*Node, len(collections)) + for name, items := range collections { + fields[name] = &Node{Items: items, Collection: true} + } + return &Node{Fields: fields} +} + +// Resource returns one logical resource with its id field set, so that a +// keyed selector such as agents[id="support"] can find it. +func Resource(id string, fields map[string]*Node) *Node { + fields["id"] = &Node{Value: id} + return &Node{Fields: fields} +} + +// IsCollection reports whether a path can index the node. +func (node *Node) IsCollection() bool { + if node == nil { + return false + } + if node.Collection || node.Items != nil { + return true + } + value := reflect.ValueOf(node.Value) + return value.IsValid() && (value.Kind() == reflect.Array || value.Kind() == reflect.Slice) +} + +// MarkReadOnly makes the node and every node below it read-only. +func (node *Node) MarkReadOnly(reason string) { + if node == nil { + return + } + node.Capability, node.Reason, node.Encode = CapabilityReadOnly, reason, nil + for _, child := range node.Fields { + child.MarkReadOnly(reason) + } + for _, child := range node.Items { + child.MarkReadOnly(reason) + } +} + +// SetFile records the file that holds the node and every node below it that +// has no file yet. An imported value keeps the file that it came from. +func (node *Node) SetFile(file string) { + node.setFile(file, map[*Node]struct{}{}) +} + +func (node *Node) setFile(file string, visited map[*Node]struct{}) { + if node == nil { + return + } + if _, seen := visited[node]; seen { + return + } + visited[node] = struct{}{} + if node.File == "" { + node.File = file + } + for _, child := range node.Fields { + child.setFile(file, visited) + } + for _, child := range node.Items { + child.setFile(file, visited) + } +} + +// AsNumber returns the float64 value of a number from any parser. +func AsNumber(value any) (float64, bool) { + switch value := value.(type) { + case float64: + return value, true + case int: + return float64(value), true + case int64: + return float64(value), true + default: + return 0, false + } +} + +// TypeMismatch is the problem of a mapped field whose value has the wrong type. +func TypeMismatch(field, expected string) error { + return syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("mapped field %q needs %s", field, expected), + "Map the field to a compatible static value.", + ) +} + +// JSONEncoder encodes a value as JSON in the layout of the raw text that it +// replaces. A value that spanned lines keeps its indentation unit and the +// indentation of its closing line. A value on one line stays on one line. +func JSONEncoder(raw []byte) func(any) ([]byte, error) { + return func(next any) ([]byte, error) { + unit, prefix, multiline := jsonIndentation(raw) + if !multiline { + return json.Marshal(next) + } + return json.MarshalIndent(next, prefix, unit) + } +} + +// jsonIndentation returns the indentation unit and the indentation of the +// closing line of a multi-line JSON value. +func jsonIndentation(raw []byte) (unit, prefix string, multiline bool) { + lines := bytes.Split(raw, []byte("\n")) + if len(lines) < 2 { + return "", "", false + } + last := lines[len(lines)-1] + prefix = string(last[:len(last)-len(bytes.TrimLeft(last, " \t"))]) + for _, line := range lines[1 : len(lines)-1] { + indent := line[:len(line)-len(bytes.TrimLeft(line, " \t"))] + if len(indent) > len(prefix) { + return string(indent[len(prefix):]), prefix, true + } + } + return " ", prefix, true +} diff --git a/internal/sync/adapter/parse.go b/internal/sync/adapter/parse.go new file mode 100644 index 000000000..c38f3799d --- /dev/null +++ b/internal/sync/adapter/parse.go @@ -0,0 +1,324 @@ +package adapter + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// ParseSource reads one mapped resource from current source bytes. A +// variation projection has the mode that the source implies. The planner +// converts it to the mode of the config. +func ParseSource( + bundle SourceBundle, + id syncdomain.ResourceID, + reference syncdomain.Reference, +) (syncdomain.Projection, error) { + projection, _, err := ParseSourceWithSelections(bundle, id, reference) + return projection, err +} + +// ParseSourceWithSelections reads one mapped resource and the identity of +// each indexed item on its paths. +func ParseSourceWithSelections( + bundle SourceBundle, + id syncdomain.ResourceID, + reference syncdomain.Reference, +) (syncdomain.Projection, []syncdomain.IndexedSelection, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return nil, nil, err + } + root, err := Parse(bundle, reference.Format) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selected, selectedIndexes, err := root.resolve(reference.Select, "select") + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + selections, err := indexedSelections(selectedIndexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + fields := make([]string, 0, len(reference.Map)) + for field := range reference.Map { + fields = append(fields, field) + } + slices.Sort(fields) + for _, field := range fields { + _, indexes, err := selected.resolve(reference.Map[field], "map."+field) + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("map field %q: %w", field, err), id, bundle.Primary.Path) + } + mappedSelections, err := indexedSelections(indexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selections = append(selections, mappedSelections...) + } + slices.SortFunc(selections, func(left, right syncdomain.IndexedSelection) int { + return strings.Compare(left.Path, right.Path) + }) + projection, err := buildProjection(id.Kind, selected, reference.Map) + if variation, ok := projection.(syncdomain.VariationProjection); ok { + if reference.Tools != nil { + keys := attachmentKeys(*reference.Tools) + variation.Tools = &keys + variation.FieldStates[syncdomain.FieldTools] = syncdomain.FieldState{ + Capability: syncdomain.CapabilityDirect, Known: true, + } + } + if reference.Skills != nil { + keys := attachmentKeys(*reference.Skills) + variation.Skills = &keys + variation.FieldStates[syncdomain.FieldSkills] = syncdomain.FieldState{ + Capability: syncdomain.CapabilityDirect, Known: true, + } + } + projection = variation + } + return projection, selections, annotateProblem(err, id, bundle.Primary.Path) +} + +// buildProjection reads each mapped field below the selected node into the +// typed projection of the kind. +func buildProjection( + kind syncdomain.Kind, + selected *Node, + fieldMap map[string]string, +) (syncdomain.Projection, error) { + switch kind { + case syncdomain.KindVariation: + _, promptMapped := fieldMap[string(syncdomain.FieldPrompt)] + _, messagesMapped := fieldMap[string(syncdomain.FieldMessages)] + switch { + case !promptMapped && !messagesMapped: + return nil, fmt.Errorf("map field %q is required for a variation", syncdomain.FieldPrompt) + case promptMapped && messagesMapped: + // Both names fill the one prompt of the variation, so a second one + // would overwrite or shadow the first. + return nil, fmt.Errorf( + "map fields %q and %q cannot both be set for a variation", + syncdomain.FieldPrompt, syncdomain.FieldMessages, + ) + } + projection := syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, FieldStates: make(map[syncdomain.Field]syncdomain.FieldState), + } + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: node.Value != nil, Reason: node.Reason, Code: node.Code, File: node.File, + } + parameterKey, parameterField := syncdomain.ModelParameterKey(field) + if parameterField { + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: true, Reason: node.Reason, Code: node.Code, File: node.File, + } + value, ok := jsonValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "a JSON-compatible value") + } + if projection.Parameters == nil { + parameters := map[string]any{} + projection.Parameters = ¶meters + } + (*projection.Parameters)[parameterKey] = value + continue + } + if field == syncdomain.FieldModelParameters && node.Value == nil { + return nil, TypeMismatch(name, "an object") + } + if node.Value == nil { + continue + } + switch field { + case syncdomain.FieldPrompt, syncdomain.FieldMessages: + switch value := syncdomain.NormalizePrompt(node.Value).(type) { + case string: + projection.Instructions = &value + case []syncdomain.Message: + for _, message := range value { + if !syncdomain.ValidMessageRole(message.Role) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("message role %q is not supported", message.Role), + "Use system, user, or assistant.", + ) + } + } + projection.Mode = syncdomain.VariationModeCompletion + projection.Messages = &value + default: + return nil, TypeMismatch(name, "a string or a message list") + } + case syncdomain.FieldName: + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + projection.Name = &value + case syncdomain.FieldModel: + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + projection.Model = &value + case syncdomain.FieldModelParameters: + value, ok := objectValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "an object") + } + value = cloneObject(value) + delete(value, "tools") + projection.Parameters = &value + } + } + return projection, nil + case syncdomain.KindTool: + projection := syncdomain.ToolProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: node.Value != nil, Reason: node.Reason, Code: node.Code, File: node.File, + } + if node.Value == nil { + continue + } + switch field { + case syncdomain.FieldDescription: + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + projection.Description = &value + case syncdomain.FieldSchema: + value, ok := objectValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "an object") + } + projection.Schema = &value + case syncdomain.FieldCustomParameters: + value, ok := objectValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "an object") + } + projection.CustomParameters = &value + case syncdomain.FieldTags: + value, ok := stringList(node.Value) + if !ok { + return nil, TypeMismatch(name, "a string collection") + } + projection.Tags = &value + } + } + return projection, nil + case syncdomain.KindSkill: + projection := syncdomain.SkillProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: node.Value != nil, Reason: node.Reason, Code: node.Code, File: node.File, + } + if node.Value == nil { + continue + } + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + if field == syncdomain.FieldDescription { + projection.Description = &value + } else { + projection.Markdown = &value + } + } + return projection, nil + default: + return nil, fmt.Errorf("unsupported resource kind %q", kind) + } +} + +func mappedNode(root *Node, name, path string) (*Node, error) { + node, err := root.Resolve(path) + if err != nil { + return nil, fmt.Errorf("map field %q: %w", name, err) + } + return node, nil +} + +func attachmentKeys(refs []syncdomain.AttachmentRef) []string { + keys := make([]string, len(refs)) + for index := range refs { + keys[index] = refs[index].Key + } + slices.Sort(keys) + return keys +} + +func jsonValue(value any) (any, bool) { + encoded, err := json.Marshal(value) + if err != nil { + return nil, false + } + var decoded any + if err := json.Unmarshal(encoded, &decoded); err != nil { + return nil, false + } + return decoded, true +} + +func objectValue(value any) (map[string]any, bool) { + object, ok := value.(map[string]any) + if !ok { + return nil, false + } + return object, true +} + +func cloneObject(value map[string]any) map[string]any { + cloned, ok := jsonValue(value) + if !ok { + return map[string]any{} + } + object, ok := cloned.(map[string]any) + if !ok { + return map[string]any{} + } + return object +} + +func stringList(value any) ([]string, bool) { + switch values := value.(type) { + case []string: + return slices.Clone(values), true + case []any: + result := make([]string, len(values)) + for index, value := range values { + text, ok := value.(string) + if !ok { + return nil, false + } + result[index] = text + } + return result, true + default: + return nil, false + } +} diff --git a/internal/sync/adapter/path.go b/internal/sync/adapter/path.go new file mode 100644 index 000000000..767aef38c --- /dev/null +++ b/internal/sync/adapter/path.go @@ -0,0 +1,270 @@ +package adapter + +import ( + "encoding/json" + "fmt" + "strconv" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type sourcePath struct { + root bool + steps []pathStep +} + +type pathStep struct { + key string + filters []pathFilter + index *int +} + +type pathFilter struct { + key string + value string +} + +// ParsePath checks the shared select and map path grammar. +func ParsePath(text string) error { + _, err := parsePath(text) + return err +} + +func parsePath(text string) (sourcePath, error) { + if text == "$" { + return sourcePath{root: true}, nil + } + if text == "" { + return sourcePath{}, pathSyntaxError(text, 0, "path is empty") + } + + parser := pathParser{text: text} + key, err := parser.readKey() + if err != nil { + return sourcePath{}, err + } + result := sourcePath{steps: []pathStep{{key: key}}} + for parser.offset < len(text) { + switch text[parser.offset] { + case '.': + parser.offset++ + key, err = parser.readIdentifier() + if err != nil { + return sourcePath{}, err + } + result.steps = append(result.steps, pathStep{key: key}) + case '[': + step, kind, err := parser.readBracket() + if err != nil { + return sourcePath{}, err + } + if kind == bracketFilter { + last := len(result.steps) - 1 + if result.steps[last].index != nil { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a selector must follow a field") + } + if len(result.steps[last].filters) != 0 { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a path step can have only one selector") + } + result.steps[last].filters = step.filters + } else { + result.steps = append(result.steps, step) + } + default: + return sourcePath{}, pathSyntaxError(text, parser.offset, "expected a dot or bracket") + } + } + return result, nil +} + +type pathParser struct { + text string + offset int +} + +type bracketKind int + +const ( + bracketField bracketKind = iota + bracketFilter + bracketIndex +) + +func (parser *pathParser) readKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '[' { + step, kind, err := parser.readBracket() + if err != nil { + return "", err + } + if kind != bracketField { + return "", pathSyntaxError(parser.text, parser.offset, "a path cannot start with a selector or index") + } + return step.key, nil + } + return parser.readIdentifier() +} + +func (parser *pathParser) readIdentifier() (string, error) { + start := parser.offset + if start >= len(parser.text) || !identifierStart(parser.text[start]) { + return "", pathSyntaxError(parser.text, start, "expected an identifier") + } + parser.offset++ + for parser.offset < len(parser.text) && identifierContinue(parser.text[parser.offset]) { + parser.offset++ + } + return parser.text[start:parser.offset], nil +} + +func (parser *pathParser) readBracket() (pathStep, bracketKind, error) { + start := parser.offset + parser.offset++ + if parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + return parser.readIndex(start) + } + key, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketField, err + } + if parser.offset >= len(parser.text) { + return pathStep{}, bracketField, pathSyntaxError(parser.text, start, "unclosed bracket") + } + if parser.text[parser.offset] == ']' { + parser.offset++ + return pathStep{key: key}, bracketField, nil + } + if parser.text[parser.offset] != '=' { + return pathStep{}, bracketField, pathSyntaxError(parser.text, parser.offset, "expected equals or closing bracket") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + value, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters := []pathFilter{{key: key, value: value}} + for parser.offset < len(parser.text) && parser.text[parser.offset] == ',' { + parser.offset++ + nextKey, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketFilter, err + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '=' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected equals") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + nextValue, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters = append(filters, pathFilter{key: nextKey, value: nextValue}) + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected a closing bracket") + } + parser.offset++ + return pathStep{filters: filters}, bracketFilter, nil +} + +func (parser *pathParser) readIndex(start int) (pathStep, bracketKind, error) { + numberStart := parser.offset + for parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + parser.offset++ + } + raw := parser.text[numberStart:parser.offset] + if len(raw) > 1 && raw[0] == '0' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "an index cannot have a leading zero") + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, parser.offset, "an index must be a decimal integer") + } + value, err := strconv.ParseUint(raw, 10, 31) + if err != nil { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "index exceeds 2147483647") + } + parser.offset++ + index := int(value) + return pathStep{index: &index}, bracketIndex, nil +} + +func (parser *pathParser) readBracketKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '"' { + return parser.readJSONString() + } + return parser.readIdentifier() +} + +func (parser *pathParser) readJSONString() (string, error) { + start := parser.offset + parser.offset++ + for parser.offset < len(parser.text) { + switch parser.text[parser.offset] { + case '\\': + parser.offset += 2 + case '"': + parser.offset++ + quoted := parser.text[start:parser.offset] + if err := rejectUnpairedSurrogate(quoted); err != nil { + return "", pathSyntaxError(parser.text, start, err.Error()) + } + var value string + if err := json.Unmarshal([]byte(quoted), &value); err != nil { + return "", pathSyntaxError(parser.text, start, "invalid JSON string") + } + return value, nil + default: + parser.offset++ + } + } + return "", pathSyntaxError(parser.text, start, "unclosed JSON string") +} + +func rejectUnpairedSurrogate(quoted string) error { + for index := 0; index+5 < len(quoted); index++ { + if quoted[index:index+2] != `\u` { + continue + } + code, err := strconv.ParseUint(quoted[index+2:index+6], 16, 16) + if err != nil { + continue + } + switch { + case code >= 0xd800 && code <= 0xdbff: + if index+11 >= len(quoted) || quoted[index+6:index+8] != `\u` { + return fmt.Errorf("unpaired Unicode surrogate") + } + low, err := strconv.ParseUint(quoted[index+8:index+12], 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return fmt.Errorf("unpaired Unicode surrogate") + } + index += 6 + case code >= 0xdc00 && code <= 0xdfff: + return fmt.Errorf("unpaired Unicode surrogate") + } + } + return nil +} + +func pathSyntaxError(path string, offset int, reason string) error { + if offset > len(path) { + offset = len(path) + } + return syncdomain.NewProblem( + syncdomain.ErrorInvalidPathSyntax, + fmt.Sprintf("invalid path at byte %d: %s", offset, reason), + "Use $, named fields, zero-based indexes, keyed selectors, or quoted literal keys.", + ) +} + +func identifierStart(value byte) bool { + return value == '_' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' +} + +func identifierContinue(value byte) bool { + return identifierStart(value) || value >= '0' && value <= '9' +} diff --git a/internal/sync/adapter/reference.go b/internal/sync/adapter/reference.go new file mode 100644 index 000000000..b20446631 --- /dev/null +++ b/internal/sync/adapter/reference.go @@ -0,0 +1,141 @@ +package adapter + +import ( + "fmt" + "slices" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// NormalizeReference validates a descriptor and fills conventional defaults. +func NormalizeReference(id syncdomain.ResourceID, reference syncdomain.Reference) (syncdomain.Reference, error) { + if reference.File == "" { + return syncdomain.Reference{}, invalidReference(id, "file is required", "Pass --file.") + } + registered, err := lookup(reference.Format) + if err != nil { + return syncdomain.Reference{}, err + } + defaultSelect, defaultMap := registered.Defaults(id.Kind) + if reference.Select == "" { + if defaultSelect == "" { + return syncdomain.Reference{}, invalidReference( + id, fmt.Sprintf("select is required for format %q", reference.Format), "Pass --select.", + ) + } + reference.Select = defaultSelect + } + if err := ParsePath(reference.Select); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("select %q: %w", reference.Select, err), id, reference.File, + ) + } + if len(reference.Map) == 0 { + if len(defaultMap) == 0 { + return syncdomain.Reference{}, invalidReference( + id, "map is required", "Pass at least one --map field=path value.", + ) + } + reference.Map = defaultMap + } + for target := range reference.Map { + if err := ParsePath(target); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map target %q: %w", target, err), id, reference.File, + ) + } + } + fields, err := syncdomain.ProjectionFields(reference) + if err != nil { + return syncdomain.Reference{}, invalidReference(id, err.Error(), "Use a field valid for this resource kind.") + } + wholeParameters := false + parameterKeys := map[string]syncdomain.Field{} + for _, field := range fields { + if field == syncdomain.FieldTools || field == syncdomain.FieldSkills { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map key %q is not allowed", field), + fmt.Sprintf("List %s keys under top-level %q.", field, field), + ) + } + if !fieldAllowed(id.Kind, field) { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("field %q is not valid for %s", field, id.Kind), + "Use a mapped field supported by this resource kind.", + ) + } + if field == syncdomain.FieldModelParameters { + wholeParameters = true + } + if key, parameter := syncdomain.ModelParameterKey(field); parameter { + if key == "tools" { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map target %q duplicates the tools field", field), + "List tool keys under top-level \"tools\".", + ) + } + if previous, duplicate := parameterKeys[key]; duplicate { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map targets %q and %q select the same model parameter", previous, field), + "Map each model parameter key once.", + ) + } + parameterKeys[key] = field + } + if err := ParsePath(reference.Map[string(field)]); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map field %q: %w", field, err), id, reference.File, + ) + } + } + if wholeParameters && len(parameterKeys) != 0 { + return syncdomain.Reference{}, invalidReference( + id, + "map target \"model.parameters\" overlaps an individual model parameter target", + "Map the full parameters object or individual parameter keys.", + ) + } + return reference, nil +} + +func invalidReference(id syncdomain.ResourceID, message, suggestion string) error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidReference, message, suggestion, + ).WithResource(id) +} + +func fieldAllowed(kind syncdomain.Kind, field syncdomain.Field) bool { + switch kind { + case syncdomain.KindVariation: + if _, parameter := syncdomain.ModelParameterKey(field); parameter { + return true + } + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldPrompt, syncdomain.FieldMessages, syncdomain.FieldName, syncdomain.FieldModel, + syncdomain.FieldModelParameters, + }, field) + case syncdomain.KindTool: + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldDescription, syncdomain.FieldSchema, + syncdomain.FieldCustomParameters, syncdomain.FieldTags, + }, field) + case syncdomain.KindSkill: + return field == syncdomain.FieldDescription || field == syncdomain.FieldMarkdown + default: + return false + } +} + +func annotateProblem(err error, id syncdomain.ResourceID, file string) error { + if err == nil { + return nil + } + if problem, ok := syncdomain.ProblemFrom(err); ok { + _ = problem.WithResource(id).WithFile(file, 0, 0) + } + return err +} diff --git a/internal/sync/adapter/resolve.go b/internal/sync/adapter/resolve.go new file mode 100644 index 000000000..0b769cf82 --- /dev/null +++ b/internal/sync/adapter/resolve.go @@ -0,0 +1,232 @@ +package adapter + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Resolve returns the node at a select or map path below root. +func (root *Node) Resolve(text string) (*Node, error) { + node, _, err := root.resolve(text, "") + return node, err +} + +// indexedResolution is one index step on a resolved path: where it is, how +// long the collection was, and which item it chose. +type indexedResolution struct { + path string + collectionLength int + item *Node +} + +// resolve returns the node at a path and the index steps that it took. scope +// names the path in the identity of each index step. +func (root *Node) resolve(text, scope string) (*Node, []indexedResolution, error) { + path, err := parsePath(text) + if err != nil { + return nil, nil, err + } + if path.root { + return root, nil, nil + } + current := root + var indexes []indexedResolution + for _, step := range path.steps { + if step.index != nil { + if !current.IsCollection() { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("index %d needs a collection", *step.index), + "Select a collection before this index.", + ) + } + if *step.index >= len(current.Items) { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorIndexOutOfRange, + fmt.Sprintf("index %d is out of range for collection of length %d", *step.index, len(current.Items)), + "Use an index within the collection.", + ) + } + collectionLength := len(current.Items) + current = current.Items[*step.index] + indexes = append(indexes, indexedResolution{ + path: fmt.Sprintf("%s#%d", scope, len(indexes)), + collectionLength: collectionLength, + item: current, + }) + continue + } + if current.Fields == nil { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("%q does not contain fields", step.key), + "Select an object before this field.", + ) + } + next, ok := current.Fields[step.key] + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("field %q was not found", step.key), + "Update the select or map path.", + ) + } + current = next + if len(step.filters) == 0 { + continue + } + var matches []*Node + for _, item := range current.Items { + matched := true + for _, filter := range step.filters { + identity, ok := item.Fields[filter.key] + if !ok { + matched = false + break + } + text, ok := identity.Value.(string) + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorFilterTypeMismatch, + fmt.Sprintf("selector field %q is not a string", filter.key), + "Select a direct string identity field.", + ) + } + if text != filter.value { + matched = false + break + } + } + if matched { + matches = append(matches, item) + } + } + selector := formatFilters(step.filters) + switch len(matches) { + case 0: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("selector %s matched no values", selector), + "Update the selector or restore the source value.", + ) + case 1: + current = matches[0] + default: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathAmbiguous, + fmt.Sprintf("selector %s matched %d values", selector, len(matches)), + "Use a selector that matches one resource.", + ) + } + } + return current, indexes, nil +} + +// indexedSelections records the identity of each indexed item, so that a +// later sync can tell when the item at that position changed. +func indexedSelections(indexes []indexedResolution, selectedRoot *Node) ([]syncdomain.IndexedSelection, error) { + selections := make([]syncdomain.IndexedSelection, 0, len(indexes)) + var ancestors []string + for _, index := range indexes { + identity := directSelectionIdentity(index.item) + if identity == nil { + // Without scalar identity, bind the nearest selected resource root and prior indexed ancestors. + identity = struct { + Root any `json:"root"` + Ancestors []string `json:"ancestors,omitempty"` + Item any `json:"item"` + }{ + Root: stableNodeValue(selectedRoot, map[*Node]bool{}), + Ancestors: slices.Clone(ancestors), + Item: stableNodeValue(index.item, map[*Node]bool{}), + } + } + fingerprint, err := selectionFingerprint(identity) + if err != nil { + return nil, err + } + selections = append(selections, syncdomain.IndexedSelection{ + Path: index.path, CollectionLength: index.collectionLength, Fingerprint: fingerprint, + }) + ancestors = append(ancestors, fingerprint) + } + return selections, nil +} + +func directSelectionIdentity(node *Node) any { + if node == nil || node.Fields == nil { + return nil + } + identity := map[string]any{} + for _, key := range []string{"id", "name", "role", "type"} { + field, ok := node.Fields[key] + if !ok || !scalarValue(field.Value) { + continue + } + identity[key] = field.Value + } + if len(identity) == 0 { + return nil + } + return identity +} + +func scalarValue(value any) bool { + switch value.(type) { + case string, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return true + default: + return false + } +} + +func stableNodeValue(node *Node, visited map[*Node]bool) any { + if node == nil { + return nil + } + if node.Value != nil { + return node.Value + } + if visited[node] { + return "" + } + visited[node] = true + defer delete(visited, node) + if node.Fields != nil { + value := make(map[string]any, len(node.Fields)) + for key, field := range node.Fields { + value[key] = stableNodeValue(field, visited) + } + return value + } + if node.IsCollection() { + value := make([]any, len(node.Items)) + for index, item := range node.Items { + value[index] = stableNodeValue(item, visited) + } + return value + } + return nil +} + +func selectionFingerprint(value any) (string, error) { + canonical, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode indexed selection fingerprint: %w", err) + } + sum := sha256.Sum256(canonical) + return fmt.Sprintf("sha256:%x", sum), nil +} + +func formatFilters(filters []pathFilter) string { + parts := make([]string, len(filters)) + for index, filter := range filters { + parts[index] = fmt.Sprintf("%s=%q", filter.key, filter.value) + } + return strings.Join(parts, ",") +} diff --git a/internal/sync/adapter/update.go b/internal/sync/adapter/update.go new file mode 100644 index 000000000..f9b64793d --- /dev/null +++ b/internal/sync/adapter/update.go @@ -0,0 +1,167 @@ +package adapter + +import ( + "bytes" + "encoding/json" + "fmt" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// UpdateSource plans the exact edits that give the mapped fields of a source +// the values of the desired projection. A field whose value is the same has +// no entry. A field that the source cannot write has a read-only entry. +func UpdateSource( + bundle SourceBundle, + id syncdomain.ResourceID, + reference syncdomain.Reference, + desired syncdomain.Projection, +) (UpdateResult, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return UpdateResult{}, err + } + if desired.Kind() != id.Kind { + return UpdateResult{}, fmt.Errorf("projection kind %q does not match %s", desired.Kind(), id.Label()) + } + root, err := Parse(bundle, reference.Format) + if err != nil { + return UpdateResult{}, annotateProblem(err, id, bundle.Primary.Path) + } + selected, err := root.Resolve(reference.Select) + if err != nil { + return UpdateResult{}, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + + result := UpdateResult{Fields: make(map[syncdomain.Field]FieldUpdate)} + for name, sourcePath := range reference.Map { + field := syncdomain.Field(name) + wanted, present := desired.Value(field) + if !present { + continue + } + current, err := selected.Resolve(sourcePath) + if err != nil { + return UpdateResult{}, fmt.Errorf("%s: map field %q: %w", bundle.Primary.Path, field, err) + } + if field == syncdomain.FieldPrompt || field == syncdomain.FieldMessages { + if semanticEqual(syncdomain.NormalizePrompt(current.Value), syncdomain.NormalizePrompt(wanted)) { + continue + } + if update, handled := updatePrompt(bundle, current, wanted); handled { + if len(update.Edits) != 0 || update.Capability != CapabilityDirect { + result.Fields[field] = update + } + continue + } + } + if semanticEqual(current.Value, wanted) { + continue + } + result.Fields[field] = updateLeaf(bundle, current, wanted) + } + return result, nil +} + +// updatePrompt reconciles the two prompt shapes. A text source holds one +// system message. A message list source changes the content of each message +// in place. handled is false when the leaf update applies. +func updatePrompt(bundle SourceBundle, current *Node, wanted any) (update FieldUpdate, handled bool) { + have, sourceIsMessages := current.Value.([]syncdomain.Message) + want, wantIsMessages := wanted.([]syncdomain.Message) + switch { + case sourceIsMessages: + if text, ok := wanted.(string); ok { + want = []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: text}} + } + return updateMessageContents(bundle, current, have, want), true + case wantIsMessages: + if len(want) != 1 || want[0].Role != syncdomain.RoleSystem { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "text sources support one system message"}, true + } + return updateLeaf(bundle, current, want[0].Content), true + default: + return FieldUpdate{}, false + } +} + +// updateLeaf plans the edit of one leaf value. +func updateLeaf(bundle SourceBundle, current *Node, wanted any) FieldUpdate { + if current.Capability != CapabilityDirect || current.Encode == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: current.Reason} + } + replacement, err := current.Encode(wanted) + if err != nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: err.Error()} + } + document, ok := bundleDocument(bundle, current.File) + if !ok { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "source dependency is not available"} + } + return FieldUpdate{Capability: CapabilityDirect, Edits: []TextEdit{{ + File: document.Path, + Start: current.Start, + End: current.End, + Expected: bytes.Clone(document.Data[current.Start:current.End]), + Replacement: replacement, + }}} +} + +func updateMessageContents( + bundle SourceBundle, + current *Node, + have, want []syncdomain.Message, +) FieldUpdate { + if len(have) != len(want) { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "add or remove messages by hand", + } + } + var edits []TextEdit + for index := range have { + if have[index].Role != want[index].Role { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "reorder messages or change roles by hand", + } + } + if syncdomain.NormalizePromptText(have[index].Content) == syncdomain.NormalizePromptText(want[index].Content) { + continue + } + if index >= len(current.Items) || current.Items[index].Fields["content"] == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "message content is not a writable literal"} + } + content := current.Items[index].Fields["content"] + if content.Capability != CapabilityDirect || content.Encode == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: content.Reason} + } + replacement, err := content.Encode(want[index].Content) + if err != nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: err.Error()} + } + document, ok := bundleDocument(bundle, content.File) + if !ok { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "source dependency is not available"} + } + edits = append(edits, TextEdit{ + File: document.Path, Start: content.Start, End: content.End, + Expected: bytes.Clone(document.Data[content.Start:content.End]), Replacement: replacement, + }) + } + return FieldUpdate{Capability: CapabilityDirect, Edits: edits} +} + +func bundleDocument(bundle SourceBundle, file string) (Document, bool) { + if file == "" || file == bundle.Primary.Path { + return bundle.Primary, true + } + document, ok := bundle.Dependencies[file] + return document, ok +} + +func semanticEqual(left, right any) bool { + leftJSON, leftErr := json.Marshal(left) + rightJSON, rightErr := json.Marshal(right) + return leftErr == nil && rightErr == nil && bytes.Equal(leftJSON, rightJSON) +} diff --git a/internal/sync/adapter/yamltree/yamltree.go b/internal/sync/adapter/yamltree/yamltree.go new file mode 100644 index 000000000..c4df24ffd --- /dev/null +++ b/internal/sync/adapter/yamltree/yamltree.go @@ -0,0 +1,331 @@ +// Package yamltree builds logical nodes from a YAML document, with the exact +// byte span of each scalar. The YAML adapter and Markdown front matter use it. +package yamltree + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +// Region is a YAML document, or part of one, and the byte offset of its first +// byte in the file that contains it. +type Region struct { + Data []byte + StartOffset int +} + +// ScalarSpan returns the byte span and the raw text of one scalar. flow +// is true inside a flow sequence or a flow mapping, where a comma or a closing +// bracket ends a plain scalar. A plain scalar that spans lines has no exact +// span, so the caller treats it as read-only. +func ScalarSpan(region Region, node *yaml.Node, flow bool) (start, end int, raw string, exact bool, err error) { + lines := bytes.SplitAfter(region.Data, []byte("\n")) + if node.Line < 1 || node.Line > len(lines) || node.Column < 1 { + return 0, 0, "", false, fmt.Errorf("cannot locate YAML scalar") + } + start = region.StartOffset + for index := 0; index < node.Line-1; index++ { + start += len(lines[index]) + } + line := bytes.TrimRight(lines[node.Line-1], "\r\n") + localStart := byteOffsetForColumn(line, node.Column) + if localStart > len(line) { + return 0, 0, "", false, fmt.Errorf("cannot locate YAML scalar") + } + rest := line[localStart:] + switch { + case len(rest) != 0 && (rest[0] == '\'' || rest[0] == '"'): + var closed bool + end, closed := yamlQuotedEnd(rest) + raw, exact = string(rest[:end]), closed + default: + raw = string(yamlPlainScalar(rest, flow)) + exact = raw == node.Value + } + start += localStart + return start, start + len(raw), raw, exact, nil +} + +func byteOffsetForColumn(line []byte, column int) int { + offset := 0 + for current := 1; current < column && offset < len(line); current++ { + _, width := utf8.DecodeRune(line[offset:]) + offset += width + } + return offset +} + +// yamlQuotedEnd returns the length of the quoted scalar at the start of text, +// and whether its closing quote is on this line. A quoted scalar that +// continues on the next line has no exact span. +func yamlQuotedEnd(text []byte) (int, bool) { + quote := text[0] + for index := 1; index < len(text); index++ { + switch { + case quote == '"' && text[index] == '\\': + index++ + case text[index] == quote && quote == '\'' && index+1 < len(text) && text[index+1] == '\'': + index++ + case text[index] == quote: + return index + 1, true + } + } + return len(text), false +} + +// yamlPlainScalar returns the plain scalar at the start of text. A space and +// a hash start a comment. In a flow collection, a comma or a closing bracket +// ends the scalar. +func yamlPlainScalar(text []byte, flow bool) []byte { + end := len(text) + if comment := bytes.Index(text, []byte(" #")); comment >= 0 { + end = comment + } + if flow { + if index := bytes.IndexAny(text[:end], ",]}"); index >= 0 { + end = index + } + } + return bytes.TrimRight(text[:end], " \t") +} + +func quoteYAML(value, previous string) string { + if strings.HasPrefix(previous, "'") { + if strings.ContainsAny(value, "\r\n") { + encoded, _ := json.Marshal(value) + return string(encoded) + } + return "'" + strings.ReplaceAll(value, "'", "''") + "'" + } + if strings.HasPrefix(previous, `"`) { + encoded, _ := json.Marshal(value) + return string(encoded) + } + if value != "" && !strings.ContainsAny(value, ":\n\r#{}[],&*!|>'\"%@`") && + value == strings.TrimSpace(value) { + var decoded any + if yaml.Unmarshal([]byte(value), &decoded) == nil { + if _, isString := decoded.(string); isString { + return value + } + } + } + encoded, _ := json.Marshal(value) + return string(encoded) +} + +// EncodeString writes a string in the quoting style of the scalar that it +// replaces. It rejects characters that YAML cannot hold. +func EncodeString(value, previous string) ([]byte, error) { + for _, character := range value { + valid := character == '\n' || character == '\r' || character == '\t' || + character >= 0x20 && character <= 0xd7ff || + character >= 0xe000 && character <= 0xfffd || + character >= 0x10000 && character <= utf8.MaxRune + if !valid || unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { + return nil, fmt.Errorf("YAML string contains unsupported character U+%04X", character) + } + } + return []byte(quoteYAML(value, previous)), nil +} + +// Build builds the logical node of one YAML node. flow is true +// inside a flow sequence or a flow mapping. +func Build(data []byte, node *yaml.Node, offset int, flow bool) (*adapter.Node, error) { + return build(data, node, offset, flow, map[*yaml.Node]bool{}) +} + +// build is Build with the set of anchored nodes that are still being built. +// An alias to one of those nodes is a cycle, which has no finite value. +func build(data []byte, node *yaml.Node, offset int, flow bool, open map[*yaml.Node]bool) (*adapter.Node, error) { + if node.Anchor != "" { + if open[node] { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + fmt.Sprintf("YAML alias *%s refers to a node that contains it", node.Anchor), + "Remove the circular alias.", + ) + } + open[node] = true + defer delete(open, node) + } + flow = flow || node.Style&yaml.FlowStyle != 0 + switch node.Kind { + case yaml.MappingNode: + return yamlMappingNode(data, node, offset, flow, open) + case yaml.SequenceNode: + return yamlSequenceNode(data, node, offset, flow, open) + case yaml.ScalarNode: + return yamlValueNode(data, node, offset, flow) + case yaml.AliasNode: + value, err := build(data, node.Alias, offset, flow, open) + if value != nil { + value.MarkReadOnly("YAML aliases are read-only") + } + return value, err + default: + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML node is not supported", "Use mappings, lists, or scalar values.", + ) + } +} + +func yamlMappingNode(data []byte, node *yaml.Node, offset int, flow bool, open map[*yaml.Node]bool) (*adapter.Node, error) { + fields := make(map[string]*adapter.Node, len(node.Content)/2) + value := make(map[string]any, len(node.Content)/2) + unsafe := node.Anchor != "" || customYAMLTag(node.Tag) + reason := "YAML anchors and tags are read-only" + var merges []*adapter.Node + merged := false + for index := 0; index+1 < len(node.Content); index += 2 { + keyNode, valueNode := node.Content[index], node.Content[index+1] + if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" && keyNode.Tag != "!!merge" { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML mapping key is not a string", + "Use direct string keys.", + ) + } + key := keyNode.Value + if _, duplicate := fields[key]; duplicate || key == "<<" && merged { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("YAML mapping contains duplicate key %q", key), + "Remove the duplicate key.", + ) + } + child, err := build(data, valueNode, offset, flow, open) + if err != nil { + return nil, err + } + if key == "<<" { + unsafe, reason, merged = true, "YAML merge keys are read-only", true + switch { + case child.Fields != nil: + merges = []*adapter.Node{child} + case child.Collection: + merges = child.Items + default: + return nil, invalidMerge() + } + continue + } + fields[key], value[key] = child, child.Value + } + // Merged keys come last: an explicit key wins over a merged key, and an + // earlier mapping in a merge list wins over a later one. + for _, merged := range merges { + if merged.Fields == nil { + return nil, invalidMerge() + } + for mergedKey, mergedValue := range merged.Fields { + if _, exists := fields[mergedKey]; !exists { + fields[mergedKey] = mergedValue + value[mergedKey] = mergedValue.Value + } + } + } + result := &adapter.Node{ + Value: value, Fields: fields, Capability: adapter.CapabilityReadOnly, + Reason: "YAML structural changes are read-only", + } + if unsafe { + result.MarkReadOnly(reason) + } + return result, nil +} + +func invalidMerge() error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML merge key value is not a mapping or a list of mappings", + "Merge one mapping or a list of mappings.", + ) +} + +func yamlSequenceNode(data []byte, node *yaml.Node, offset int, flow bool, open map[*yaml.Node]bool) (*adapter.Node, error) { + items := make([]*adapter.Node, 0, len(node.Content)) + values := make([]any, 0, len(node.Content)) + for _, childNode := range node.Content { + child, err := build(data, childNode, offset, flow, open) + if err != nil { + return nil, err + } + items, values = append(items, child), append(values, child.Value) + } + result := &adapter.Node{ + Value: values, Items: items, Collection: true, Capability: adapter.CapabilityReadOnly, + Reason: "YAML structural changes are read-only", + } + if node.Anchor != "" || customYAMLTag(node.Tag) { + result.MarkReadOnly("YAML anchors and tags are read-only") + } + return result, nil +} + +func yamlValueNode(data []byte, node *yaml.Node, offset int, flow bool) (*adapter.Node, error) { + region := Region{Data: data, StartOffset: offset} + start, end, raw, exact, err := ScalarSpan(region, node, flow) + if err != nil { + return nil, err + } + readOnly := !exact || node.Anchor != "" || customYAMLTag(node.Tag) || + node.Style == yaml.LiteralStyle || node.Style == yaml.FoldedStyle + reason := "YAML anchors, tags, block scalars, and multi-line scalars are read-only" + var result *adapter.Node + switch node.Tag { + case "!!str": + result = adapter.String(node.Value, start, end, func(text string) ([]byte, error) { + return EncodeString(text, raw) + }) + case "!!bool": + var value bool + if err := node.Decode(&value); err != nil { + result = unreadableScalar(node, start, end) + break + } + result = adapter.Bool(value, start, end, nil) + case "!!int", "!!float": + var number float64 + if err := node.Decode(&number); err != nil || math.IsInf(number, 0) || math.IsNaN(number) { + result = unreadableScalar(node, start, end) + break + } + result = adapter.Number(number, start, end, func(next float64) ([]byte, error) { + return []byte(strconv.FormatFloat(next, 'g', -1, 64)), nil + }) + case "!!null": + result = adapter.ReadOnly(start, end, "YAML null values are read-only") + default: + result = adapter.ReadOnly(start, end, "YAML scalar type is read-only") + result.Value = node.Value + } + if readOnly { + result.Capability, result.Reason, result.Encode = adapter.CapabilityReadOnly, reason, nil + } + return result, nil +} + +// unreadableScalar keeps a scalar that the YAML library cannot decode, or +// that has no JSON form such as infinity, as a read-only value, so that one +// odd spelling does not reject the document. +func unreadableScalar(node *yaml.Node, start, end int) *adapter.Node { + result := adapter.ReadOnly(start, end, "YAML scalar spelling is read-only") + result.Value = node.Value + return result +} + +func customYAMLTag(tag string) bool { + return tag != "" && !strings.HasPrefix(tag, "!!") +} diff --git a/internal/sync/adapter/yamltree/yamltree_test.go b/internal/sync/adapter/yamltree/yamltree_test.go new file mode 100644 index 000000000..8c1f5c6f5 --- /dev/null +++ b/internal/sync/adapter/yamltree/yamltree_test.go @@ -0,0 +1,140 @@ +package yamltree + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +func TestScalarSpanHelpers(t *testing.T) { + _, _, _, _, err := ScalarSpan(Region{}, &yaml.Node{}, false) + require.Error(t, err) + assert.Equal(t, `"line\nnext"`, quoteYAML("line\nnext", "'old'")) + assert.Equal(t, "'it''s'", quoteYAML("it's", "'old'")) + _, err = EncodeString("bad\x01", "old") + require.Error(t, err) + + end, closed := yamlQuotedEnd([]byte(`"unterminated`)) + assert.Equal(t, len(`"unterminated`), end) + assert.False(t, closed) + end, closed = yamlQuotedEnd([]byte(`'it''s'`)) + assert.Equal(t, len(`'it''s'`), end) + assert.True(t, closed) + assert.Equal(t, "value, next", string(yamlPlainScalar([]byte("value, next # comment"), false))) + assert.Equal(t, "value", string(yamlPlainScalar([]byte("value, next]"), true))) +} + +func TestQuotedScalarsThatSpanLinesAreReadOnly(t *testing.T) { + data := []byte("prompt: \"first line\n second line\"\nnext: value\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + + prompt := root.Fields["prompt"] + assert.Equal(t, "first line second line", prompt.Value) + assert.Equal(t, adapter.CapabilityReadOnly, prompt.Capability) + assert.Equal(t, adapter.CapabilityDirect, root.Fields["next"].Capability) +} + +func TestNumberSpellingsDecodeOrStayReadOnly(t *testing.T) { + data := []byte("hex: 0x1A\noctal: 0o17\nseparated: 1_000\nflag: True\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + + assert.Equal(t, 26.0, root.Fields["hex"].Value) + assert.Equal(t, 15.0, root.Fields["octal"].Value) + assert.Equal(t, 1000.0, root.Fields["separated"].Value) + assert.Equal(t, true, root.Fields["flag"].Value) + assert.Equal(t, adapter.CapabilityReadOnly, unreadableScalar(&yaml.Node{Value: "odd"}, 0, 3).Capability) +} + +func TestValidYAMLNumbersDoNotFailTheDocument(t *testing.T) { + for _, value := range []string{"0x10", "0o17", ".inf", "-.inf", ".nan"} { + t.Run(value, func(t *testing.T) { + source := "limit: " + value + "\nname: kept\n" + var document yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &document)) + root, err := Build([]byte(source), document.Content[0], 0, false) + require.NoError(t, err) + assert.Equal(t, "kept", root.Fields["name"].Value) + limit := root.Fields["limit"] + if _, finite := limit.Value.(float64); finite { + assert.Equal(t, adapter.CapabilityDirect, limit.Capability) + return + } + // Infinity and NaN have no JSON form, so they stay read-only text. + assert.Equal(t, value, limit.Value) + assert.Equal(t, adapter.CapabilityReadOnly, limit.Capability) + }) + } +} + +func TestCircularAliasIsRejected(t *testing.T) { + data := []byte("self: &self\n again: *self\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + _, err := Build(data, document.Content[0], 0, false) + require.Error(t, err) + assert.Contains(t, err.Error(), "*self refers to a node that contains it") + + data = []byte("base: &base\n name: shared\ncopy: *base\nlist: [*base, *base]\n") + document = yaml.Node{} + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + assert.Equal(t, "shared", root.Fields["copy"].Fields["name"].Value) + assert.Len(t, root.Fields["list"].Items, 2) +} + +func TestNullScalarsHaveNoValue(t *testing.T) { + data := []byte("tilde: ~\nword: null\nempty:\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + for _, key := range []string{"tilde", "word", "empty"} { + assert.Nil(t, root.Fields[key].Value, key) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields[key].Capability, key) + } +} + +func TestMergeKeysLetExplicitKeysWin(t *testing.T) { + data := []byte("base: &base\n name: shared\n limit: 1\nother: &other\n limit: 2\n extra: true\nagent:\n <<: *base\n name: own\nlist:\n limit: 9\n <<: [*base, *other]\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + + agent := root.Fields["agent"] + assert.Equal(t, "own", agent.Fields["name"].Value) + assert.Equal(t, 1.0, agent.Fields["limit"].Value) + assert.NotContains(t, agent.Fields, "<<") + assert.Equal(t, adapter.CapabilityReadOnly, agent.Capability) + + list := root.Fields["list"] + assert.Equal(t, 9.0, list.Fields["limit"].Value) + assert.Equal(t, "shared", list.Fields["name"].Value) + assert.Equal(t, true, list.Fields["extra"].Value) + assert.Equal(t, map[string]any{"limit": 9.0, "name": "shared", "extra": true}, list.Value) + + for name, source := range map[string]string{ + "duplicate key": "a: 1\na: 2\n", + "duplicate merge": "b: &b\n x: 1\nc:\n <<: *b\n <<: *b\n", + "scalar merge": "c:\n <<: text\n", + "scalar in list": "b: &b\n x: 1\nc:\n <<: [*b, text]\n", + } { + t.Run(name, func(t *testing.T) { + var document yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &document)) + _, err := Build([]byte(source), document.Content[0], 0, false) + require.Error(t, err) + }) + } +}