feat: vendor add end-to-end with deterministic lockfile v2
- go-git in-process clone, default-branch/tag resolution, optional token auth - web-tree URL normalization (github/gitlab) with implied discovery root - discovery of skills/commands/agents/mcp, lenient frontmatter with warnings - selection: all by default, --include with missing-entry hard errors - external/<source>/<type>/ materialization preserving exec bits - lockfile v2: deterministic JSON, pinned url/ref/rev, inventory, warnings - offline black-box e2e suite against local fixture git repositories
This commit is contained in:
@@ -0,0 +1,171 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/m3tam3re/agent-lib/internal/lockfile"
|
||||
)
|
||||
|
||||
// Tree abstracts a readable content tree: a git commit tree or a filesystem
|
||||
// root. Paths are slash-separated and relative to the tree root.
|
||||
type Tree interface {
|
||||
Walk(fn func(rel string, isDir bool) error) error
|
||||
ReadFile(rel string) ([]byte, error)
|
||||
}
|
||||
|
||||
// Config directs Scan; zero values select the standard layout.
|
||||
type Config struct {
|
||||
Root string
|
||||
SkillsDir string
|
||||
CommandsDir string
|
||||
AgentsDir string
|
||||
McpDir string
|
||||
}
|
||||
|
||||
func FromLockfileDiscovery(d lockfile.Discovery) Config {
|
||||
r := d.Resolve()
|
||||
return Config{Root: r.Root, SkillsDir: r.SkillsDir, CommandsDir: r.CommandsDir, AgentsDir: r.AgentsDir, McpDir: r.McpDir}
|
||||
}
|
||||
|
||||
// Item is one discovered deployable artifact.
|
||||
type Item struct {
|
||||
Type string
|
||||
UpstreamID string
|
||||
RelPath string
|
||||
Frontmatter *Frontmatter
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
// Scan discovers items of all four content types under cfg. Discovery is a
|
||||
// read-only scan; it never evaluates content beyond reading files.
|
||||
func Scan(tree Tree, cfg Config) ([]Item, error) {
|
||||
dirs := lockfile.Discovery{
|
||||
Root: cfg.Root, SkillsDir: cfg.SkillsDir, CommandsDir: cfg.CommandsDir,
|
||||
AgentsDir: cfg.AgentsDir, McpDir: cfg.McpDir,
|
||||
}.Resolve()
|
||||
|
||||
skillRoot := join(dirs.Root, dirs.SkillsDir)
|
||||
commandRoot := join(dirs.Root, dirs.CommandsDir)
|
||||
agentRoot := join(dirs.Root, dirs.AgentsDir)
|
||||
mcpRoot := join(dirs.Root, dirs.McpDir)
|
||||
|
||||
dirEntries := map[string]map[string]bool{}
|
||||
fileEntries := map[string]map[string]bool{}
|
||||
for _, d := range []string{skillRoot, commandRoot, agentRoot, mcpRoot} {
|
||||
dirEntries[d] = map[string]bool{}
|
||||
fileEntries[d] = map[string]bool{}
|
||||
}
|
||||
|
||||
err := tree.Walk(func(rel string, isDir bool) error {
|
||||
parent, base := splitPath(rel)
|
||||
if isDir {
|
||||
if set, ok := dirEntries[parent]; ok {
|
||||
set[base] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if set, ok := fileEntries[parent]; ok {
|
||||
set[base] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walking source tree: %w", err)
|
||||
}
|
||||
|
||||
var items []Item
|
||||
for name := range dirEntries[skillRoot] {
|
||||
skillMd := join(skillRoot, name, "SKILL.md")
|
||||
data, err := tree.ReadFile(skillMd)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
it := Item{Type: lockfile.TypeSkill, UpstreamID: name, RelPath: join(skillRoot, name)}
|
||||
fm, warnings := ParseFrontmatter(data)
|
||||
it.Frontmatter = &fm
|
||||
for _, w := range warnings {
|
||||
it.Warnings = append(it.Warnings, fmt.Sprintf("skills/%s: %s", name, w))
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
items = append(items, scanFlatFiles(tree, fileEntries, commandRoot, lockfile.TypeCommand)...)
|
||||
items = append(items, scanFlatFiles(tree, fileEntries, agentRoot, lockfile.TypeAgent)...)
|
||||
items = append(items, scanMcpFiles(tree, fileEntries, mcpRoot)...)
|
||||
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i].Type != items[j].Type {
|
||||
return typeRank(items[i].Type) < typeRank(items[j].Type)
|
||||
}
|
||||
return items[i].UpstreamID < items[j].UpstreamID
|
||||
})
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func scanFlatFiles(tree Tree, fileEntries map[string]map[string]bool, root, typ string) []Item {
|
||||
var items []Item
|
||||
for base := range fileEntries[root] {
|
||||
if !strings.HasSuffix(base, ".md") {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSuffix(base, ".md")
|
||||
rel := join(root, base)
|
||||
data, err := tree.ReadFile(rel)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
it := Item{Type: typ, UpstreamID: id, RelPath: rel}
|
||||
fm, warnings := ParseFrontmatter(data)
|
||||
it.Frontmatter = &fm
|
||||
for _, w := range warnings {
|
||||
it.Warnings = append(it.Warnings, fmt.Sprintf("%s/%s: %s", typ, id, w))
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func scanMcpFiles(tree Tree, fileEntries map[string]map[string]bool, root string) []Item {
|
||||
var items []Item
|
||||
for base := range fileEntries[root] {
|
||||
var id string
|
||||
switch {
|
||||
case strings.HasSuffix(base, ".yaml"):
|
||||
id = strings.TrimSuffix(base, ".yaml")
|
||||
case strings.HasSuffix(base, ".yml"):
|
||||
id = strings.TrimSuffix(base, ".yml")
|
||||
default:
|
||||
continue
|
||||
}
|
||||
items = append(items, Item{Type: lockfile.TypeMcp, UpstreamID: id, RelPath: join(root, base)})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func typeRank(t string) int {
|
||||
for i, tt := range lockfile.Types {
|
||||
if tt == t {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return len(lockfile.Types)
|
||||
}
|
||||
|
||||
func join(parts ...string) string {
|
||||
var nonEmpty []string
|
||||
for _, p := range parts {
|
||||
if p != "" {
|
||||
nonEmpty = append(nonEmpty, p)
|
||||
}
|
||||
}
|
||||
return strings.Join(nonEmpty, "/")
|
||||
}
|
||||
|
||||
func splitPath(rel string) (parent, base string) {
|
||||
i := strings.LastIndex(rel, "/")
|
||||
if i < 0 {
|
||||
return "", rel
|
||||
}
|
||||
return rel[:i], rel[i+1:]
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Frontmatter holds the leniently parsed leading YAML block of an item.
|
||||
type Frontmatter struct {
|
||||
Name string
|
||||
Description string
|
||||
Tags []string
|
||||
Requires []string
|
||||
}
|
||||
|
||||
// ParseFrontmatter extracts name, description, tags and requires from a
|
||||
// leading `---` fenced block. Parsing is deliberately lenient: a block that
|
||||
// opens but never closes, or contains lines without a colon, yields warnings
|
||||
// instead of failures; the parseable prefix still wins. Files without a
|
||||
// leading fence have no frontmatter and no warning.
|
||||
func ParseFrontmatter(data []byte) (Frontmatter, []string) {
|
||||
var fm Frontmatter
|
||||
var warnings []string
|
||||
lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n")
|
||||
if len(lines) == 0 || strings.TrimSpace(lines[0]) != "---" {
|
||||
return fm, nil
|
||||
}
|
||||
closed := false
|
||||
for _, line := range lines[1:] {
|
||||
if strings.TrimSpace(line) == "---" {
|
||||
closed = true
|
||||
break
|
||||
}
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(trimmed, ":")
|
||||
if !ok {
|
||||
warnings = append(warnings, "malformed frontmatter line (missing colon): "+trimmed)
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.TrimSpace(value)
|
||||
switch strings.ToLower(key) {
|
||||
case "name":
|
||||
fm.Name = value
|
||||
case "description":
|
||||
fm.Description = value
|
||||
case "tags":
|
||||
fm.Tags = splitList(value)
|
||||
case "requires":
|
||||
fm.Requires = splitList(value)
|
||||
}
|
||||
}
|
||||
if !closed {
|
||||
warnings = append(warnings, "frontmatter block not closed")
|
||||
}
|
||||
return fm, warnings
|
||||
}
|
||||
|
||||
func splitList(v string) []string {
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(v, ",")
|
||||
var out []string
|
||||
for _, p := range parts {
|
||||
if t := strings.TrimSpace(p); t != "" {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package discovery
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseFrontmatter(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want Frontmatter
|
||||
warnLen int
|
||||
}{
|
||||
{
|
||||
name: "complete block",
|
||||
in: "---\nname: Test Skill\ndescription: does things\ntags: a, b\nrequires: c,d\n---\nbody\n",
|
||||
want: Frontmatter{Name: "Test Skill", Description: "does things", Tags: []string{"a", "b"}, Requires: []string{"c", "d"}},
|
||||
},
|
||||
{
|
||||
name: "no frontmatter",
|
||||
in: "# Just a file\nbody\n",
|
||||
want: Frontmatter{},
|
||||
},
|
||||
{
|
||||
name: "unclosed block",
|
||||
in: "---\nname: Broken\ndescription: never closed\n",
|
||||
want: Frontmatter{Name: "Broken", Description: "never closed"},
|
||||
warnLen: 1,
|
||||
},
|
||||
{
|
||||
name: "line missing colon",
|
||||
in: "---\nname: X\nbroken line without colon\n---\n",
|
||||
want: Frontmatter{Name: "X"},
|
||||
warnLen: 1,
|
||||
},
|
||||
{
|
||||
name: "crlf line endings",
|
||||
in: "---\r\nname: Win\r\ndescription: windows author\r\n---\r\n",
|
||||
want: Frontmatter{Name: "Win", Description: "windows author"},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm, warnings := ParseFrontmatter([]byte(tc.in))
|
||||
if fm.Name != tc.want.Name || fm.Description != tc.want.Description {
|
||||
t.Errorf("got %+v, want %+v", fm, tc.want)
|
||||
}
|
||||
if len(fm.Tags) != len(tc.want.Tags) || len(fm.Requires) != len(tc.want.Requires) {
|
||||
t.Errorf("tags/requires mismatch: got %+v want %+v", fm, tc.want)
|
||||
}
|
||||
if len(warnings) != tc.warnLen {
|
||||
t.Errorf("warnings = %v (len %d), want len %d", warnings, len(warnings), tc.warnLen)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FsTree adapts a filesystem root to the Tree interface (used for scanning
|
||||
// the work repository's own items).
|
||||
type FsTree struct {
|
||||
Root string
|
||||
}
|
||||
|
||||
func (f *FsTree) Walk(fn func(rel string, isDir bool) error) error {
|
||||
var paths []string
|
||||
err := filepath.WalkDir(f.Root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p == f.Root {
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, p)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(paths)
|
||||
for _, p := range paths {
|
||||
rel, err := filepath.Rel(f.Root, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fn(filepath.ToSlash(rel), info.IsDir()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FsTree) ReadFile(rel string) ([]byte, error) {
|
||||
clean := filepath.FromSlash(rel)
|
||||
if strings.Contains(clean, "..") {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return os.ReadFile(filepath.Join(f.Root, clean))
|
||||
}
|
||||
Reference in New Issue
Block a user