feat: vendor add end-to-end with deterministic lockfile v2
- go-git in-process clone, default-branch/tag resolution, optional token auth - web-tree URL normalization (github/gitlab) with implied discovery root - discovery of skills/commands/agents/mcp, lenient frontmatter with warnings - selection: all by default, --include with missing-entry hard errors - external/<source>/<type>/ materialization preserving exec bits - lockfile v2: deterministic JSON, pinned url/ref/rev, inventory, warnings - offline black-box e2e suite against local fixture git repositories
This commit is contained in:
Vendored
+280
@@ -0,0 +1,280 @@
|
||||
// Package vendor implements the curator-side commands: add, update, diff,
|
||||
// remove, list and inspect over the external area and lockfile v2.
|
||||
package vendor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/m3tam3re/agent-lib/internal/discovery"
|
||||
"github.com/m3tam3re/agent-lib/internal/gitsource"
|
||||
"github.com/m3tam3re/agent-lib/internal/lockfile"
|
||||
)
|
||||
|
||||
const externalDir = "external"
|
||||
|
||||
// AddOptions parameterize vendor add.
|
||||
type AddOptions struct {
|
||||
Name string
|
||||
URL string
|
||||
Ref string
|
||||
Include []string
|
||||
}
|
||||
|
||||
// Add vendors a new source into the work repository at workDir: it clones,
|
||||
// discovers, applies the selection and materializes external/<name>/ plus the
|
||||
// lockfile entry. Nothing is written before every check has passed.
|
||||
func Add(workDir string, opts AddOptions, stdout io.Writer) error {
|
||||
lockPath := filepath.Join(workDir, lockfile.FileName)
|
||||
lf, err := lockfile.Load(lockPath)
|
||||
if err != nil {
|
||||
if err != lockfile.ErrNotExist {
|
||||
return err
|
||||
}
|
||||
lf = lockfile.New()
|
||||
}
|
||||
if _, exists := lf.Sources[opts.Name]; exists {
|
||||
return fmt.Errorf("source %q already exists in %s", opts.Name, lockfile.FileName)
|
||||
}
|
||||
extRoot := filepath.Join(workDir, externalDir, opts.Name)
|
||||
if _, err := os.Stat(extRoot); err == nil {
|
||||
return fmt.Errorf("%s exists but %q is not in the lockfile; remove the directory or pick another name", extRoot, opts.Name)
|
||||
}
|
||||
|
||||
normalized, root, err := gitsource.NormalizeURL(opts.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
repo, err := gitsource.Clone(normalized, opts.Ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tree, err := repo.Tree()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg := lockfile.Discovery{Root: root}
|
||||
items, err := discovery.Scan(tree, discovery.FromLockfileDiscovery(cfg))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sel := lockfile.Selection{Mode: lockfile.ModeAll}
|
||||
if len(opts.Include) > 0 {
|
||||
sel = lockfile.Selection{Mode: lockfile.ModeInclude, Include: opts.Include}
|
||||
if err := checkIncludeEntries(items, opts.Include, opts.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
selected := applySelection(items, sel)
|
||||
if len(selected) == 0 {
|
||||
return fmt.Errorf("source %q selected zero items; refusing to add an empty source", opts.Name)
|
||||
}
|
||||
|
||||
src := &lockfile.Source{
|
||||
URL: normalized,
|
||||
Ref: repo.Ref,
|
||||
Rev: repo.Rev,
|
||||
Selection: sel,
|
||||
Renames: map[string]string{},
|
||||
}
|
||||
if cfg != (lockfile.Discovery{}) {
|
||||
src.Discovery = &cfg
|
||||
}
|
||||
for _, it := range items {
|
||||
src.Warnings = append(src.Warnings, it.Warnings...)
|
||||
}
|
||||
src.Warnings = dedupe(src.Warnings)
|
||||
|
||||
inventory, err := materialize(workDir, opts.Name, tree, selected, src.Renames)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
src.Inventory = inventory
|
||||
|
||||
lf.Sources[opts.Name] = src
|
||||
if err := lf.Save(lockPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reportAdd(stdout, opts.Name, src, items, selected)
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkIncludeEntries(items []discovery.Item, include []string, name string) error {
|
||||
available := map[string]bool{}
|
||||
for _, it := range items {
|
||||
available[it.UpstreamID] = true
|
||||
}
|
||||
for _, want := range include {
|
||||
if !available[want] {
|
||||
return fmt.Errorf("source %q: include entry %q not found upstream (available: %s)", name, want, strings.Join(sortedKeys(available), ", "))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func applySelection(items []discovery.Item, sel lockfile.Selection) []discovery.Item {
|
||||
exclude := map[string]bool{}
|
||||
for _, e := range sel.Exclude {
|
||||
exclude[e] = true
|
||||
}
|
||||
include := map[string]bool{}
|
||||
for _, i := range sel.Include {
|
||||
include[i] = true
|
||||
}
|
||||
var out []discovery.Item
|
||||
for _, it := range items {
|
||||
switch sel.Mode {
|
||||
case lockfile.ModeInclude:
|
||||
if include[it.UpstreamID] {
|
||||
out = append(out, it)
|
||||
}
|
||||
default:
|
||||
if !exclude[it.UpstreamID] {
|
||||
out = append(out, it)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func deployedName(upstreamID string, renames map[string]string) string {
|
||||
if d, ok := renames[upstreamID]; ok && d != "" {
|
||||
return d
|
||||
}
|
||||
return upstreamID
|
||||
}
|
||||
|
||||
func materialize(workDir, name string, tree discovery.Tree, selected []discovery.Item, renames map[string]string) (map[string][]string, error) {
|
||||
inventory := map[string][]string{}
|
||||
for _, it := range selected {
|
||||
dep := deployedName(it.UpstreamID, renames)
|
||||
base := filepath.Join(workDir, externalDir, name, it.Type)
|
||||
var dst string
|
||||
var err error
|
||||
switch it.Type {
|
||||
case lockfile.TypeSkill:
|
||||
dst = filepath.Join(base, dep)
|
||||
err = copyTreeDir(tree, it.RelPath, dst)
|
||||
default:
|
||||
dst = filepath.Join(base, dep+filepath.Ext(it.RelPath))
|
||||
err = copyTreeFile(tree, it.RelPath, dst)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("materializing %s/%s: %w", it.Type, it.UpstreamID, err)
|
||||
}
|
||||
inventory[it.Type] = append(inventory[it.Type], it.UpstreamID)
|
||||
}
|
||||
return inventory, nil
|
||||
}
|
||||
|
||||
func copyTreeDir(tree discovery.Tree, srcRel, dst string) error {
|
||||
lister, ok := tree.(interface {
|
||||
FilesUnder(prefix string) ([]string, error)
|
||||
})
|
||||
if !ok {
|
||||
return fmt.Errorf("tree does not support directory copy")
|
||||
}
|
||||
files, err := lister.FilesUnder(srcRel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("no files under %s", srcRel)
|
||||
}
|
||||
prefix := strings.TrimSuffix(srcRel, "/") + "/"
|
||||
for _, f := range files {
|
||||
relInside := strings.TrimPrefix(f, prefix)
|
||||
target := filepath.Join(dst, filepath.FromSlash(relInside))
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeFileFromTree(tree, f, target); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyTreeFile(tree discovery.Tree, srcRel, dst string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFileFromTree(tree, srcRel, dst)
|
||||
}
|
||||
|
||||
func writeFileFromTree(tree discovery.Tree, rel, dst string) error {
|
||||
if mt, ok := tree.(interface {
|
||||
MaterializeFile(rel, dst string) error
|
||||
}); ok {
|
||||
return mt.MaterializeFile(rel, dst)
|
||||
}
|
||||
data, err := tree.ReadFile(rel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(dst, data, 0o644)
|
||||
}
|
||||
|
||||
func reportAdd(w io.Writer, name string, src *lockfile.Source, discovered, selected []discovery.Item) {
|
||||
counts := func(items []discovery.Item) string {
|
||||
per := map[string]int{}
|
||||
for _, it := range items {
|
||||
per[it.Type]++
|
||||
}
|
||||
var parts []string
|
||||
for _, t := range lockfile.Types {
|
||||
if per[t] > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d %s", per[t], t))
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "0 items"
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
fmt.Fprintf(w, "source %s pinned\n", name)
|
||||
fmt.Fprintf(w, " url: %s\n", src.URL)
|
||||
fmt.Fprintf(w, " ref: %s\n", src.Ref)
|
||||
fmt.Fprintf(w, " rev: %s\n", src.Rev)
|
||||
fmt.Fprintf(w, "discovered %d items (%s)\n", len(discovered), counts(discovered))
|
||||
fmt.Fprintf(w, "selected %d items (%s)\n", len(selected), counts(selected))
|
||||
fmt.Fprintf(w, "materialized under %s/\n", filepath.Join(externalDir, name))
|
||||
if len(src.Warnings) > 0 {
|
||||
fmt.Fprintf(w, "warnings:\n")
|
||||
for _, warn := range src.Warnings {
|
||||
fmt.Fprintf(w, " - %s\n", warn)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(w, "lockfile %s written\n", lockfile.FileName)
|
||||
}
|
||||
|
||||
func dedupe(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, s := range in {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user