From 49dbcf469e9c5489f2b07d0a1d64a2db0f1663f4 Mon Sep 17 00:00:00 2001 From: m3ta-chiron Date: Sun, 23 Aug 2026 10:31:00 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20client=20sync=20core=20=E2=80=94=20conf?= =?UTF-8?q?ig,=20pull,=20inventory,=20manifest,=20OpenCode=20target?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - admin-protected client config (flag > AGENT_LIB_CONFIG > platform default), token used for basic auth only, redacted from all errors and files - work-repo pull via go-git (bare cache clone + fetch, remote re-pointed on config change), clean failure keeps previous state - inventory: own items + external areas from lockfile; MCP inventoried but never deployed - manifest v1 records name/type/origin/revision/content-hash per item, deterministic bytes; folder hashes over sorted relpath+filehash lines so tree and disk hashes agree - shared deploy package (atomic temp+rename, exec-bit preserving) now backs both curator materialization and client deployment - sync validates and hashes everything before the first mutation; second run is a byte-level no-op --- .beads/interactions.jsonl | 1 + internal/cli/root.go | 1 + internal/cli/sync.go | 30 ++++ internal/client/config.go | 111 ++++++++++++++ internal/client/inventory.go | 93 ++++++++++++ internal/client/manifest.go | 157 +++++++++++++++++++ internal/client/pull.go | 139 +++++++++++++++++ internal/client/sync.go | 172 +++++++++++++++++++++ internal/deploy/deploy.go | 79 ++++++++++ internal/discovery/prefix.go | 66 ++++++++ internal/e2e/sync_test.go | 279 ++++++++++++++++++++++++++++++++++ internal/gitsource/git.go | 5 + internal/gitsource/redact.go | 16 ++ internal/lockfile/lockfile.go | 15 ++ internal/target/opencode.go | 41 +++++ internal/vendor/add.go | 53 +------ internal/vendor/update.go | 5 +- 17 files changed, 1211 insertions(+), 52 deletions(-) create mode 100644 internal/cli/sync.go create mode 100644 internal/client/config.go create mode 100644 internal/client/inventory.go create mode 100644 internal/client/manifest.go create mode 100644 internal/client/pull.go create mode 100644 internal/client/sync.go create mode 100644 internal/deploy/deploy.go create mode 100644 internal/discovery/prefix.go create mode 100644 internal/e2e/sync_test.go create mode 100644 internal/gitsource/redact.go create mode 100644 internal/target/opencode.go diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index 89a38b8..6ffcdc2 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -15,3 +15,4 @@ {"id":"int-090714c2a2d985183cc965b11b5c53d8","kind":"field_change","created_at":"2026-08-22T19:52:30.546382863Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.10","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} {"id":"int-c5834aa5e95b421826d076fd129d7a1a","kind":"field_change","created_at":"2026-08-22T19:54:32.168882535Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.2","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} {"id":"int-b7efd31fd41bbb5acd082eb3c2f34dbd","kind":"field_change","created_at":"2026-08-22T19:56:21.734777976Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.3","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} +{"id":"int-d0f6b2e3946129f47ed8f071902a5c78","kind":"field_change","created_at":"2026-08-22T20:01:15.960626197Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.4","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} diff --git a/internal/cli/root.go b/internal/cli/root.go index c4c4084..1d1cf32 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -23,6 +23,7 @@ func NewRootCmd() *cobra.Command { root.AddCommand(newVersionCmd()) root.AddCommand(newVendorCmd()) root.AddCommand(newValidateCmd()) + root.AddCommand(newSyncCmd()) return root } diff --git a/internal/cli/sync.go b/internal/cli/sync.go new file mode 100644 index 0000000..9b830af --- /dev/null +++ b/internal/cli/sync.go @@ -0,0 +1,30 @@ +package cli + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/m3tam3re/agent-lib/internal/client" +) + +func newSyncCmd() *cobra.Command { + var configPath string + cmd := &cobra.Command{ + Use: "sync", + Short: "Pull the work repository and deploy its content to OpenCode", + Long: "Pulls the fleet-configured work repository (config from --config, " + + "AGENT_LIB_CONFIG, or the admin-protected platform default) and deploys " + + "skills, commands and agents into the OpenCode runtime paths. " + + "Idempotent: re-running changes nothing when content and disk match.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if _, err := client.Sync(client.SyncOptions{ConfigPath: configPath}, cmd.OutOrStdout()); err != nil { + return fmt.Errorf("sync: %w", err) + } + return nil + }, + } + cmd.Flags().StringVar(&configPath, "config", "", "client config path (default: "+client.ConfigEnv+" env or platform default)") + return cmd +} diff --git a/internal/client/config.go b/internal/client/config.go new file mode 100644 index 0000000..3402b31 --- /dev/null +++ b/internal/client/config.go @@ -0,0 +1,111 @@ +// Package client implements the employee side: config loading, work-repo +// pull, inventory, manifest and deployment into the OpenCode target. +package client + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" +) + +// ClientConfig is the admin-protected fleet configuration for sync. +type ClientConfig struct { + RepoURL string `json:"repo_url"` + Ref string `json:"ref,omitempty"` + Token string `json:"token,omitempty"` + TokenUser string `json:"token_user,omitempty"` +} + +// ConfigEnv names the override environment variables. +const ( + ConfigEnv = "AGENT_LIB_CONFIG" + StateDirEnv = "AGENT_LIB_STATE_DIR" + defaultConfig = `{ + "repo_url": "", + "ref": "", + "token": "", + "token_user": "" +}` +) + +// DefaultConfigPath returns the admin-protected config location per platform: +// ProgramData on Windows, /etc on Linux/macOS. +func DefaultConfigPath() string { + if runtime.GOOS == "windows" { + if pd := os.Getenv("ProgramData"); pd != "" { + return filepath.Join(pd, "agent-lib", "config.json") + } + return `C:\ProgramData\agent-lib\config.json` + } + return "/etc/agent-lib/config.json" +} + +// LoadClientConfig reads the config from flagPath, the override env, or the +// platform default — in that order. +func LoadClientConfig(flagPath string) (*ClientConfig, error) { + path := flagPath + if path == "" { + path = os.Getenv(ConfigEnv) + } + if path == "" { + path = DefaultConfigPath() + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading client config %s: %w", path, err) + } + var cfg ClientConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("client config %s is not valid JSON: %w", path, err) + } + if cfg.RepoURL == "" { + return nil, fmt.Errorf("client config %s: repo_url is required", path) + } + return &cfg, nil +} + +// StateDir returns the client state directory: the override env, then the +// platform default (LocalAppData on Windows, XDG state elsewhere). +func StateDir() (string, error) { + if d := os.Getenv(StateDirEnv); d != "" { + return d, nil + } + if runtime.GOOS == "windows" { + base := os.Getenv("LocalAppData") + if base == "" { + return "", fmt.Errorf("LocalAppData is not set") + } + return filepath.Join(base, "agent-lib"), nil + } + if xdg := os.Getenv("XDG_STATE_HOME"); xdg != "" { + return filepath.Join(xdg, "agent-lib"), nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".local", "state", "agent-lib"), nil +} + +// StatePaths bundles the well-known files inside the state dir. +type StatePaths struct { + Root string + CacheDir string + Manifest string + LogFile string +} + +func ResolveStatePaths() (StatePaths, error) { + root, err := StateDir() + if err != nil { + return StatePaths{}, err + } + return StatePaths{ + Root: root, + CacheDir: filepath.Join(root, "work-repo"), + Manifest: filepath.Join(root, "manifest.json"), + LogFile: filepath.Join(root, "sync.log"), + }, nil +} diff --git a/internal/client/inventory.go b/internal/client/inventory.go new file mode 100644 index 0000000..949f38e --- /dev/null +++ b/internal/client/inventory.go @@ -0,0 +1,93 @@ +package client + +import ( + "fmt" + "sort" + + "github.com/m3tam3re/agent-lib/internal/discovery" + "github.com/m3tam3re/agent-lib/internal/gitsource" + "github.com/m3tam3re/agent-lib/internal/lockfile" +) + +// InventoryItem is one deployable item found in the work repository. +type InventoryItem struct { + Type string + Name string + Origin string + RelPath string + Hash string + Warnings []string +} + +// Inventory is the full content of a work-repo revision: own contributions +// plus every external area. MCP fragments are inventoried but filtered at +// deployment time. +type Inventory struct { + Revision string + Items []InventoryItem +} + +// BuildInventory reads the work repository tree and the lockfile (when +// present) and resolves the flat deployed namespace. +func BuildInventory(tree *gitsource.GitTree, rev string) (*Inventory, error) { + inv := &Inventory{Revision: rev} + + own, err := discovery.Scan(tree, discovery.Config{}) + if err != nil { + return nil, fmt.Errorf("scanning own items: %w", err) + } + for _, it := range own { + inv.Items = append(inv.Items, InventoryItem{ + Type: it.Type, Name: it.UpstreamID, Origin: "own", RelPath: it.RelPath, + }) + } + + var lf *lockfile.Lockfile + if data, err := tree.ReadFile(lockfile.FileName); err == nil { + lf, err = lockfile.ParseBytes(data) + if err != nil { + return nil, fmt.Errorf("work repository lockfile is invalid: %w", err) + } + } else { + lf = lockfile.New() + } + + sources := make([]string, 0, len(lf.Sources)) + for name := range lf.Sources { + sources = append(sources, name) + } + sort.Strings(sources) + for _, name := range sources { + sub := &discovery.PrefixTree{Tree: tree, Prefix: "external/" + name} + items, err := discovery.Scan(sub, discovery.Config{}) + if err != nil { + return nil, fmt.Errorf("scanning external area of %q: %w", name, err) + } + for _, it := range items { + inv.Items = append(inv.Items, InventoryItem{ + Type: it.Type, Name: it.UpstreamID, Origin: name, RelPath: "external/" + name + "/" + it.RelPath, + Warnings: it.Warnings, + }) + } + } + + sort.Slice(inv.Items, func(i, j int) bool { + if inv.Items[i].Type != inv.Items[j].Type { + return inv.Items[i].Type < inv.Items[j].Type + } + return inv.Items[i].Name < inv.Items[j].Name + }) + return inv, nil +} + +// Deployable filters the inventory down to the types the target deploys. +func (inv *Inventory) Deployable() []InventoryItem { + var out []InventoryItem + for _, it := range inv.Items { + if it.Type == lockfile.TypeMcp { + continue + } + out = append(out, it) + } + return out +} diff --git a/internal/client/manifest.go b/internal/client/manifest.go new file mode 100644 index 0000000..6e38e57 --- /dev/null +++ b/internal/client/manifest.go @@ -0,0 +1,157 @@ +package client + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/m3tam3re/agent-lib/internal/gitsource" + "github.com/m3tam3re/agent-lib/internal/lockfile" +) + +// ManifestVersion is the local manifest schema version. +const ManifestVersion = 1 + +// ManifestItem records one deployed item: name, type, origin, the work-repo +// revision it came from, and the content hash at deploy time. +type ManifestItem struct { + Name string `json:"name"` + Type string `json:"type"` + Origin string `json:"origin"` + Revision string `json:"revision"` + Hash string `json:"hash"` +} + +// Manifest is the local record of everything agent-lib deployed. +type Manifest struct { + Version int `json:"version"` + Revision string `json:"revision"` + Items map[string]*ManifestItem `json:"items"` +} + +// NewManifest returns an empty manifest v1. +func NewManifest() *Manifest { + return &Manifest{Version: ManifestVersion, Items: map[string]*ManifestItem{}} +} + +// LoadManifest reads the manifest at path; a missing file yields an empty +// manifest, not an error (first sync). +func LoadManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return NewManifest(), nil + } + return nil, err + } + var m Manifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("manifest %s is not valid JSON: %w", path, err) + } + if m.Items == nil { + m.Items = map[string]*ManifestItem{} + } + return &m, nil +} + +// Save writes the manifest deterministically (sorted item keys, two-space +// indent, trailing newline). +func (m *Manifest) Save(path string) error { + data, err := marshalManifest(m) + if err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + +func marshalManifest(m *Manifest) ([]byte, error) { + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return nil, err + } + return append(data, '\n'), nil +} + +// Key identifies an item in the manifest: "/". +func manifestKey(typ, name string) string { + return typ + "/" + name +} + +// HashItem computes the content hash of an inventory item from its git tree: +// single files hash their bytes; folders hash "relpath 0x00 hash" lines over +// all contained files, sorted. +func HashItem(tree *gitsource.GitTree, item InventoryItem) (string, error) { + if item.Type != lockfile.TypeSkill { + data, err := tree.ReadFile(item.RelPath) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]), nil + } + files, err := tree.FilesUnder(item.RelPath) + if err != nil { + return "", err + } + prefix := strings.TrimSuffix(item.RelPath, "/") + "/" + h := sha256.New() + for _, f := range files { + data, err := tree.ReadFile(f) + if err != nil { + return "", err + } + fsum := sha256.Sum256(data) + fmt.Fprintf(h, "%s\x00%s\n", strings.TrimPrefix(f, prefix), hex.EncodeToString(fsum[:])) + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +// HashDiskPath computes the same content hash for a deployed path on disk. +func HashDiskPath(typ, path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", err + } + if !info.IsDir() { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]), nil + } + var lines []string + err = filepath.WalkDir(path, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(path, p) + if err != nil { + return err + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + fsum := sha256.Sum256(data) + lines = append(lines, fmt.Sprintf("%s\x00%s", filepath.ToSlash(rel), hex.EncodeToString(fsum[:]))) + return nil + }) + if err != nil { + return "", err + } + sort.Strings(lines) + h := sha256.New() + for _, l := range lines { + fmt.Fprintf(h, "%s\n", l) + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/client/pull.go b/internal/client/pull.go new file mode 100644 index 0000000..202534c --- /dev/null +++ b/internal/client/pull.go @@ -0,0 +1,139 @@ +package client + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + gittransport "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + + "github.com/m3tam3re/agent-lib/internal/gitsource" +) + +// PullResult is one successful work-repo pull: the resolved revision plus +// read access to its commit tree. +type PullResult struct { + Rev string + Tree *gitsource.GitTree +} + +// Pull fetches the work repository into the cache dir (bare clone, then +// fetch) and resolves ref — the default branch when empty. An unreachable +// repository yields a clean error before any deployment state is touched. +func Pull(cacheDir, url, ref, token, tokenUser string) (*PullResult, error) { + auth := basicAuth(token, tokenUser) + repo, err := openOrCreate(cacheDir, url, ref, auth) + if err != nil { + return nil, err + } + hash, err := resolveRemoteRef(repo, ref) + if err != nil { + return nil, fmt.Errorf("resolving %q in %s: %w", refLabel(ref), gitsource.RedactURL(url), err) + } + commit, err := repo.CommitObject(hash) + if err != nil { + return nil, fmt.Errorf("reading commit %s: %w", hash, err) + } + tree, err := commit.Tree() + if err != nil { + return nil, fmt.Errorf("reading tree of %s: %w", hash, err) + } + return &PullResult{Rev: hash.String(), Tree: gitsource.NewGitTree(tree)}, nil +} + +func basicAuth(token, tokenUser string) gittransport.AuthMethod { + if token == "" { + return nil + } + user := tokenUser + if user == "" { + user = "agent-lib" + } + return &http.BasicAuth{Username: user, Password: token} +} + +func openOrCreate(cacheDir, url, ref string, auth gittransport.AuthMethod) (*git.Repository, error) { + if _, err := os.Stat(filepath.Join(cacheDir, "HEAD")); err == nil { + repo, err := git.PlainOpen(cacheDir) + if err != nil { + return nil, fmt.Errorf("opening cached work repo: %w", err) + } + if err := repointRemote(repo, url); err != nil { + return nil, err + } + if err := repo.Fetch(&git.FetchOptions{Auth: auth}); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return nil, fmt.Errorf("fetching %s: %w", gitsource.RedactURL(url), err) + } + return repo, nil + } + if err := os.MkdirAll(filepath.Dir(cacheDir), 0o755); err != nil { + return nil, err + } + opts := &git.CloneOptions{URL: url, Auth: auth, NoCheckout: true} + if ref != "" { + opts.ReferenceName = plumbing.NewBranchReferenceName(ref) + } + repo, err := git.PlainClone(cacheDir, true, opts) + if err != nil { + os.RemoveAll(cacheDir) + return nil, fmt.Errorf("cloning %s: %w", gitsource.RedactURL(url), err) + } + return repo, nil +} + +// repointRemote aligns the cached origin with the configured URL so a fleet +// re-point takes effect on the next sync. +func repointRemote(repo *git.Repository, url string) error { + remote, err := repo.Remote(git.DefaultRemoteName) + if err != nil { + return fmt.Errorf("reading cached remote: %w", err) + } + if len(remote.Config().URLs) == 0 || remote.Config().URLs[0] == url { + return nil + } + if err := repo.DeleteRemote(git.DefaultRemoteName); err != nil { + return fmt.Errorf("replacing cached remote: %w", err) + } + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: git.DefaultRemoteName, + URLs: []string{url}, + Fetch: []config.RefSpec{ + config.RefSpec("+refs/heads/*:refs/remotes/origin/*"), + }, + }) + return err +} + +func resolveRemoteRef(repo *git.Repository, ref string) (plumbing.Hash, error) { + candidates := remoteRefCandidates(ref) + for _, name := range candidates { + if h, err := repo.ResolveRevision(plumbing.Revision(name)); err == nil { + return *h, nil + } + } + return plumbing.ZeroHash, fmt.Errorf("reference not found (tried %v)", candidates) +} + +func remoteRefCandidates(ref string) []string { + if ref == "" { + return []string{"refs/remotes/origin/HEAD", "HEAD"} + } + return []string{ + "refs/remotes/origin/" + ref, + "refs/tags/" + ref, + "refs/heads/" + ref, + ref, + } +} + +func refLabel(ref string) string { + if ref == "" { + return "default branch" + } + return ref +} diff --git a/internal/client/sync.go b/internal/client/sync.go new file mode 100644 index 0000000..9a00266 --- /dev/null +++ b/internal/client/sync.go @@ -0,0 +1,172 @@ +package client + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/m3tam3re/agent-lib/internal/deploy" + "github.com/m3tam3re/agent-lib/internal/gitsource" + "github.com/m3tam3re/agent-lib/internal/lockfile" + "github.com/m3tam3re/agent-lib/internal/target" +) + +// SyncOptions parameterize a client sync run. +type SyncOptions struct { + ConfigPath string +} + +// SyncReport summarizes one sync run. +type SyncReport struct { + Revision string + Deployed int + Updated int + Kept int + Manifest string +} + +// Sync pulls the work repository and deploys its content into the OpenCode +// target. Everything is validated and hashed before the first mutation; +// unreachable repositories fail cleanly with the previous state intact. +func Sync(opts SyncOptions, stdout io.Writer) (*SyncReport, error) { + cfg, err := LoadClientConfig(opts.ConfigPath) + if err != nil { + return nil, err + } + paths, err := ResolveStatePaths() + if err != nil { + return nil, err + } + if err := os.MkdirAll(paths.Root, 0o755); err != nil { + return nil, err + } + + pull, err := Pull(paths.CacheDir, cfg.RepoURL, cfg.Ref, cfg.Token, cfg.TokenUser) + if err != nil { + return nil, err + } + inv, err := BuildInventory(pull.Tree, pull.Rev) + if err != nil { + return nil, err + } + oc, err := target.NewOpenCode() + if err != nil { + return nil, err + } + manifest, err := LoadManifest(paths.Manifest) + if err != nil { + return nil, err + } + + type prepared struct { + item InventoryItem + hash string + dst string + } + var plan []prepared + for _, it := range inv.Deployable() { + dst, ok := oc.DeployPath(it.Type, it.Name) + if !ok { + continue + } + hash, err := HashItem(pull.Tree, it) + if err != nil { + return nil, fmt.Errorf("hashing %s/%s: %w", it.Type, it.Name, err) + } + if !safePath(dst) { + return nil, fmt.Errorf("refusing unsafe deploy path %q", dst) + } + plan = append(plan, prepared{item: it, hash: hash, dst: dst}) + } + + report := &SyncReport{Revision: pull.Rev, Manifest: paths.Manifest} + for _, p := range plan { + key := manifestKey(p.item.Type, p.item.Name) + record := manifest.Items[key] + if record != nil && record.Hash == p.hash { + if diskHash, err := HashDiskPath(p.item.Type, p.dst); err == nil && diskHash == p.hash { + report.Kept++ + continue + } + } + if err := deployItem(pull.Tree, p.item, p.dst); err != nil { + return nil, fmt.Errorf("deploying %s/%s: %w", p.item.Type, p.item.Name, err) + } + manifest.Items[key] = &ManifestItem{ + Name: p.item.Name, Type: p.item.Type, Origin: p.item.Origin, + Revision: pull.Rev, Hash: p.hash, + } + if record == nil { + report.Deployed++ + } else { + report.Updated++ + } + } + + manifest.Version = ManifestVersion + manifest.Revision = pull.Rev + unchanged, err := manifestUnchanged(manifest, paths.Manifest) + if err != nil { + return nil, err + } + if !unchanged { + if err := manifest.Save(paths.Manifest); err != nil { + return nil, err + } + } + + printSyncReport(stdout, report, len(inv.Items)) + return report, nil +} + +func deployItem(tree *gitsource.GitTree, it InventoryItem, dst string) error { + if it.Type == lockfile.TypeSkill { + return deploy.Dir(tree, it.RelPath, dst) + } + return deploy.File(tree, it.RelPath, dst) +} + +// safePath rejects destinations that escape the user's home directory. +func safePath(dst string) bool { + home, err := os.UserHomeDir() + if err != nil { + return false + } + rel, err := filepath.Rel(home, dst) + if err != nil { + return false + } + return rel != ".." && !filepath.IsAbs(rel) +} + +func manifestUnchanged(m *Manifest, path string) (bool, error) { + existing, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + fresh, err := marshalManifest(m) + if err != nil { + return false, err + } + return bytes.Equal(existing, fresh), nil +} + +func printSyncReport(w io.Writer, r *SyncReport, totalItems int) { + fmt.Fprintf(w, "synced to revision %s\n", shortRev(r.Revision)) + fmt.Fprintf(w, " deployed: %d\n", r.Deployed) + fmt.Fprintf(w, " updated: %d\n", r.Updated) + fmt.Fprintf(w, " kept: %d\n", r.Kept) + fmt.Fprintf(w, " manifest: %s\n", r.Manifest) +} + +func shortRev(rev string) string { + if len(rev) > 12 { + return rev[:12] + } + return rev +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go new file mode 100644 index 0000000..d2b5650 --- /dev/null +++ b/internal/deploy/deploy.go @@ -0,0 +1,79 @@ +// Package deploy copies items from a discovery.Tree onto the filesystem, +// preserving executable bits and writing atomically (temp + rename). +package deploy + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/m3tam3re/agent-lib/internal/discovery" +) + +// Dir copies a folder item (e.g. one skill) from the tree into dst, +// replacing whatever occupied dst before. +func Dir(tree discovery.Tree, srcRel, dst string) error { + lister, ok := tree.(interface { + FilesUnder(prefix string) ([]string, error) + }) + if !ok { + return fmt.Errorf("tree does not support directory copy") + } + files, err := lister.FilesUnder(srcRel) + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("no files under %s", srcRel) + } + parent := filepath.Dir(dst) + if err := os.MkdirAll(parent, 0o755); err != nil { + return err + } + tmp, err := os.MkdirTemp(parent, ".agent-lib-tmp-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + prefix := strings.TrimSuffix(srcRel, "/") + "/" + for _, f := range files { + relInside := strings.TrimPrefix(f, prefix) + target := filepath.Join(tmp, filepath.FromSlash(relInside)) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + if err := File(tree, f, target); err != nil { + return err + } + } + if err := os.RemoveAll(dst); err != nil { + return err + } + return os.Rename(tmp, dst) +} + +// File copies a single file from the tree to dst via temp file + rename. +func File(tree discovery.Tree, srcRel, dst string) error { + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + if mt, ok := tree.(interface { + MaterializeFile(rel, dst string) error + }); ok { + tmp := dst + ".agent-lib-tmp" + if err := mt.MaterializeFile(srcRel, tmp); err != nil { + return err + } + return os.Rename(tmp, dst) + } + data, err := tree.ReadFile(srcRel) + if err != nil { + return err + } + tmp := dst + ".agent-lib-tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, dst) +} diff --git a/internal/discovery/prefix.go b/internal/discovery/prefix.go new file mode 100644 index 0000000..239e861 --- /dev/null +++ b/internal/discovery/prefix.go @@ -0,0 +1,66 @@ +package discovery + +import ( + "os" + "strings" +) + +// PrefixTree narrows a Tree to the subtree below Prefix; paths passed in and +// reported back are relative to that subtree. +type PrefixTree struct { + Tree Tree + Prefix string +} + +func (p *PrefixTree) Walk(fn func(rel string, isDir bool) error) error { + prefix := strings.TrimSuffix(p.Prefix, "/") + "/" + return p.Tree.Walk(func(rel string, isDir bool) error { + if rel == p.Prefix || strings.HasPrefix(rel, prefix) { + sub := strings.TrimPrefix(rel, prefix) + if sub == "" { + return nil + } + return fn(sub, isDir) + } + return nil + }) +} + +func (p *PrefixTree) ReadFile(rel string) ([]byte, error) { + return p.Tree.ReadFile(p.Prefix + "/" + rel) +} + +// FilesUnder delegates to the underlying tree with the prefix reapplied. +func (p *PrefixTree) FilesUnder(prefix string) ([]string, error) { + lister, ok := p.Tree.(interface { + FilesUnder(prefix string) ([]string, error) + }) + if !ok { + return nil, os.ErrNotExist + } + full := prefix + if p.Prefix != "" { + full = p.Prefix + "/" + strings.TrimSuffix(prefix, "/") + } + files, err := lister.FilesUnder(full) + if err != nil { + return nil, err + } + trim := p.Prefix + "/" + out := make([]string, 0, len(files)) + for _, f := range files { + out = append(out, strings.TrimPrefix(f, trim)) + } + return out, nil +} + +// MaterializeFile delegates to the underlying tree with the prefix reapplied. +func (p *PrefixTree) MaterializeFile(rel, dst string) error { + mt, ok := p.Tree.(interface { + MaterializeFile(rel, dst string) error + }) + if !ok { + return os.ErrNotExist + } + return mt.MaterializeFile(p.Prefix+"/"+rel, dst) +} diff --git a/internal/e2e/sync_test.go b/internal/e2e/sync_test.go new file mode 100644 index 0000000..fed9b5a --- /dev/null +++ b/internal/e2e/sync_test.go @@ -0,0 +1,279 @@ +package e2e + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func agentLibCmd(args ...string) *exec.Cmd { + return exec.Command(binPath, args...) +} + +type syncEnv struct { + home string + stateDir string + configDir string + config string + workRepo string +} + +func newSyncEnv(t *testing.T, h *harness) *syncEnv { + t.Helper() + base := t.TempDir() + env := &syncEnv{ + home: filepath.Join(base, "home"), + stateDir: filepath.Join(base, "state"), + configDir: filepath.Join(base, "config"), + workRepo: filepath.Join(base, "work-repo"), + } + for _, d := range []string{env.home, env.stateDir, env.configDir, env.workRepo} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + return env +} + +func (e *syncEnv) writeConfig(t *testing.T, repoURL, token string) { + t.Helper() + cfg := map[string]string{"repo_url": repoURL, "ref": "main"} + if token != "" { + cfg["token"] = token + } + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + e.config = filepath.Join(e.configDir, "config.json") + os.WriteFile(e.config, data, 0o600) +} + +func (e *syncEnv) run(t *testing.T, args ...string) (string, error) { + t.Helper() + cmd := agentLibCmd(append([]string{"sync", "--config", e.config}, args...)...) + cmd.Env = append(os.Environ(), + "HOME="+e.home, "USERPROFILE="+e.home, + "AGENT_LIB_STATE_DIR="+e.stateDir, + ) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func buildWorkRepo(t *testing.T, h *harness, dir string) { + t.Helper() + h.git(t, dir, "init", "-b", "main") + write := func(rel, content string) { + p := filepath.Join(dir, rel) + os.MkdirAll(filepath.Dir(p), 0o755) + os.WriteFile(p, []byte(content), 0o644) + } + write("skills/own-skill/SKILL.md", "---\nname: Own Skill\n---\n# Own\n") + write("skills/own-skill/run.sh", "#!/bin/sh\necho hi\n") + os.Chmod(filepath.Join(dir, "skills/own-skill/run.sh"), 0o755) + write("commands/deploy.md", "---\nname: Deploy\n---\nbody\n") + write("agents/helper.md", "---\nname: Helper\n---\nbody\n") + write("mcp/internal.yaml", "servers: {}\n") + h.git(t, dir, "add", "-A") + h.git(t, dir, "commit", "-m", "work repo") +} + +func TestSyncHappyPathAndManifest(t *testing.T) { + h := newHarness(t) + env := newSyncEnv(t, h) + buildWorkRepo(t, h, env.workRepo) + env.writeConfig(t, env.workRepo, "") + + out, err := env.run(t) + if err != nil { + t.Fatalf("sync failed: %v\n%s", err, out) + } + + skillMd := filepath.Join(env.home, ".agents/skills/own-skill/SKILL.md") + if _, err := os.Stat(skillMd); err != nil { + t.Error("skill must deploy to ~/.agents/skills//") + } + runSh := filepath.Join(env.home, ".agents/skills/own-skill/run.sh") + info, err := os.Stat(runSh) + if err != nil { + t.Fatal("skill folder artifacts must be preserved") + } + if info.Mode()&0o111 == 0 { + t.Error("executable bit must survive deployment") + } + if _, err := os.Stat(filepath.Join(env.home, ".config/opencode/commands/deploy.md")); err != nil { + t.Error("command must deploy to ~/.config/opencode/commands/.md") + } + if _, err := os.Stat(filepath.Join(env.home, ".config/opencode/agents/helper.md")); err != nil { + t.Error("agent must deploy to ~/.config/opencode/agents/.md") + } + if _, err := os.Stat(filepath.Join(env.home, ".config/opencode/mcp")); err == nil { + t.Error("MCP must never be deployed") + } + mcpEverywhere := false + filepath.WalkDir(env.home, func(p string, d os.DirEntry, err error) error { + if d != nil && !d.IsDir() && strings.HasSuffix(p, ".yaml") { + mcpEverywhere = true + } + return nil + }) + if mcpEverywhere { + t.Error("no yaml fragment may land anywhere in HOME") + } + + manifestData, err := os.ReadFile(filepath.Join(env.stateDir, "manifest.json")) + if err != nil { + t.Fatal("manifest must exist in the state dir") + } + var manifest struct { + Version int `json:"version"` + Items map[string]struct { + Name string `json:"name"` + Type string `json:"type"` + Origin string `json:"origin"` + Revision string `json:"revision"` + Hash string `json:"hash"` + } `json:"items"` + } + if err := json.Unmarshal(manifestData, &manifest); err != nil { + t.Fatalf("manifest not parseable: %v", err) + } + skill := manifest.Items["skills/own-skill"] + if skill.Origin != "own" || skill.Hash == "" || skill.Revision == "" { + t.Errorf("manifest record incomplete: %+v", skill) + } +} + +func TestSyncIdempotentSecondRun(t *testing.T) { + h := newHarness(t) + env := newSyncEnv(t, h) + buildWorkRepo(t, h, env.workRepo) + env.writeConfig(t, env.workRepo, "") + + if _, err := env.run(t); err != nil { + t.Fatal(err) + } + skillMd := filepath.Join(env.home, ".agents/skills/own-skill/SKILL.md") + firstInfo, _ := os.Stat(skillMd) + manifestBefore := readFile(t, filepath.Join(env.stateDir, "manifest.json")) + + out, err := env.run(t) + if err != nil { + t.Fatalf("second sync failed: %v\n%s", err, out) + } + if !contains(out, "kept: 3") { + t.Errorf("second run must keep everything:\n%s", out) + } + + secondInfo, _ := os.Stat(skillMd) + if !firstInfo.ModTime().Equal(secondInfo.ModTime()) { + t.Error("unchanged items must not be rewritten (byte-level no-op)") + } + if got := readFile(t, filepath.Join(env.stateDir, "manifest.json")); got != manifestBefore { + t.Error("manifest bytes must not change on a no-op sync") + } +} + +func TestSyncUnreachableRepoKeepsState(t *testing.T) { + h := newHarness(t) + env := newSyncEnv(t, h) + buildWorkRepo(t, h, env.workRepo) + env.writeConfig(t, env.workRepo, "") + + if _, err := env.run(t); err != nil { + t.Fatal(err) + } + skillBefore := readFile(t, filepath.Join(env.home, ".agents/skills/own-skill/SKILL.md")) + + env.writeConfig(t, filepath.Join(t.TempDir(), "does-not-exist"), "") + out, err := env.run(t) + if err == nil { + t.Fatalf("unreachable repo must fail:\n%s", out) + } + if !contains(out, "error") { + t.Errorf("failure must be a clean error:\n%s", out) + } + if got := readFile(t, filepath.Join(env.home, ".agents/skills/own-skill/SKILL.md")); got != skillBefore { + t.Error("previous state must survive a failed sync") + } +} + +func TestSyncPicksUpChanges(t *testing.T) { + h := newHarness(t) + env := newSyncEnv(t, h) + buildWorkRepo(t, h, env.workRepo) + env.writeConfig(t, env.workRepo, "") + env.run(t) + + p := filepath.Join(env.workRepo, "commands/deploy.md") + os.WriteFile(p, []byte("---\nname: Deploy\ndescription: updated\n---\nnew body\n"), 0o644) + h.git(t, env.workRepo, "add", "-A") + h.git(t, env.workRepo, "commit", "-m", "update command") + + out, err := env.run(t) + if err != nil { + t.Fatalf("sync after upstream change failed: %v\n%s", err, out) + } + if !contains(out, "updated: 1") { + t.Errorf("changed item must be updated:\n%s", out) + } + deployed := readFile(t, filepath.Join(env.home, ".config/opencode/commands/deploy.md")) + if !contains(deployed, "new body") { + t.Error("updated content must land on disk") + } +} + +func TestSyncVendoredContentDeploys(t *testing.T) { + h := newHarness(t) + standardFixture(h, t) + env := newSyncEnv(t, h) + buildWorkRepo(t, h, env.workRepo) + + upstreamWork := env.workRepo + cmd := agentLibCmd("vendor", "add", "superpowers", h.upstreamDir) + cmd.Dir = upstreamWork + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("vendor add into work repo: %v\n%s", err, out) + } + h.git(t, upstreamWork, "add", "-A") + h.git(t, upstreamWork, "commit", "-m", "vendor superpowers") + + env.writeConfig(t, upstreamWork, "") + out, err := env.run(t) + if err != nil { + t.Fatalf("sync with vendored content failed: %v\n%s", err, out) + } + if _, err := os.Stat(filepath.Join(env.home, ".agents/skills/good-skill/SKILL.md")); err != nil { + t.Error("vendored skill must deploy under its deployed name") + } + manifestData := []byte(readFile(t, filepath.Join(env.stateDir, "manifest.json"))) + if !contains(string(manifestData), `"origin": "superpowers"`) { + t.Errorf("manifest must record the source origin:\n%s", manifestData) + } +} + +func TestSyncTokenNeverLeaked(t *testing.T) { + h := newHarness(t) + env := newSyncEnv(t, h) + buildWorkRepo(t, h, env.workRepo) + env.writeConfig(t, env.workRepo, "super-secret-token") + + out, err := env.run(t) + if err != nil { + t.Fatalf("sync with token failed: %v\n%s", err, out) + } + if contains(out, "super-secret-token") { + t.Errorf("token must never appear in output:\n%s", out) + } + for _, path := range []string{ + filepath.Join(env.stateDir, "manifest.json"), + filepath.Join(env.workRepo, "agent-lib.lock.json"), + } { + if data, err := os.ReadFile(path); err == nil && strings.Contains(string(data), "super-secret-token") { + t.Errorf("token leaked into %s", path) + } + } +} diff --git a/internal/gitsource/git.go b/internal/gitsource/git.go index c511894..f99ebc9 100644 --- a/internal/gitsource/git.go +++ b/internal/gitsource/git.go @@ -111,6 +111,11 @@ type GitTree struct { tree *object.Tree } +// NewGitTree wraps a commit tree for discovery and materialization. +func NewGitTree(t *object.Tree) *GitTree { + return &GitTree{tree: t} +} + func (g *GitTree) Walk(fn func(rel string, isDir bool) error) error { walker := object.NewTreeWalker(g.tree, true, nil) defer walker.Close() diff --git a/internal/gitsource/redact.go b/internal/gitsource/redact.go new file mode 100644 index 0000000..0296cc7 --- /dev/null +++ b/internal/gitsource/redact.go @@ -0,0 +1,16 @@ +package gitsource + +import "strings" + +// RedactURL strips user:password credentials embedded in a URL so tokens +// never surface in error messages or logs. +func RedactURL(u string) string { + scheme, rest, found := strings.Cut(u, "://") + if !found { + return u + } + if at := strings.Index(rest, "@"); at >= 0 && !strings.Contains(rest[:at], "/") { + return scheme + "://" + rest[at+1:] + } + return u +} diff --git a/internal/lockfile/lockfile.go b/internal/lockfile/lockfile.go index 561d7cc..0a377af 100644 --- a/internal/lockfile/lockfile.go +++ b/internal/lockfile/lockfile.go @@ -98,6 +98,21 @@ func New() *Lockfile { return &Lockfile{Version: Version, Sources: map[string]*Source{}} } +// ParseBytes unmarshals and validates lockfile JSON content. +func ParseBytes(data []byte) (*Lockfile, error) { + var l Lockfile + if err := json.Unmarshal(data, &l); err != nil { + return nil, fmt.Errorf("lockfile is not valid JSON: %w", err) + } + if l.Version != Version { + return nil, fmt.Errorf("lockfile has schema version %d, want %d", l.Version, Version) + } + if l.Sources == nil { + l.Sources = map[string]*Source{} + } + return &l, nil +} + // Load reads the lockfile at path. A missing file yields ErrNotExist. func Load(path string) (*Lockfile, error) { data, err := os.ReadFile(path) diff --git a/internal/target/opencode.go b/internal/target/opencode.go new file mode 100644 index 0000000..8b7ad2b --- /dev/null +++ b/internal/target/opencode.go @@ -0,0 +1,41 @@ +// Package target maps content types to runtime-specific deployment paths. +// The OpenCode target is the thin v2 implementation: a type filter plus +// path mapping, leaving the door open for future targets. +package target + +import ( + "os" + "path/filepath" + + "github.com/m3tam3re/agent-lib/internal/lockfile" +) + +// OpenCode maps items into the OpenCode runtime layout, rooted at the +// user's home directory. +type OpenCode struct { + Home string +} + +// NewOpenCode resolves the home directory once for all path mappings. +func NewOpenCode() (*OpenCode, error) { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + return &OpenCode{Home: home}, nil +} + +// DeployPath returns the absolute destination for an item; ok is false for +// types this target never deploys (MCP stays under fleet control). +func (o *OpenCode) DeployPath(typ, name string) (path string, ok bool) { + switch typ { + case lockfile.TypeSkill: + return filepath.Join(o.Home, ".agents", "skills", name), true + case lockfile.TypeCommand: + return filepath.Join(o.Home, ".config", "opencode", "commands", name+".md"), true + case lockfile.TypeAgent: + return filepath.Join(o.Home, ".config", "opencode", "agents", name+".md"), true + default: + return "", false + } +} diff --git a/internal/vendor/add.go b/internal/vendor/add.go index 76e5456..a2ef689 100644 --- a/internal/vendor/add.go +++ b/internal/vendor/add.go @@ -10,6 +10,7 @@ import ( "sort" "strings" + "github.com/m3tam3re/agent-lib/internal/deploy" "github.com/m3tam3re/agent-lib/internal/discovery" "github.com/m3tam3re/agent-lib/internal/gitsource" "github.com/m3tam3re/agent-lib/internal/lockfile" @@ -171,10 +172,10 @@ func materialize(workDir, name string, tree discovery.Tree, selected []discovery switch it.Type { case lockfile.TypeSkill: dst = filepath.Join(base, dep) - err = copyTreeDir(tree, it.RelPath, dst) + err = deploy.Dir(tree, it.RelPath, dst) default: dst = filepath.Join(base, dep+filepath.Ext(it.RelPath)) - err = copyTreeFile(tree, it.RelPath, dst) + err = deploy.File(tree, it.RelPath, dst) } if err != nil { return nil, fmt.Errorf("materializing %s/%s: %w", it.Type, it.UpstreamID, err) @@ -184,54 +185,6 @@ func materialize(workDir, name string, tree discovery.Tree, selected []discovery return inventory, nil } -func copyTreeDir(tree discovery.Tree, srcRel, dst string) error { - lister, ok := tree.(interface { - FilesUnder(prefix string) ([]string, error) - }) - if !ok { - return fmt.Errorf("tree does not support directory copy") - } - files, err := lister.FilesUnder(srcRel) - if err != nil { - return err - } - if len(files) == 0 { - return fmt.Errorf("no files under %s", srcRel) - } - prefix := strings.TrimSuffix(srcRel, "/") + "/" - for _, f := range files { - relInside := strings.TrimPrefix(f, prefix) - target := filepath.Join(dst, filepath.FromSlash(relInside)) - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { - return err - } - if err := writeFileFromTree(tree, f, target); err != nil { - return err - } - } - return nil -} - -func copyTreeFile(tree discovery.Tree, srcRel, dst string) error { - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - return err - } - return writeFileFromTree(tree, srcRel, dst) -} - -func writeFileFromTree(tree discovery.Tree, rel, dst string) error { - if mt, ok := tree.(interface { - MaterializeFile(rel, dst string) error - }); ok { - return mt.MaterializeFile(rel, dst) - } - data, err := tree.ReadFile(rel) - if err != nil { - return err - } - return os.WriteFile(dst, data, 0o644) -} - func reportAdd(w io.Writer, name string, src *lockfile.Source, discovered, selected []discovery.Item) { counts := func(items []discovery.Item) string { per := map[string]int{} diff --git a/internal/vendor/update.go b/internal/vendor/update.go index e03e26a..0f48a89 100644 --- a/internal/vendor/update.go +++ b/internal/vendor/update.go @@ -10,6 +10,7 @@ import ( "sort" "strings" + "github.com/m3tam3re/agent-lib/internal/deploy" "github.com/m3tam3re/agent-lib/internal/discovery" "github.com/m3tam3re/agent-lib/internal/gitsource" "github.com/m3tam3re/agent-lib/internal/lockfile" @@ -257,10 +258,10 @@ func materializeTree(tree discovery.Tree, selected []discovery.Item, renames map switch it.Type { case lockfile.TypeSkill: target = filepath.Join(base, dep) - err = copyTreeDir(tree, it.RelPath, target) + err = deploy.Dir(tree, it.RelPath, target) default: target = filepath.Join(base, dep+filepath.Ext(it.RelPath)) - err = copyTreeFile(tree, it.RelPath, target) + err = deploy.File(tree, it.RelPath, target) } if err != nil { return fmt.Errorf("materializing %s/%s: %w", it.Type, it.UpstreamID, err)