feat: recursive skill discovery + tree-url type-root folding

- skills are found at any depth under the skills dir, so category-nested
  layouts (skills/<category>/<id>/SKILL.md) vendor correctly; the folder
  basename is the upstream id and the outermost SKILL.md wins over anything
  nested inside a skill folder
- web-tree urls pointing directly at a type directory (.../tree/main/skills)
  now map that directory as the type root instead of nesting it under the
  default layout
- verified against the real github.com/mattpocock/skills repository
  (36 skills across categories; include-selection by basename)
This commit is contained in:
2026-08-23 14:50:56 +02:00
parent 67cd4e4737
commit 1b2fcb6fbb
5 changed files with 165 additions and 16 deletions
+3
View File
@@ -19,3 +19,6 @@
{"id":"int-baf99558287967cfd7c3aeb9c0e6354c","kind":"field_change","created_at":"2026-08-23T08:31:08.602480836Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.5","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}}
{"id":"int-41040467e9dbadf410d0563d3775e728","kind":"field_change","created_at":"2026-08-23T08:36:53.924991634Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.6","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}}
{"id":"int-06e7ec6923aa26fc3a5eb446c5aa001c","kind":"field_change","created_at":"2026-08-23T08:40:34.06547132Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.7","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}}
{"id":"int-b75bf2027708c976d3772b28e613cd65","kind":"field_change","created_at":"2026-08-23T08:47:26.764431384Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w.8","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}}
{"id":"int-cebf102ccf1bf8528c4fec36a8f42cc7","kind":"field_change","created_at":"2026-08-23T08:47:26.967094464Z","actor":"m3ta-chiron","issue_id":"agent-lib-j2w","extra":{"field":"status","new_value":"closed","old_value":"open"}}
{"id":"int-bd14ae37b3379485b36911ca7a2bbe15","kind":"field_change","created_at":"2026-08-23T08:47:27.131938505Z","actor":"m3ta-chiron","issue_id":"agent-lib-4wx","extra":{"field":"status","new_value":"closed","old_value":"open"}}
+6 -3
View File
@@ -53,11 +53,14 @@ agent-lib.lock.json lockfile v2 — pins every source
agent-lib vendor add superpowers https://github.com/obra/superpowers/tree/main/skills
```
Pasted GitHub/GitLab web-tree URLs are normalized automatically (the tree
sub-path becomes the discovery root). `--ref <branch-or-tag>` pins a ref; the
Pasted GitHub/GitLab web-tree URLs are normalized automatically. A URL
pointing directly at a type directory (e.g. `…/tree/main/skills`) selects
that directory as the skills root. `--ref <branch-or-tag>` pins a ref; the
resolved revision is pinned in the lockfile. Discovery understands the four
standard directories (`skills/`, `commands/`, `agents/`, `mcp/`); skills are
folders containing `SKILL.md`, commands and agents are flat `.md` files, MCP
folders containing `SKILL.md` — found at **any depth**, so category-nested
layouts (`skills/engineering/tdd/SKILL.md`) work too, with the folder's base
name as the skill id — commands and agents are flat `.md` files, MCP
fragments are `.yaml`. Everything is read-only scanned — upstream code is
never executed.
+52 -12
View File
@@ -57,6 +57,7 @@ func Scan(tree Tree, cfg Config) ([]Item, error) {
dirEntries[d] = map[string]bool{}
fileEntries[d] = map[string]bool{}
}
var skillDescendants []string
err := tree.Walk(func(rel string, isDir bool) error {
parent, base := splitPath(rel)
@@ -64,6 +65,9 @@ func Scan(tree Tree, cfg Config) ([]Item, error) {
if set, ok := dirEntries[parent]; ok {
set[base] = true
}
if skillRoot != "" && (parent == skillRoot || strings.HasPrefix(parent, skillRoot+"/")) {
skillDescendants = append(skillDescendants, rel)
}
return nil
}
if set, ok := fileEntries[parent]; ok {
@@ -76,18 +80,7 @@ func Scan(tree Tree, cfg Config) ([]Item, error) {
}
var items []Item
for name := range dirEntries[skillRoot] {
skillMd := join(skillRoot, name, "SKILL.md")
data, err := tree.ReadFile(skillMd)
if err != nil {
continue
}
it := Item{Type: lockfile.TypeSkill, UpstreamID: name, RelPath: join(skillRoot, name)}
fm, warnings := ParseFrontmatter(data)
it.Frontmatter = &fm
for _, w := range warnings {
it.Warnings = append(it.Warnings, fmt.Sprintf("skills/%s: %s", name, w))
}
for _, it := range scanSkills(tree, skillRoot, skillDescendants) {
items = append(items, it)
}
items = append(items, scanFlatFiles(tree, fileEntries, commandRoot, lockfile.TypeCommand)...)
@@ -103,6 +96,53 @@ func Scan(tree Tree, cfg Config) ([]Item, error) {
return items, nil
}
// scanSkills finds skill folders at any depth under skillRoot (flat layouts
// and category-nested layouts alike). The upstream id is the folder's base
// name; a directory that is itself a skill wins over anything nested inside
// it, and same-basename folders collide at the vendor layer.
func scanSkills(tree Tree, skillRoot string, descendants []string) []Item {
if skillRoot == "" {
return nil
}
sorted := append([]string(nil), descendants...)
sort.Strings(sorted)
withSkillMd := map[string]bool{}
for _, d := range sorted {
if _, err := tree.ReadFile(join(d, "SKILL.md")); err == nil {
withSkillMd[d] = true
}
}
var items []Item
for _, d := range sorted {
if !withSkillMd[d] {
continue
}
nested := false
for other := range withSkillMd {
if other != d && strings.HasPrefix(d, other+"/") {
nested = true
break
}
}
if nested {
continue
}
data, err := tree.ReadFile(join(d, "SKILL.md"))
if err != nil {
continue
}
name := d[strings.LastIndex(d, "/")+1:]
it := Item{Type: lockfile.TypeSkill, UpstreamID: name, RelPath: d}
fm, warnings := ParseFrontmatter(data)
it.Frontmatter = &fm
for _, w := range warnings {
it.Warnings = append(it.Warnings, fmt.Sprintf("skills/%s: %s", name, w))
}
items = append(items, it)
}
return items
}
func scanFlatFiles(tree Tree, fileEntries map[string]map[string]bool, root, typ string) []Item {
var items []Item
for base := range fileEntries[root] {
+73
View File
@@ -0,0 +1,73 @@
package discovery
import (
"os"
"path/filepath"
"testing"
)
func writeTree(t *testing.T, root string, files map[string]string) {
t.Helper()
for rel, content := range files {
p := filepath.Join(root, filepath.FromSlash(rel))
os.MkdirAll(filepath.Dir(p), 0o755)
os.WriteFile(p, []byte(content), 0o644)
}
}
func TestScanNestedSkillCategories(t *testing.T) {
dir := t.TempDir()
writeTree(t, dir, map[string]string{
"skills/flat-skill/SKILL.md": "---\nname: Flat\n---\n",
"skills/engineering/tdd/SKILL.md": "---\nname: TDD\n---\n",
"skills/productivity/grill-me/SKILL.md": "---\nname: Grill Me\n---\n",
"skills/engineering/tdd/reference.md": "extra artifact",
"skills/deprecated/old/README.md": "no skill here",
"commands/deploy.md": "---\nname: Deploy\n---\n",
})
items, err := Scan(&FsTree{Root: dir}, Config{})
if err != nil {
t.Fatal(err)
}
got := map[string]Item{}
for _, it := range items {
got[it.UpstreamID] = it
}
for _, want := range []string{"flat-skill", "tdd", "grill-me"} {
it, ok := got[want]
if !ok {
t.Fatalf("skill %q not discovered (items: %+v)", want, items)
}
if it.Type != "skills" {
t.Errorf("%q type = %q", want, it.Type)
}
}
if _, ok := got["old"]; ok {
t.Error("folder without SKILL.md must not be a skill")
}
tdd := got["tdd"]
if tdd.RelPath != "skills/engineering/tdd" {
t.Errorf("tdd RelPath = %q", tdd.RelPath)
}
if got["deploy"].Type != "commands" {
t.Error("flat command types must still be discovered")
}
}
func TestScanOuterSkillWinsOverNested(t *testing.T) {
dir := t.TempDir()
writeTree(t, dir, map[string]string{
"skills/bundle/SKILL.md": "---\nname: Bundle\n---\n",
"skills/bundle/inner/SKILL.md": "---\nname: Inner\n---\n",
"skills/bundle/inner/deep/SKILL.md": "---\nname: Deep\n---\n",
})
items, err := Scan(&FsTree{Root: dir}, Config{})
if err != nil {
t.Fatal(err)
}
if len(items) != 1 || items[0].UpstreamID != "bundle" {
t.Fatalf("outermost skill must win, got %+v", items)
}
}
+31 -1
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"sort"
"strings"
@@ -51,6 +52,7 @@ func Add(workDir string, opts AddOptions, stdout io.Writer) error {
if err != nil {
return err
}
cfg := discoveryFromTreeRoot(root)
repo, err := gitsource.Clone(normalized, opts.Ref)
if err != nil {
@@ -61,7 +63,6 @@ func Add(workDir string, opts AddOptions, stdout io.Writer) error {
return err
}
cfg := lockfile.Discovery{Root: root}
items, err := discovery.Scan(tree, discovery.FromLockfileDiscovery(cfg))
if err != nil {
return err
@@ -117,6 +118,35 @@ func Add(workDir string, opts AddOptions, stdout io.Writer) error {
return nil
}
// discoveryFromTreeRoot maps a pasted tree sub-path onto discovery config.
// A sub-path ending in a standard type directory (e.g. ".../tree/main/skills")
// selects that directory as the type root; anything else becomes the
// discovery root under which the standard layout is expected.
func discoveryFromTreeRoot(root string) lockfile.Discovery {
if root == "" {
return lockfile.Discovery{}
}
parent, base := path.Split(strings.TrimSuffix(root, "/"))
parent = strings.TrimSuffix(parent, "/")
d := lockfile.Discovery{Root: parent}
switch base {
case lockfile.TypeSkill:
d.SkillsDir = base
case lockfile.TypeCommand:
d.CommandsDir = base
case lockfile.TypeAgent:
d.AgentsDir = base
case lockfile.TypeMcp:
d.McpDir = base
default:
return lockfile.Discovery{Root: root}
}
if d == (lockfile.Discovery{}) {
return lockfile.Discovery{}
}
return d
}
func checkIncludeEntries(items []discovery.Item, include []string, name string) error {
available := map[string]bool{}
for _, it := range items {