From b8798db398eaa136b263b749158088f5272e669b Mon Sep 17 00:00:00 2001 From: m3tam3re Date: Fri, 2 Oct 2026 09:37:50 +0200 Subject: [PATCH] fix agent definitions --- agent-lib.lock.json | 51 - agents/chiron-forge.md | 70 +- agents/chiron.md | 109 +- external/okf/agents/bundle-weaver.md | 187 --- external/okf/agents/event-analyzer.md | 144 --- external/okf/skills/backfill/SKILL.md | 338 ------ .../backfill/scripts/okf_backfill_events.py | 1007 ---------------- external/okf/skills/okf/SKILL.md | 159 --- .../okf/skills/okf/reference/APACHE-2.0.txt | 202 ---- external/okf/skills/okf/reference/SPEC.md | 1016 ----------------- external/okf/skills/okf/scripts/okf_init.py | 112 -- external/okf/skills/okf/templates/concept.md | 38 - external/okf/skills/okf/templates/index.md | 8 - external/okf/skills/okf/templates/log.md | 8 - external/okf/skills/validate/SKILL.md | 85 -- .../skills/validate/scripts/okf_validate.py | 573 ---------- external/okf/skills/visualize/SKILL.md | 64 -- .../skills/visualize/scripts/okf_visualize.py | 432 ------- 18 files changed, 128 insertions(+), 4475 deletions(-) delete mode 100644 external/okf/agents/bundle-weaver.md delete mode 100644 external/okf/agents/event-analyzer.md delete mode 100644 external/okf/skills/backfill/SKILL.md delete mode 100644 external/okf/skills/backfill/scripts/okf_backfill_events.py delete mode 100644 external/okf/skills/okf/SKILL.md delete mode 100644 external/okf/skills/okf/reference/APACHE-2.0.txt delete mode 100644 external/okf/skills/okf/reference/SPEC.md delete mode 100644 external/okf/skills/okf/scripts/okf_init.py delete mode 100644 external/okf/skills/okf/templates/concept.md delete mode 100644 external/okf/skills/okf/templates/index.md delete mode 100644 external/okf/skills/okf/templates/log.md delete mode 100644 external/okf/skills/validate/SKILL.md delete mode 100644 external/okf/skills/validate/scripts/okf_validate.py delete mode 100644 external/okf/skills/visualize/SKILL.md delete mode 100644 external/okf/skills/visualize/scripts/okf_visualize.py diff --git a/agent-lib.lock.json b/agent-lib.lock.json index e293250..32be6c2 100644 --- a/agent-lib.lock.json +++ b/agent-lib.lock.json @@ -52,57 +52,6 @@ "writing-shape" ] } - }, - "okf": { - "url": "https://github.com/scaccogatto/okf-skills.git", - "ref": "main", - "rev": "8e3187875e66051bb52f91a5ed27342e2c3208da", - "selection": { - "mode": "all" - }, - "inventory": { - "agents": [ - "bundle-weaver", - "event-analyzer" - ], - "skills": [ - "backfill", - "okf", - "validate", - "visualize" - ] - }, - "warnings": [ - "agents/bundle-weaver: malformed frontmatter line (missing colon): Fold event analyses into an OKF knowledge bundle in deterministic sequence. This is the", - "agents/bundle-weaver: malformed frontmatter line (missing colon): concepts over time, enforce anti-degeneration rules, and guarantee no events are lost.", - "agents/bundle-weaver: malformed frontmatter line (missing colon): only agent that writes `.okf/`, executing the reduce phase of backfill replay to evolve", - "agents/event-analyzer: malformed frontmatter line (missing colon): # Retier by forking this file; the backfill skill resolves the agent by name.", - "agents/event-analyzer: malformed frontmatter line (missing colon): # on the condition that the two explicit instructions below (flag source, evidence-only claims) stay.", - "agents/event-analyzer: malformed frontmatter line (missing colon): Analyze a single repository history event (git commit or session turn) to extract domain", - "agents/event-analyzer: malformed frontmatter line (missing colon): concepts and semantic content. Use in parallel during the map phase of OKF backfill replay", - "agents/event-analyzer: malformed frontmatter line (missing colon): to materialize decision rationale from raw commit diffs and session outcomes.", - "skills/backfill: malformed frontmatter line (missing colon): Reconstruct an OKF bundle by event-sourcing a repository's history (git log", - "skills/backfill: malformed frontmatter line (missing colon): and Claude session transcripts). Use when creating an `.okf/` bundle for an", - "skills/backfill: malformed frontmatter line (missing colon): existing repository that predates this skill, or when resuming an interrupted", - "skills/backfill: malformed frontmatter line (missing colon): knowledge bundle\", \"event-source the history\".", - "skills/okf: malformed frontmatter line (missing colon): \"capture this as a concept\", or any work in a repo that has an OKF bundle.", - "skills/okf: malformed frontmatter line (missing colon): Author, maintain, and consume Open Knowledge Format (OKF) knowledge bundles —", - "skills/okf: malformed frontmatter line (missing colon): capturing project knowledge (services, APIs, schemas, metrics, runbooks,", - "skills/okf: malformed frontmatter line (missing colon): decisions) into an OKF bundle, when updating one after code or docs change, or", - "skills/okf: malformed frontmatter line (missing colon): portable markdown + YAML frontmatter that both humans and agents read. Use when", - "skills/okf: malformed frontmatter line (missing colon): when a repository contains an `.okf/` (or other OKF) bundle that should inform", - "skills/validate: malformed frontmatter line (missing colon): Check that an Open Knowledge Format (OKF) bundle is conformant with the v0.2", - "skills/validate: malformed frontmatter line (missing colon): committing changes to one. Runs a deterministic Python checker — not an", - "skills/validate: malformed frontmatter line (missing colon): eyeball pass. Also migrates a v0.1 bundle to v0.2 in place with `--migrate`.", - "skills/validate: malformed frontmatter line (missing colon): spec (§11). Use when asked to validate, lint, or check an OKF bundle, or before", - "skills/visualize: malformed frontmatter line (missing colon): Render an Open Knowledge Format (OKF) bundle as a single self-contained,", - "skills/visualize: malformed frontmatter line (missing colon): Use when asked to visualize, graph, preview, or explore an OKF bundle.", - "skills/visualize: malformed frontmatter line (missing colon): a wiki-style detail panel with rendered markdown, v0.2 trust/lifecycle/provenance", - "skills/visualize: malformed frontmatter line (missing colon): filter and search.", - "skills/visualize: malformed frontmatter line (missing colon): interactive HTML graph (viz.html) — concepts as nodes coloured by type and", - "skills/visualize: malformed frontmatter line (missing colon): metadata, and \"Links to\" / \"Cited by\" backlinks, layout switching, per-type", - "skills/visualize: malformed frontmatter line (missing colon): sized by body length, markdown links and bundle-internal `sources` as edges," - ] } } } diff --git a/agents/chiron-forge.md b/agents/chiron-forge.md index 982a42c..2ce754c 100644 --- a/agents/chiron-forge.md +++ b/agents/chiron-forge.md @@ -2,28 +2,56 @@ name: chiron-forge description: Personal AI assistant (Build Mode). Full execution and task completion capabilities with safety prompts mode: primary -tools: [bash, edit, fetch_content, web_search, read, grep, find, ls] -model: zai/glm-5.3 +model: zai-coding-plan/glm-5.3 permissions: - bash: - "*": allow - "rm -rf *": ask - "git reset --hard*": ask - "git push*": ask - "git push --force*": deny - "git push -f *": deny - edit: - "*": allow - "/run/agenix/**": deny - external_directory: - "*": ask - "~/p/**": allow - "~/.config/opencode/**": allow - "/tmp/**": allow - "/run/agenix/**": allow - question: allow - webfetch: allow - websearch: allow + - action: shell + resource: "*" + effect: allow + - action: shell + resource: rm -rf * + effect: ask + - action: shell + resource: git reset --hard* + effect: ask + - action: shell + resource: git push* + effect: ask + - action: shell + resource: git push --force* + effect: deny + - action: shell + resource: git push -f * + effect: deny + - action: edit + resource: "*" + effect: allow + - action: edit + resource: /run/agenix/** + effect: deny + - action: external_directory + resource: "*" + effect: ask + - action: external_directory + resource: ~/p/** + effect: allow + - action: external_directory + resource: ~/.config/opencode/** + effect: allow + - action: external_directory + resource: /tmp/** + effect: allow + - action: external_directory + resource: /run/agenix/** + effect: allow + - action: question + resource: "*" + effect: allow + - action: webfetch + resource: "*" + effect: allow + - action: websearch + resource: "*" + effect: allow --- You are Chiron-Forge, the Greek centaur smith of Hephaestus, specializing in execution and task completion as Chiron's build counterpart. diff --git a/agents/chiron.md b/agents/chiron.md index ec44463..92b4856 100644 --- a/agents/chiron.md +++ b/agents/chiron.md @@ -2,37 +2,86 @@ name: chiron description: Personal AI assistant (Plan Mode). Read-only analysis, planning, and guidance mode: primary -tools: [bash, fetch_content, web_search, read, grep, find, ls] -model: zai/glm-5.3 +model: zai-coding-plan/glm-5.3 permissions: - bash: - "*": ask - "git status*": allow - "git log*": allow - "git diff*": allow - "git branch*": allow - "git show*": allow - "grep *": allow - "ls *": allow - "cat *": allow - "head *": allow - "tail *": allow - "wc *": allow - "which *": allow - "echo *": allow - "td *": allow - "bd *": allow - "nix *": allow - edit: deny - external_directory: - "*": ask - "~/p/**": allow - "~/.config/opencode/**": allow - "/tmp/**": allow - "/run/agenix/**": allow - question: allow - webfetch: allow - websearch: allow + - action: shell + resource: "*" + effect: ask + - action: shell + resource: git status* + effect: allow + - action: shell + resource: git log* + effect: allow + - action: shell + resource: git diff* + effect: allow + - action: shell + resource: git branch* + effect: allow + - action: shell + resource: git show* + effect: allow + - action: shell + resource: grep * + effect: allow + - action: shell + resource: ls * + effect: allow + - action: shell + resource: cat * + effect: allow + - action: shell + resource: head * + effect: allow + - action: shell + resource: tail * + effect: allow + - action: shell + resource: wc * + effect: allow + - action: shell + resource: which * + effect: allow + - action: shell + resource: echo * + effect: allow + - action: shell + resource: td * + effect: allow + - action: shell + resource: bd * + effect: allow + - action: shell + resource: nix * + effect: allow + - action: edit + resource: "*" + effect: deny + - action: external_directory + resource: "*" + effect: ask + - action: external_directory + resource: ~/p/** + effect: allow + - action: external_directory + resource: ~/.config/opencode/** + effect: allow + - action: external_directory + resource: /tmp/** + effect: allow + - action: external_directory + resource: /run/agenix/** + effect: allow + - action: question + resource: "*" + effect: allow + - action: webfetch + resource: "*" + effect: allow + - action: websearch + resource: "*" + effect: allow --- You are Chiron, the wise centaur from Greek mythology, serving as the main orchestrator in plan and analysis mode. You coordinate specialized subagents and provide high-level guidance without direct execution. diff --git a/external/okf/agents/bundle-weaver.md b/external/okf/agents/bundle-weaver.md deleted file mode 100644 index 838b479..0000000 --- a/external/okf/agents/bundle-weaver.md +++ /dev/null @@ -1,187 +0,0 @@ ---- -name: bundle-weaver -description: >- - Fold event analyses into an OKF knowledge bundle in deterministic sequence. This is the - only agent that writes `.okf/`, executing the reduce phase of backfill replay to evolve - concepts over time, enforce anti-degeneration rules, and guarantee no events are lost. -model: sonnet -effort: high -tools: Read, Write, Edit, Bash, Grep, Glob -color: green ---- - -You are the semantic weaver for OKF bundle reconstruction. Your task is to take the analyses -produced by `okf:event-analyzer` (one per event, in `analyses/` directory) and fold them into -the `.okf/` bundle in deterministic order. You are the only agent that writes to `.okf/`; your -decisions on concept naming, grouping, and log entries become the bundle's permanent record. - -## Input you receive - -- A `.okf/` bundle directory (may be empty or partially populated from a prior resume). -- An `analyses/` directory with one `.md` file per event, timestamped in frontmatter. File names - are event ids (content hashes) and carry no order. -- A cursor state file (`.okf/.backfill-state.json`) indicating which events have already been - folded (resume case). -- A list of "live" event ids (those without a `skip` field) to process in order. -- Each analysis carries a `truncated` flag in its frontmatter: `true` means the analyzer saw a - capped diff. Treat that rationale as partial evidence (lean even harder toward update over - create) and count it for your reply. - -## Your responsibilities - -### 1. Process events in order - -Read analyses in chronological order: follow the live-event-id list you were given (it carries -the event-stream order), or sort by each analysis's frontmatter `timestamp`. Do NOT sort by -filename: event ids are content hashes and their lexical order is unrelated to time. -For each event: - -1. Load the analysis file. -2. Extract candidate concept names and rationale. -3. Decide: update existing concepts, or create new ones (prefer update). -4. For each touched concept: append source reference (the event id, with `:` sanitized to `-` - in the filename). Every live event lands in at least one concept's `sources`: that, not - `log.md`, is what the coverage check relies on. -5. If the event created, deprecated or superseded a concept, add a `log.md` bullet under the - event's date (§4). A routine update gets no bullet: its "why" goes into the concept body. -6. Update cursor: save `event_id` and increment `done` count. -7. Repeat. - -### 2. Concept lifecycle rules - -- **Name by domain entity, not by change.** Existing concept names are canonical; candidate names - from analyses are suggestions. If an analysis proposes `cache-invalidation` and `cache-invalidation.md` - already exists, update it. If it proposes `invalidation-strategy` and `cache-invalidation` exists, - check if the analysis belongs in the existing concept (usually yes); only create `invalidation-strategy` - if its scope is truly distinct. -- **1–3 concepts per event is typical.** If an analysis touches many subsystems, group them into - higher-level concepts rather than creating one per subsystem. -- **Update > Create.** The bundle evolves, not explodes. Each concept's body grows with sources and - discoveries; the frontmatter's `sources` section records all events that touched it. -- **A reversal is not growth.** Events arrive in chronological order, so a later one can overturn - what an earlier one established: a phase dropped, a limit changed, an option moved in or out of - scope, a target downgraded. Growth appends; a reversal must replace, or the bundle ends up - asserting both states in the present tense. Before writing, `grep` the whole bundle (every - directory, not the one you are about to write in) for the candidate's distinctive tokens and - reread every hit. Then: - - The analysis contradicts the concept you are updating → rewrite the body to what holds today, - and move the previous position to a dated line under a `## History` section ("2026-08-24: was - X, replaced by Y"). - - The new state already lives in a *different* concept → the outdated one gets - `status: deprecated` and a link to its successor, keeping its body and all its `sources` (so - the coverage check stays green). Note both sides in the log bullet. - - Contradiction you cannot resolve from the evidence → keep the newer state and mark the older - line `[UNCLEAR]` rather than leaving two present-tense claims standing. - Count each resolution for `superseded` in your reply. - -### 3. Frontmatter contract for concepts - -Each `.okf/*/*.md` file must have YAML frontmatter: - -```yaml -type: -title: -description: -status: draft # or `deprecated` once superseded (§2); never `stable` -tags: [tag1, tag2] -generated: - by: okf-backfill/0.9.5 - at: -sources: - - id: git-abc1234 # Sanitized event id (colons → dashes) - resource: git:abc1234 # Original id for reverse lookup - last_modified: - - id: session-file-42 - resource: session:file:42 - last_modified: -``` - -**`status` is always written, and never `stable`.** SPEC §5.4 reads an absent `status` as -`stable`, "ready for consumption", and defines `draft` as "not yet reviewed; possibly -incomplete" — the literal description of a machine-replayed history. Every concept you write is -`draft` until a human reviews it; the only other value you may write is `deprecated`, per §2. - -**For session events, the `resource` and `id` differ:** `session::` is the resource; -`session--` is the sanitized id (all `:` → `-`). - -### 4. Log entry format - -`log.md` records lifecycle events only, not every update: a concept **created**, a concept -**deprecated**, one concept **superseding** another. The date heading is the event's date, not -today's. Format: - -``` -## 2026-09-01 - -- **Creation**: [Presales pipeline](/concepts/sales/presales-pipeline.md), the sales workflow - formalized to clarify handoff points (from session: "Discuss..."). Sources: `git-abc1234` -- **Supersession**: [Vendor sync](/concepts/sales/vendor-sync.md) supersedes - [Manual vendor import](/concepts/sales/manual-vendor-import.md), now `deprecated`. Sources: `git-def5678` -``` - -Rules: -- Link to the concept file with Markdown syntax. -- Explain the "why" from the analysis, not just what changed. -- Never a bare restatement of the commit subject (that's the analysis's job; here you explain intent). -- A routine update, growth or a reversal inside one concept gets no bullet: the body carries - today's state, `## History` carries what it replaced, `sources` carries the events. - -### 5. Anti-degeneration enforcement - -Before finalizing each concept, run these checks: - -- **No change-derived names:** filenames must not match `merge-pull-request-*`, `feat:-*`, `fix:-*`, etc. -- **Kebab-case only:** `[a-z0-9-]` in filenames (no spaces, underscores, capitals, or special chars). -- **Concepts must be entities, not actions.** A concept represents "the thing", not "the change to the thing". - -If an analysis proposes a violating name, keep it out of the bundle and note the conflict in the log -(as a comment, or a separate line marked `[CONFLICT]` if the rule is unclear). - -### 6. Cursor and resume - -After each event: -```json -{"last_id": "git-abc1234", "done": 42} -``` - -If the job crashes or resumes, start from `last_id` + 1. The cursor is your audit trail: it records -how far you got and enables recovery without re-analyzing or re-writing. - -### 7. Directory structure - -Create a hierarchy based on the domain you infer from concepts: -- `skills/` for skills (backfill, validate, etc.) -- `decisions/` for architectural choices -- `components/` for technical components -- `integrations/` or `services/` for external integrations -- `tools/` for utilities - -If unsure, default to `concepts/`. - -## Output signature - -When done with all live events (or a batch, if resuming): -- `.okf//.md` files created/updated with sources and frontmatter. -- `.okf/log.md` updated with dated lifecycle bullets (creations, deprecations, supersessions). -- `.okf/.backfill-state.json` cursor updated. - -The bundle is production-ready after the orchestrator runs the finalize step (validator + coverage check). - -## What you return - -Your reply to the orchestrator is one line of counts and nothing else: - -``` -folded= created= updated= bullets= conflicts= superseded= truncated_inputs= last_id= -``` - -The bundle is on disk; never paste concept bodies, log sections, or analyses into the reply. - -## Hard rules - -- **Only you touch `.okf/`.** If you see modifications from earlier runs, they are canonical; - don't replay changes to concepts that already exist (compare `last_modified` in sources and - skip if you're re-analyzing the same event). -- **Atomic per event:** update multiple concepts, log once, save cursor once per event. -- **No external reasoning:** the bundle is built on the analyses you receive. Don't invent - concepts or connections not supported by the event record. diff --git a/external/okf/agents/event-analyzer.md b/external/okf/agents/event-analyzer.md deleted file mode 100644 index 21f2b86..0000000 --- a/external/okf/agents/event-analyzer.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -name: event-analyzer -description: >- - Analyze a single repository history event (git commit or session turn) to extract domain - concepts and semantic content. Use in parallel during the map phase of OKF backfill replay - to materialize decision rationale from raw commit diffs and session outcomes. -# Bulk-read worker: it moves evidence into structure, it does not judge the bundle. -# Retier by forking this file; the backfill skill resolves the agent by name. -# haiku since benchmark/map-tier/RESULTS.md: parity with sonnet on commits at under half the cost, -# on the condition that the two explicit instructions below (flag source, evidence-only claims) stay. -model: haiku -effort: medium -tools: Bash, Read, Write -color: blue ---- - -You are a semantic analyzer for repository history events. You examine one git commit or -Claude session turn (or a small batch of them) and extract the domain concepts it touches, the -rationale behind the change, and candidate names for OKF knowledge concepts. You are a bulk -reader: you move evidence into structure. Judgment about the bundle (naming, merging, what to -keep) belongs to the weaver, not to you. - -## Input you receive - -- One or more event ids, and `events`: the path of `events.jsonl`. Fetch each event yourself, - one call per id (the orchestrator never reads event content): - - ```bash - jq -c --arg id '' 'select(.id==$id)' - ``` - - Each event is a JSON object with: - - `id`: unique event identifier (e.g., `git:abc1234` or `session:file.jsonl:42`) - - `source`: either `git` or `session` - - git events: `sha`, `subject`, `body`, `author`, `files` (list of `{path, add, del}`) - - session events: `user` (question/prompt), `outcome` (assistant wrap-up), `title` - - `ts`: ISO 8601 timestamp -- `repo`: the repository root path. -- `out`: the directory where you write analyses. -- `emitter`: the path of the diff emitter script (`okf_backfill_events.py`), plus any - `--skip-globs` the orchestrator wants applied. - -## How to read the evidence - -- **Git commits**: the subject is the claim, the diff is the evidence. Read the diff **only** - through the emitter: - - ```bash - uv run --show [--skip-globs ...] - ``` - - The emitter prints the complete stat (every file, with line counts), then the patches capped - per file and cut only at hunk or file boundaries, with a bracketed marker at every cut, and a - fixed last line: `[diff: shown=X total=Y files_shown=A files_total=B truncated=true|false]`. - - Never run a raw `git show `: the harness would cut it blindly, mid-hunk. - - If `truncated=true` and the stat points at one file whose patch you need for the rationale, - you may make **one** follow-up call: `uv run --show --only `. - That is the only permitted escalation; then write what you have. -- **Session turns**: the `user` text and the `outcome` text are the whole semantic unit. The - jq fetch is the only tool call they need; do not go looking for more. - -Extract, per event: -1. **Claim**: what the event asserts or achieves. -2. **Entities**: which domain concepts, subsystems, files, or features it touches. Name only - paths and entities that appear in this event's own evidence. -3. **Rationale**: why, from the commit body, session context, or diff evidence. -4. **Candidate concepts**: kebab-case concept names (`presales-pipeline`, `vendor-sync`, - `cache-invalidation`; never `feat:-add-feature` or `merge-pull-request-...`). Suggestions - only; the weaver decides. - -## Anti-degeneration rules - -- **Concept names describe the domain entity, not the change type.** Forbidden: - - Names derived from a git subject (`feat:`, `fix:`, `chore:`, `merge-pull-request-...`) - - Non-kebab-case (allowed: `[a-z0-9-]` only; no spaces, underscores, capitals) - - Generic placeholders (`changes`, `update`, `task`) -- **Every candidate name justifies its existence.** Three subsystems touched, three candidates, - each with a phrase saying which subsystem it covers. - -## Output format - -One file per event: `/.md`, where `event-id` is the id with every `:` replaced -by `-` (`git-abc1234.md`, `session-file.jsonl-42.md`). Exactly this structure, nothing outside -it (no greeting, no preamble, no commentary, no restating the diff): - -```markdown ---- -event_id: -source: git|session -timestamp: -truncated: true|false ---- - -# Claim - -- - -# Entities - -- : - -# Rationale - -- -- - -# Candidate concepts - -- `concept-name`: ; -``` - -- `truncated` copies the last line of the **first** `--show` call for the commit: `true` if it - said `truncated=true`; `false` for complete diffs and for session events. A `--only` - follow-up never changes it: its own summary line describes one file, not the commit. -- Bullets only, one line each, leading with a name or a path. Rationale is at most two bullets. -- The claim states what the evidence shows happened, nothing more. A session outcome that only - shows intent or a first step is reported as intent or a first step; never extend it to what - probably happened next. When the evidence does not settle the why, write `[UNCLEAR]` in the - bullet instead of a plausible guess; the weaver routes it. -- When the prompt carries several events, write one file per event, each from its own evidence - only. Never let one event's content leak into another's analysis. - -## What you return - -Your reply to the orchestrator is **one line per event** and nothing else: - -``` -/.md candidates= truncated= -``` - -The analysis is on disk; the reply carries counts. If an event could not be analyzed, the line -is ` FAILED: `. - -## Hard constraints - -- **Never modify `.okf/`**: extraction only. The weaver is the only actor that writes the bundle. -- **No external knowledge**: describe what the evidence shows, never what you know from elsewhere. -- **No raw diffs**: the emitter and its `--only` follow-up are the only ways to read a commit. - -## Notes on reading a diff - -- `-` lines were removed, `+` lines replaced them. A hunk the emitter cut is marked; do not - guess what the missing part did. -- Filenames tell you the scope; hunks tell you the intent. diff --git a/external/okf/skills/backfill/SKILL.md b/external/okf/skills/backfill/SKILL.md deleted file mode 100644 index 0c9f38f..0000000 --- a/external/okf/skills/backfill/SKILL.md +++ /dev/null @@ -1,338 +0,0 @@ ---- -name: backfill -description: >- - Reconstruct an OKF bundle by event-sourcing a repository's history (git log - and Claude session transcripts). Use when creating an `.okf/` bundle for an - existing repository that predates this skill, or when resuming an interrupted - backfill session. Triggers on: "reconstruct the OKF bundle", "backfill the - knowledge bundle", "event-source the history". -user-invocable: true -argument-hint: "[repo-dir] [--no-sessions] [--sessions-dir DIR]" -allowed-tools: Bash Read Write Edit ---- - -# Reconstruct an OKF bundle from history - -This skill replays a repository's decision-making history (git commits and -Claude session transcripts) to rebuild its OKF knowledge bundle as if the -[okf](/okf) skill's Stop hook had been active from the start. - -The extraction layer is **deterministic** (same repo → byte-identical events); -the replay layer is an LLM loop, **replayable and auditable but not -byte-identical** (timestamps, summaries change per run). See the event schema -(§1) and skip rules (§3). - -## 1. Event schema - -Git commits and session turns become events (JSONL, one per line, sorted by -timestamp then source): - -```json -{"id":"git:","source":"git","ts":"","sha":"...","author":"...","subject":"...","body":"...","files":[{"path":"...","add":N,"del":N}]} -{"id":"session::","source":"session","ts":"","user":"...","outcome":"...","title":"...","branch":"...","skip":"..."} -``` - -- **Git events** come from `git log --first-parent --numstat`. -- **Session events** pair each user message with the **last** assistant text - block of that turn (the wrap-up), extracted from - `~/.claude/projects//*.jsonl` and worktree subdirs. -- **Timestamps** are normalized to UTC ISO 8601 strings ending in `Z`. -- **Skip field** (optional, added by extraction): marks low-signal events — - see §3, never overridden by replay. - -## 2. Protocol: Preflight → Extract → Bootstrap → Replay (Map+Reduce) → Finalize - -### Preflight: check if bundle can be rebuilt - -```bash -if [ -d /.okf ] && ! [ -f /.okf/.backfill-state.json ]; then - echo "ERROR: .okf/ exists but is incomplete. Delete it to rebuild from scratch, or pass --resume to continue from the last checkpoint." - exit 1 -fi -``` - -If `.backfill-state.json` exists, resume from the cursor; otherwise, fresh bootstrap. - -### Extract: generate event stream - -```bash -uv run "${CLAUDE_SKILL_DIR}/scripts/okf_backfill_events.py" \ - --out events.jsonl \ - [--no-sessions] [--sessions-dir ~/.claude/projects] \ - [--max-text 2000] [--skip-globs "vendor/**"] -``` - -Write `events.jsonl` to the scratchpad (never committed). Report: -- Total events extracted, per-source counts, and per-rule skip counts. - -### Bootstrap: initialize bundle (fresh only) - -Create `.okf/` with: - -```bash -mkdir -p /.okf -echo 'okf_version: "0.2"' > /.okf/index.md -echo '# Update Log' > /.okf/log.md -``` - -Cursor (`.okf/.backfill-state.json`): - -```json -{"last_id": null, "done": 0} -``` - -### Replay: two-phase map/reduce protocol - -The replay is now structured in two phases to enforce semantic depth and anti-degeneration rules: - -#### Phase 1: Map (parallel analysis, per-event) - -Launch `okf:event-analyzer` agents for the live events (those without `skip` field), in waves of -4 to 10 (see the cost note in §5), via Claude Code's Workflow `agentType` parameter. Each analyzer -receives its event ids, the `events.jsonl` path, the repo path, the output directory and the -emitter path, and: -- Fetches its own event JSON with `jq` (the orchestrator dispatches ids, never content) -- Reads git evidence only through the capped diff emitter (§6); session events need no other call -- Writes `analyses/.md` (event id with `:` sanitized to `-`) to scratchpad, with a - `truncated` flag copied from the emitter's last line -- Replies with one line of counts per event; the analysis never travels back in the reply -- Never touches `.okf/` -- Is resumable: skip events already in `analyses/` - -**Dispatch rule** (deterministic, from `events.jsonl`, no content read): a live event is -*small* when it is a session turn or a commit whose numstat totals at most 60 changed lines. -Small events are grouped chronologically, eight per analyzer call; large commits go one per -call. Measured on this repository (`benchmark/map-tier/RESULTS.md`): 57 calls instead of 147, -a third off the map phase at the same tier, no loss on any per-event metric and no -cross-event contamination. - -```bash -jq -c 'select(.skip==null) | {id, small: (.source=="session" or (([.files[]?|.add+.del]|add)//0) <= 60)}' events.jsonl -``` - -**Host-agnostic fallback** (if no Workflow support): spawn generic subagents with the same -system prompt as `agents/event-analyzer.md`, one per event, collecting analyses to scratchpad. - -#### Phase 2: Reduce (sequential folding, one agent) - -Run a single `okf:bundle-weaver` agent that: -- Reads all analyses in chronological order -- Folds them into `.okf/`, updating or creating concepts -- Enforces anti-degeneration rules (§4) -- Adds a dated `log.md` bullet for each lifecycle event only (a concept created, deprecated or - superseded); the "why" of a routine update goes into the concept body -- Manages cursor (`.okf/.backfill-state.json`) for resume capability -- Is the only actor that writes `.okf/` -- Replies with one line of counts (folded, created, updated, bullets, conflicts, superseded, - truncated inputs); the bundle never travels back in the reply - -**Resume behavior:** both phases support resumption. Phase 1 skips already-analyzed event ids; -Phase 2 restarts from `last_id` in the cursor. - -### Domain priming (advisory) - -Before starting the map phase, read the repository's README and directory structure to sketch -a candidate taxonomy of concepts (skills, integrations, decisions, etc.). This priming is -*advisory only* — it helps the analyzer emit better candidate names. The *guarantee* that -no events are lost comes from the deterministic coverage check in Finalize (§5), not from -priming; analyses without candidate names still flow to the weaver. - -### Context hygiene (orchestrator) - -The orchestrator handles ids and counts, never content. Events, analyses and diffs are read by -the agents; the orchestrator reads: - -```bash -wc -l events.jsonl # total events -jq -r 'select(.skip==null) | .id' events.jsonl # live ids to dispatch -ls analyses | wc -l # analyses written -``` - -Never `cat events.jsonl` or open an analysis from the orchestrator: whatever enters its context -is billed at the frontier rate on every following turn, and the routing exists to prevent that. - -### Finalize: validate and clean up - -1. **Write `index.md` per directory by hand** (do NOT run `okf_init.py` — it - would scaffold placeholder files): one `# Section` per directory, one - bullet per concept, `* [Title](file.md) - description` taken from each - concept's frontmatter (SPEC §8). The root `index.md` keeps its - `okf_version: "0.2"` frontmatter and links every subdirectory. - -2. **Add final log entry** (today's date, appended to existing dated section if present): - ``` - ## 2026-09-01 - - - **Backfill**: reconstructed from N events by okf-backfill/0.9.5 - ``` - -3. **Delete cursor:** - ```bash - rm -f .okf/.backfill-state.json - ``` - -4. **Run self-checks for anti-degeneration rules:** - ```bash - # No concept filenames derived from change type - ! find .okf -name "*.md" | grep -E "merge-pull-request|(^|/)(feat|fix|chore|docs)[:-]" - - # No non-kebab-case characters in filenames - ! find .okf -name "*.md" | grep -E "[^a-z0-9/.-]" - - # No identical consecutive log bullets - awk 'p==$0 && /^- / {exit 1} {p=$0}' .okf/log.md - - # Every concept declares its lifecycle (absent status reads as `stable`, SPEC §5.4) - [ -z "$(grep -rL '^status:' --include='*.md' .okf | grep -v '/index.md$\|/log.md$')" ] - ``` - These are lexical guards; they do not catch a bundle that asserts a superseded state. That - check is semantic and belongs to the weaver, at fold time, when it has both the analysis and - the earlier concepts on disk (§4 rule 5). - -5. **Run coverage check to guarantee every event is mapped:** - ```bash - uv run "${CLAUDE_SKILL_DIR}/scripts/okf_backfill_events.py" \ - --check-coverage events.jsonl .okf - ``` - Exit code 0 means all live events (non-skipped) appear in the bundle's `sources` or log. - Exit code 1 with a list of unmapped ids means the weaver skipped some events; fix and re-run. - -6. **Validate bundle schema:** - ```bash - uv run "${CLAUDE_SKILL_DIR}/../validate/scripts/okf_validate.py" .okf --strict - ``` - Fix every error before finishing. - -7. **Report** the resulting bundle: - - Concept count per directory - - Log entry samples (first and last) - - Coverage check result (all events mapped) - - Validation result (pass/fail, warnings) - - Superseded resolutions (the weaver's `superseded` count) and deprecated concepts: - `grep -rl '^status: deprecated' --include='*.md' .okf | wc -l` - - Agents spawned per phase (analyzers, weaver invocations) and, when the host reports it, - tokens per phase - - Truncated analyses: `grep -l '^truncated: true' analyses/*.md | wc -l`, next to the - deterministic estimate of git events whose full diff exceeds the default cap: - `jq -r 'select(.skip==null and .source=="git") | [.files[]|.add+.del] | add' events.jsonl | awk '$1>300' | wc -l` - -## 3. Skip rules - -Events matching a rule below are marked `skip: ` and skipped during -replay (cursor advances, no log entry): - -| Rule | Condition | Rationale | -|------|-----------|-----------| -| `paths-only-generated` | Git commit touches ONLY lockfiles or generated code (vendor/, node_modules/, dist/, *.min.*, *.lock, etc.) | Noise: build byproducts, no knowledge content | -| `merge-no-files` | Git merge commit with no file changes (first-parent only) | Housekeeping | -| `session-command-noise` | Session user message is a slash command or <20 chars and has no useful outcome | Ephemeral UI/chatter | - -Extend the first rule with `--skip-globs "extra/**"` for repo-specific patterns. - -## 4. Anti-degeneration rules - -The analyzer and weaver enforce these rules to prevent the bundle from devolving into -a mechanical listing of commits or a taxonomy-by-accident: - -1. **Concept names describe domain entities, not changes.** Forbidden: - - Filenames derived from git subject: `merge-pull-request-#2-....md`, `feat:-add-docs-skill.md`, `fix:-handle-edge-case.md` - - Bare action words: `update.md`, `fix.md`, `add-feature.md` - - Non-kebab-case: spaces, underscores, capitals, special characters (allowed: `[a-z0-9-]` only) - - Example: a commit with subject "feat: add presales pipeline" touches a domain concept. - The *concept* is named `presales-pipeline.md` (the entity), not `feat:-add-presales-pipeline.md` - (the change). The change history lives in frontmatter `sources` and the body's `## History`; - `log.md` holds only lifecycle events. - -2. **Prefer update over create.** Every event is analyzed for which concepts it touches; if - a concept already exists and the analysis fits, update its `sources` and body. Only create - a new concept if the analysis reveals a distinct domain entity not yet captured. - -3. **Log bullets are lifecycle events, and explain intent.** Only a concept created, - deprecated or superseded gets a bullet; a routine update does not. A bullet answers "why?" - from the commit body or session outcome, never just re-reads the subject: - - Bad: `- Added presales-pipeline.md feature` - - Good: `- **Creation**: [Presales pipeline](...), formalized to clarify handoff points` - -4. **No identical consecutive bullets.** A run of same-concept bullets means updates leaked - into the log: - ``` - - Feature X update - - Feature X update - - Feature X update - ``` - Instead: no bullet at all; the updates live in the concept's body and `sources`. - -5. **A reversal is not growth.** A replay walks the history forward, so a later event routinely - overturns an earlier one (a phase dropped, a limit changed, a target downgraded). "Prefer - update over create" is a merge rule and says nothing about this: applied alone it leaves the - bundle asserting both states in the present tense, in one concept or in two. The weaver greps - the whole bundle before writing and, on contradiction, replaces rather than appends: the body - states what holds today, the previous position drops to a dated line under `## History`, and - an outdated concept that a newer one supersedes gets `status: deprecated` plus a link to its - successor. A deprecated concept keeps its `sources`, so coverage stays green. None of the - guards in Finalize can catch this — they are lexical, and a superseded claim is well-formed. - -6. **Reconstructed concepts are `draft`, never `stable`.** SPEC §5.4 reads an absent `status` as - `stable` ("ready for consumption"); a bundle nobody has reread is `draft` ("not yet reviewed; - possibly incomplete"). The weaver writes `status:` on every concept. - -## 5. Implementation notes - -- **Determinism**: extraction is byte-identical; replay is not (time, LLM variance). - Both are auditable: events.jsonl is deterministic, map analyses are stored and resumable, - reduce loop is human-readable and cursor-backed. -- **Privacy**: events.jsonl goes to scratchpad, never committed. Session turn text is - truncated (head+tail) before extraction. -- **Cost note**: deep replay reads one capped diff per git commit (§6) to extract - rationale. For histories >500 events, consider splitting into sub-ranges and - replaying sequentially, or use `--skip-globs` to exclude low-signal paths (e.g., - vendored dependencies, generated code). Map phase is parallel in waves of 4 to 10: the - gate benchmark lost 260 of 299 runs to a high-concurrency mass failure and finished at - concurrency 4 (`benchmark/gate/RESULTS.md`). Reduce is sequential but much cheaper - (concepts already analyzed). -- **Future sources** (not implemented): GitHub PR/issue text, release notes, CI/deploy logs - — all optional post-MVP. **Lore protocol** (arXiv 2603.15566): on repos that adopt git - trailers (structured decision metadata), the extracted event's `body` already contains - trailers in a machine-readable format. The analyzer and weaver should use trailers as - the primary source of "why" (overriding generic inference from diff content). -- **Trust metadata**: `generated.by` is the backfill agent (`okf-backfill/0.9.5`), not - claimed as human-reviewed (`human:...`); concepts are correctly `unverified` (SPEC §5.3) and - `status: draft` (§5.4) — the two axes are independent: trust is who says it, lifecycle is - whether it has been reread. Writing `status` is not optional here: absent, §5.4 reads it as - `stable`, so a bundle nobody has reread would declare itself ready for consumption. - Map-phase analyses are working artifacts (stored for auditability during reduce); the - weaver's output is the canonical bundle. - -## 6. Capped diff emitter - -Analyzers never read a raw `git show`: the harness cuts long tool output blindly (mid-hunk, -character-based, host-dependent) and a cheap worker may not notice the cut. The extractor reads -the whole diff instead and emits a deterministic sample: - -```bash -uv run "${CLAUDE_SKILL_DIR}/scripts/okf_backfill_events.py" --show \ - [--only ] [--max-diff-lines 300] [--per-file-lines 120] [--max-line-chars 400] \ - [--skip-globs "vendor/**"] -``` - -- Diff against the first parent, so merge commits agree with the extracted numstat. -- The stat is always complete: entities survive any cap. -- Patches are capped per file (breadth over depth) and cut at hunk or file boundaries, with a - bracketed marker at every cut; lines longer than `--max-line-chars` are shortened with a - marker (a generated one-line file is "1 line" but can be hundreds of KB). -- Patches of generated files (lockfiles, `vendor/**`, `--skip-globs`) are omitted even inside - mixed commits; their stat line stays. -- The last line is fixed-form: `[diff: shown=X total=Y files_shown=A files_total=B truncated=true|false]`. - The analyzer copies `truncated` into its frontmatter; finalize reports the count. -- `--only ` is the one permitted follow-up when a truncated diff hides the rationale: - same cap logic, one file. - -Defaults keep one call under the harness limits with margin; tune per repo with the flags. - -The analyzer's tier is configuration, not infrastructure: `agents/event-analyzer.md` carries -`model` and `effort` in its frontmatter. Fork the file to retier the map phase; the skill -resolves the agent by name and nothing else changes. The default is `haiku` since the -map-tier benchmark (`benchmark/map-tier/RESULTS.md`): parity with `sonnet` on commits at -under half the map cost, on the condition that the two explicit instructions in the agent -file stay (which summary line the flag copies; claims report intent as intent). diff --git a/external/okf/skills/backfill/scripts/okf_backfill_events.py b/external/okf/skills/backfill/scripts/okf_backfill_events.py deleted file mode 100644 index 967b40d..0000000 --- a/external/okf/skills/backfill/scripts/okf_backfill_events.py +++ /dev/null @@ -1,1007 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.11" -# /// -"""Event-sourcing extractor: reconstruct OKF bundle from git + session history. - -Deterministic extraction: same input → byte-identical events.jsonl. - -Run: uv run okf_backfill_events.py [--out events.jsonl] - [--branch REF ...] [--no-sessions] [--sessions-dir DIR] [--max-text 2000] - [--skip-globs GLOB ...] - - uv run okf_backfill_events.py --show [--only ] - [--max-diff-lines 300] [--per-file-lines 120] [--max-line-chars 400] - [--skip-globs GLOB ...] -""" -from __future__ import annotations - -import argparse -import glob as glob_module -import fnmatch -import json -import re -import subprocess -import sys -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Optional - - -# Skip rule predicates -@dataclass -class SkipRule: - id: str - check: callable - - -def matches_globs(paths: list[str], globs: list[str]) -> bool: - """True if ALL paths match at least one glob pattern.""" - if not paths: - return False - for path in paths: - if not any(fnmatch.fnmatch(path, g) for g in globs): - return False - return True - - -LOCKFILE_GLOBS = [ - "*.lock", - "package-lock.json", - "pnpm-lock.yaml", - "uv.lock", - "Cargo.lock", - "go.sum", - "vendor/**", - "node_modules/**", - "dist/**", - "*.min.*", -] - - -def rule_paths_only_generated(event: dict, skip_globs: list[str]) -> bool: - """Commit only touches generated/lockfile paths.""" - if event["source"] != "git": - return False - globs = LOCKFILE_GLOBS + skip_globs - paths = [f["path"] for f in event.get("files", [])] - return matches_globs(paths, globs) - - -def rule_merge_no_files(event: dict, skip_globs: list[str]) -> bool: - """Merge commit with no files touched.""" - if event["source"] != "git": - return False - return len(event.get("files", [])) == 0 - - -def rule_session_command_noise(event: dict, skip_globs: list[str]) -> bool: - """Session event that is just a slash command or too short with no outcome.""" - if event["source"] != "session": - return False - user = event.get("user", "").strip() - - # Bare slash command (no space or just "/" alone) - if user == "/" or (user.startswith("/") and " " not in user): - return True - - # Local command output (starts with "<") - if user.startswith("<"): - return True - - # Too short after strip (< 20 chars) AND no useful outcome - if len(user) < 20 and not event.get("outcome"): - return True - - return False - - -SKIP_RULES = [ - SkipRule("paths-only-generated", rule_paths_only_generated), - SkipRule("merge-no-files", rule_merge_no_files), - SkipRule("session-command-noise", rule_session_command_noise), -] - - -def normalize_ts(ts: str) -> float: - """Convert ISO8601 timestamp to UTC epoch float.""" - # Handle RFC3339 with or without timezone - ts = ts.strip() - - # If it ends with Z, it's UTC - if ts.endswith("Z"): - ts = ts[:-1] + "+00:00" - - # Handle offset timezone offsets like +01:00 or -05:00 - try: - # Try ISO format parsing - if "T" in ts: - dt = datetime.fromisoformat(ts) - else: - # Date only - dt = datetime.fromisoformat(ts + "T00:00:00") - return dt.timestamp() - except Exception: - print(f"Warning: could not parse timestamp {ts!r}, using epoch 0", file=sys.stderr) - return 0.0 - - -def epoch_to_iso_z(epoch: float) -> str: - """Format a UTC epoch as ISO8601 with a Z suffix, second precision.""" - return datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def git_log(repo_dir: Path, branches: list[str]) -> list[dict]: - """Extract events from git log across one or more branches (union, first - branch wins on duplicate SHAs).""" - branches = _resolve_branches(repo_dir, branches) - if not branches: - return [] - - events = [] - seen_shas = set() - for branch in branches: - for event in _git_log_one_branch(repo_dir, branch): - if event["sha"] in seen_shas: - continue - seen_shas.add(event["sha"]) - events.append(event) - return events - - -def _resolve_branches(repo_dir: Path, branches: list[str]) -> list[str]: - """Resolve the branch list to query: explicit branches, else default - branch from origin/HEAD, else current branch.""" - if branches: - return branches - - # Try to get default branch from origin/HEAD - try: - result = subprocess.run( - ["git", "-C", str(repo_dir), "symbolic-ref", "refs/remotes/origin/HEAD"], - capture_output=True, text=True, check=False - ) - if result.returncode == 0: - # Typically returns "refs/remotes/origin/main" or similar - default_branch = result.stdout.strip().split("/")[-1] - branches = [default_branch] - except Exception: - pass - - if not branches: - # Fallback to current branch - try: - result = subprocess.run( - ["git", "-C", str(repo_dir), "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, text=True, check=True - ) - branches = [result.stdout.strip()] - except Exception: - return [] - - return branches - - -def _git_log_one_branch(repo_dir: Path, branch: str) -> list[dict]: - """Extract events from git log for a single branch (first-parent, - reverse, numstat).""" - events = [] - - # Get log info and numstat separately - try: - # First, get commit info (SHA, timestamp, author, subject, body) - log_result = subprocess.run( - ["git", "-C", str(repo_dir), "log", "--first-parent", "--reverse", - "--format=%H%n%aI%n%an%n%s%n%b%n<<>>", branch], - capture_output=True, text=True, check=True - ) - - # Get numstat (SHA and file changes) - numstat_result = subprocess.run( - ["git", "-C", str(repo_dir), "log", "--first-parent", "--reverse", - "--numstat", "--format=%H%n<<>>", branch], - capture_output=True, text=True, check=True - ) - - # Parse log output (maintain order) - log_blocks = log_result.stdout.split("<<>>") - commits_ordered = [] # List to maintain order - commits_dict = {} # Dict for lookup during numstat - - for block in log_blocks: - if not block.strip(): - continue - - lines = block.strip().split("\n") - if len(lines) < 4: - continue - - sha = lines[0].strip() - ts_str = lines[1].strip() - author = lines[2].strip() - subject = lines[3].strip() - body = "\n".join(lines[4:]).strip() if len(lines) > 4 else "" - - commit_data = { - "sha": sha, - "ts": normalize_ts(ts_str), - "author": author, - "subject": subject, - "body": body, - "files": [], - } - commits_ordered.append(commit_data) - commits_dict[sha] = commit_data - - # Parse numstat output - numstat_blocks = numstat_result.stdout.split("<<>>") - current_sha = None - - for block in numstat_blocks: - lines = block.strip().split("\n") - - for line in lines: - line = line.strip() - if not line: - continue - - # First line in block (after <<>>) is SHA - if line and "\t" not in line: - current_sha = line - if current_sha not in commits_dict: - # Shouldn't happen, but handle gracefully - commits_dict[current_sha] = { - "sha": current_sha, - "ts": 0, - "author": "", - "subject": "", - "body": "", - "files": [], - } - commits_ordered.append(commits_dict[current_sha]) - elif current_sha and "\t" in line: - # Numstat line: additions\tdeletions\tpath - parts = line.split("\t", 2) - if len(parts) >= 3: - try: - add = int(parts[0]) if parts[0] != "-" else 0 - dele = int(parts[1]) if parts[1] != "-" else 0 - path = parts[2] - commits_dict[current_sha]["files"].append({ - "path": path, - "add": add, - "del": dele - }) - except (ValueError, IndexError): - pass - - # Build events in the order we saw commits - for c in commits_ordered: - event = { - "id": f"git:{c['sha']}", - "source": "git", - "ts": c["ts"], - "sha": c["sha"], - "author": c["author"], - "subject": c["subject"], - "body": c["body"], - "files": c["files"], - } - events.append(event) - - except Exception: - # Git command failed, return empty - pass - - return events - - -def repo_slug(repo_abs: Path) -> str: - """Slugify an absolute repo path the way Claude Code names transcript dirs. - - Portable: also maps Windows separators and drive colons to dashes, so the - slug is stable regardless of platform path syntax. - """ - # Claude Code maps every non-alphanumeric character to a dash - # (verified: p-045_ekar_skills -> p-045-ekar-skills), not just separators. - return re.sub(r"[^A-Za-z0-9-]", "-", str(repo_abs)) - - -def sessions_from_transcripts(repo_dir: Path, sessions_dir: Optional[Path]) -> list[dict]: - """Extract events from Claude session transcripts.""" - events = [] - - if sessions_dir is None: - sessions_dir = Path.home() / ".claude" / "projects" - - if not sessions_dir.exists(): - return events - - # Compute repo slug and canonical path (for the cwd filter) - repo_abs = repo_dir.resolve() - slug = repo_slug(repo_abs) - - # Find session files: /*.jsonl and --*/*.jsonl - session_globs = [ - str(sessions_dir / slug / "*.jsonl"), - str(sessions_dir / f"{slug}--*/") + "*.jsonl", - ] - - session_files = [] - for pattern in session_globs: - session_files.extend(glob_module.glob(pattern)) - - session_files = sorted(set(session_files)) - - for session_file in session_files: - session_path = Path(session_file) - - # Read raw lines, keeping the real 1-based line number of each record - try: - raw_lines = session_path.read_text(encoding="utf-8").splitlines() - except Exception: - continue - - records = [] # list of (lineno, dict) - for lineno, line in enumerate(raw_lines, start=1): - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - except Exception: - continue - if isinstance(rec, dict): - records.append((lineno, rec)) - - # Title: the LAST aiTitle in the file, computed once per file (not - # rescanned per turn). - title = None - for _, rec in records: - if rec.get("type") == "ai-title" and rec.get("aiTitle"): - title = rec["aiTitle"] - - n = len(records) - i = 0 - while i < n: - lineno, rec = records[i] - - # Skip non-user messages, meta, and sidechain records - if rec.get("type") != "user" or rec.get("isMeta") or rec.get("isSidechain"): - i += 1 - continue - - content = (rec.get("message") or {}).get("content") - if isinstance(content, str): - user_text = content - elif isinstance(content, list): - text_blocks = [ - b.get("text", "") for b in content - if isinstance(b, dict) and b.get("type") == "text" - ] - if not text_blocks: - # e.g. tool_result-only user record: nothing to extract - i += 1 - continue - user_text = "".join(text_blocks) - else: - i += 1 - continue - - # cwd filter: only keep records from this repo (or a worktree under it) - cwd = rec.get("cwd") or "" - cwd_path = Path(cwd) if cwd else None - if cwd_path is None or (cwd_path != repo_abs and repo_abs not in cwd_path.parents): - i += 1 - continue - - ts_str = rec.get("timestamp", "") - branch = rec.get("gitBranch") - - # Find the outcome: scan forward to the next user record, taking - # the LAST non-empty text block across the assistant records - # in between (the turn wrap-up). - outcome_text = "" - j = i + 1 - while j < n: - _, next_rec = records[j] - if next_rec.get("type") == "user": - break - if next_rec.get("type") == "assistant": - acontent = (next_rec.get("message") or {}).get("content") - if isinstance(acontent, list): - for block in acontent: - if isinstance(block, dict) and block.get("type") == "text" and block.get("text"): - outcome_text = block["text"] - j += 1 - - event = { - "id": f"session:{session_path.name}:{lineno}", - "source": "session", - "ts": normalize_ts(ts_str), - "user": user_text, - "outcome": outcome_text if outcome_text else None, - "title": title, - "branch": branch, - } - events.append(event) - - i = j - - return events - - -def apply_skip_rules(event: dict, skip_globs: list[str]) -> Optional[str]: - """Apply skip rules and return rule_id if should be skipped, None otherwise.""" - for rule in SKIP_RULES: - if rule.check(event, skip_globs): - return rule.id - return None - - -def merge_and_sort(git_events: list[dict], session_events: list[dict]) -> list[dict]: - """Merge and sort events by (ts, source, id).""" - all_events = git_events + session_events - - # Sort by timestamp, then source (git first), then id - def sort_key(event): - source_order = {"git": 0, "session": 1} - return (event["ts"], source_order.get(event["source"], 999), event["id"]) - - all_events.sort(key=sort_key) - return all_events - - -def truncate_text(text: str, max_len: int = 2000) -> str: - """Apply head+tail truncation with marker.""" - if len(text) <= max_len: - return text - - head_len = max_len * 3 // 4 - tail_len = max_len - head_len - - head = text[:head_len] - tail = text[-tail_len:] if tail_len else "" - return head + "\n[...]\n" + tail - - -def check_coverage(events_path: Path, bundle_dir: Path) -> list[str]: - """Check that all live events (without skip field) are mapped in the bundle. - - Returns a list of unmapped event ids (empty if all are covered). - Live events are found by looking for those without a 'skip' key. - Coverage is determined by searching for event ids or their short shas in - the bundle's .md files (sources and log). - """ - if not events_path.exists(): - print(f"Error: {events_path} does not exist", file=sys.stderr) - sys.exit(1) - if not bundle_dir.exists(): - print(f"Error: {bundle_dir} does not exist", file=sys.stderr) - sys.exit(1) - - # Collect all live event ids - live_ids = [] - try: - with open(events_path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - event = json.loads(line) - if "skip" not in event and "id" in event: - live_ids.append(event["id"]) - except json.JSONDecodeError: - continue - except Exception as e: - print(f"Error reading events: {e}", file=sys.stderr) - sys.exit(1) - - # Collect all text from .md files in bundle - bundle_text = "" - try: - for md_file in bundle_dir.rglob("*.md"): - try: - bundle_text += md_file.read_text(encoding="utf-8") + "\n" - except Exception: - continue - except Exception as e: - print(f"Error reading bundle: {e}", file=sys.stderr) - sys.exit(1) - - # Check coverage - unmapped = [] - for event_id in live_ids: - if event_id.startswith("git:"): - # Git events: search for full sha and any >=7-char prefix - sha = event_id[4:] # Remove "git:" prefix - short_sha = sha[:7] - # Check if full sha or short sha appears in bundle - if sha not in bundle_text and short_sha not in bundle_text: - unmapped.append(event_id) - elif event_id.startswith("session:"): - # Session events: search for literal id (with colons) - if event_id not in bundle_text: - unmapped.append(event_id) - else: - # Unknown format, consider unmapped - unmapped.append(event_id) - - return unmapped - - -def split_patch(patch_text: str) -> list[dict]: - """Split -p output into blocks on lines starting with 'diff --git'. - - Each block contains: - - path: str (the b/ path from the diff --git line, or new path for renames) - - header: list[str] (lines before first @@ line) - - hunks: list[list[str]] (groups starting at each @@ line) - """ - if not patch_text: - return [] - - lines = patch_text.split("\n") - blocks = [] - current_block = None - - for line in lines: - if line.startswith("diff --git "): - # Start new block: extract path from b/ side - # Format: diff --git a/ b/ - # ponytail: paths containing " b/" would be incorrectly split; git-quoted paths are out of scope - if " b/" in line: - path = line.split(" b/", 1)[1] - else: - path = "" - - if current_block is not None: - blocks.append(current_block) - current_block = { - "path": path, - "header": [line], - "hunks": [], - } - elif current_block is not None: - if line.startswith("@@"): - # Start new hunk - current_block["hunks"].append([line]) - elif current_block["hunks"]: - # Add to current hunk - current_block["hunks"][-1].append(line) - else: - # Header line (before first hunk) - current_block["header"].append(line) - - if current_block is not None: - blocks.append(current_block) - - return blocks - - -def cap_patch( - blocks: list[dict], - numstat_lines: list[str], - *, - per_file_lines: int, - max_diff_lines: int, - max_line_chars: int, - skip_globs: list[str], - only: Optional[str] = None, -) -> tuple[list[str], dict]: - """Cap patches per-file and globally. - - Returns (patch_lines, summary_dict) where: - - patch_lines: list of lines to emit (markers included, no '# patch' header) - - summary_dict: {shown, total, files_shown, files_total, truncated} - - Capping rules: - - Glob-omitted files: emit marker, consume nothing, don't count toward total - - Per-file budget: max(per_file_lines, max_diff_lines - shown) under normal mode, - min(max_diff_lines, max_diff_lines - shown) under --only - - Header exceeds budget: emit header lines that fit, then accounting line with - N = unshown header lines + all hunk lines, M = len(hunks) - - Hunk doesn't fit: emit it whole if it fits remaining budget; otherwise emit - [+N more lines in M more hunks] or [hunk truncated: +N more lines] - - Global cap: before starting file, if shown >= max_diff_lines, emit - [patches omitted for K more files (+N lines)] and stop - - Line-length rule: lines longer than max_line_chars are truncated with marker - """ - globs = LOCKFILE_GLOBS + skip_globs - patch_lines = [] - shown = 0 - files_shown = 0 - truncated = False - - # Parse numstat for totals and --only validation - numstat_by_path = {} - for numstat_line in numstat_lines: - parts = numstat_line.split("\t", 2) - if len(parts) >= 3: - add_str, del_str, path = parts[0], parts[1], parts[2] - numstat_by_path[path] = (add_str, del_str) - - files_total = len(numstat_lines) - - # Total counts every eligible file (not glob-omitted, matching --only), including - # the ones the global cap omits: truncated=true must mean shown < total. - eligible = [ - b for b in blocks - if not matches_globs([b["path"]], globs) and (only is None or b["path"] == only) - ] - total = sum(len(b["header"]) + sum(len(h) for h in b["hunks"]) for b in eligible) - - # Validate --only path - if only is not None: - if only not in numstat_by_path: - raise KeyError(f"Path not found in commit: {only}") - - # Process blocks - block_idx = 0 - for block_idx, block in enumerate(blocks): - block_path = block["path"] - - # Check glob omission - if matches_globs([block_path], globs): - # Emit marker, consume nothing - if block_path in numstat_by_path: - add_str, del_str = numstat_by_path[block_path] - else: - add_str, del_str = "-", "-" - patch_lines.append(f"[patch omitted: {block_path} (generated), {add_str} {del_str}]") - continue - - # Check --only filter - if only is not None and block_path != only: - continue - - # Global cap check - if shown >= max_diff_lines: - # Count remaining non-glob blocks and their lines - remaining_files = 0 - remaining_lines = 0 - for j in range(block_idx, len(blocks)): - remaining_block = blocks[j] - if matches_globs([remaining_block["path"]], globs): - continue - if only is not None and remaining_block["path"] != only: - continue - remaining_files += 1 - for hunk in remaining_block["hunks"]: - remaining_lines += len(hunk) - remaining_lines += len(remaining_block["header"]) - - if remaining_files > 0: - patch_lines.append(f"[patches omitted for {remaining_files} more files (+{remaining_lines} lines)]") - truncated = True - break - - # Calculate file budget - if only is not None: - file_budget = min(max_diff_lines, max_diff_lines - shown) - else: - file_budget = min(per_file_lines, max_diff_lines - shown) - - # Emit header lines - file_lines_emitted = 0 - header_lines_to_emit = [] - for hline in block["header"]: - if len(hline) > max_line_chars: - shortened = hline[:max_line_chars] + f" [line truncated: {len(hline)} chars]" - header_lines_to_emit.append(shortened) - else: - header_lines_to_emit.append(hline) - file_lines_emitted += 1 - if file_lines_emitted > file_budget: - # Header alone exceeds budget - header_lines_to_emit = header_lines_to_emit[: file_budget] - break - - patch_lines.extend(header_lines_to_emit) - shown += len(header_lines_to_emit) - - # Process hunks - if file_lines_emitted <= file_budget and len(block["header"]) > 0: - # Some header was emitted and didn't exceed budget - remaining_budget = file_budget - file_lines_emitted - else: - # Either no header or header exactly filled budget - remaining_budget = file_budget - len(header_lines_to_emit) - - hunks_emitted = 0 - for hunk_idx, hunk in enumerate(block["hunks"]): - hunk_size = len(hunk) - - if hunk_size <= remaining_budget: - # Hunk fits - for hline in hunk: - if len(hline) > max_line_chars: - shortened = hline[:max_line_chars] + f" [line truncated: {len(hline)} chars]" - patch_lines.append(shortened) - else: - patch_lines.append(hline) - shown += 1 - remaining_budget -= hunk_size - hunks_emitted += 1 - else: - # Hunk doesn't fit - truncated = True - - if hunks_emitted == 0: - # No hunk emitted yet: either emit partial hunk or full accounting - if remaining_budget > 0: - # Emit first remaining_budget lines of this hunk - for hline in hunk[:remaining_budget]: - if len(hline) > max_line_chars: - shortened = hline[:max_line_chars] + f" [line truncated: {len(hline)} chars]" - patch_lines.append(shortened) - else: - patch_lines.append(hline) - shown += 1 - # Emit truncation marker - remaining_hunk_lines = hunk_size - remaining_budget - patch_lines.append(f"[hunk truncated: +{remaining_hunk_lines} more lines]") - else: - # Budget is 0 or negative, emit full accounting - remaining_hunk_lines = hunk_size + sum(len(h) for h in block["hunks"][hunk_idx + 1:]) - remaining_hunks = len(block["hunks"]) - hunk_idx - patch_lines.append(f"[+{remaining_hunk_lines} more lines in {remaining_hunks} more hunks of {block_path}]") - else: - # Some hunks emitted, more remain - remaining_hunk_lines = hunk_size + sum(len(h) for h in block["hunks"][hunk_idx + 1:]) - remaining_hunks = len(block["hunks"]) - hunk_idx - patch_lines.append(f"[+{remaining_hunk_lines} more lines in {remaining_hunks} more hunks of {block_path}]") - break - - if hunks_emitted > 0 or len(header_lines_to_emit) > 0: - files_shown += 1 - - summary = { - "shown": shown, - "total": total, - "files_shown": files_shown, - "files_total": files_total, - "truncated": truncated, - } - - return patch_lines, summary - - -def format_summary(summary: dict) -> str: - """Format the fixed-form summary line.""" - return ( - f"[diff: shown={summary['shown']} total={summary['total']} " - f"files_shown={summary['files_shown']} files_total={summary['files_total']} " - f"truncated={'true' if summary['truncated'] else 'false'}]" - ) - - -def show_commit( - repo_dir: Path, - sha: str, - *, - only: Optional[str] = None, - max_diff_lines: int = 300, - per_file_lines: int = 120, - max_line_chars: int = 400, - skip_globs: list[str] = (), -) -> str: - """Run git, emit the full capped diff text. - - Raises ValueError for a bad sha (exit 1) or KeyError for --only path not found (exit 2). - """ - # Resolve the sha first, separately from the parent check - resolve_result = subprocess.run( - ["git", "-C", str(repo_dir), "rev-parse", "--verify", "--quiet", f"{sha}^{{commit}}"], - capture_output=True, text=True, errors="replace", check=False - ) - if resolve_result.returncode != 0: - raise ValueError(f"Bad commit: {sha}") - - full_sha = resolve_result.stdout.strip() - - # Check if this is a root commit or has a parent - parent_result = subprocess.run( - ["git", "-C", str(repo_dir), "rev-parse", "--verify", "--quiet", f"{full_sha}^1"], - capture_output=True, text=True, errors="replace", check=False - ) - - if parent_result.returncode == 0: - # Has a parent: diff against first parent - base = parent_result.stdout.strip() - stat_result = subprocess.run( - ["git", "-C", str(repo_dir), "diff", "--no-color", "-M", "--numstat", base, full_sha], - capture_output=True, text=True, errors="replace", check=True - ) - patch_result = subprocess.run( - ["git", "-C", str(repo_dir), "diff", "--no-color", "-M", "-p", base, full_sha], - capture_output=True, text=True, errors="replace", check=True - ) - else: - # Root commit: use git show - stat_result = subprocess.run( - ["git", "-C", str(repo_dir), "show", "--no-color", "-M", "--format=", "--numstat", full_sha], - capture_output=True, text=True, errors="replace", check=True - ) - patch_result = subprocess.run( - ["git", "-C", str(repo_dir), "show", "--no-color", "-M", "--format=", "-p", full_sha], - capture_output=True, text=True, errors="replace", check=True - ) - - # Parse stat - numstat_lines = [ - line.strip() - for line in stat_result.stdout.split("\n") - if line.strip() - ] - - # Parse patch - blocks = split_patch(patch_result.stdout) - - # Cap the patch - patch_lines, summary = cap_patch( - blocks, - numstat_lines, - per_file_lines=per_file_lines, - max_diff_lines=max_diff_lines, - max_line_chars=max_line_chars, - skip_globs=list(skip_globs), - only=only, - ) - - # Build output - lines = [] - lines.append(f"# commit {full_sha}") - lines.append("# stat (complete)") - lines.extend(numstat_lines) - lines.append("# patch") - lines.extend(patch_lines) - lines.append(format_summary(summary)) - - return "\n".join(lines) + "\n" - - -def main(): - parser = argparse.ArgumentParser( - description="Event-sourcing extractor for OKF bundle reconstruction" - ) - - # Check if this is a coverage check (--check-coverage mode) - if "--check-coverage" in sys.argv: - parser.add_argument("--check-coverage", nargs=2, metavar=("EVENTS_JSONL", "BUNDLE_DIR"), - help="Check coverage of events in bundle") - args = parser.parse_args() - if args.check_coverage: - events_path = Path(args.check_coverage[0]) - bundle_dir = Path(args.check_coverage[1]) - unmapped = check_coverage(events_path, bundle_dir) - if unmapped: - print(f"Unmapped events (exit 1):") - for event_id in unmapped: - print(f" {event_id}") - sys.exit(1) - else: - print("All events mapped (exit 0)") - sys.exit(0) - - # Normal extraction mode / show mode - parser.add_argument("repo_dir", nargs="?", help="Repository directory") - parser.add_argument("--out", default="events.jsonl", help="Output file") - parser.add_argument( - "--branch", action="append", default=[], help="Git branch(es) to extract" - ) - parser.add_argument( - "--no-sessions", action="store_true", help="Skip session extraction" - ) - parser.add_argument( - "--sessions-dir", type=Path, help="Override sessions directory" - ) - parser.add_argument( - "--max-text", type=int, default=2000, help="Max text length per field" - ) - parser.add_argument( - "--skip-globs", action="append", default=[], help="Additional skip globs" - ) - parser.add_argument( - "--show", help="Show capped diff for a commit (exit 0 on success, 1 on bad sha)" - ) - parser.add_argument( - "--only", help="Restrict patch to a single path (with --show)" - ) - parser.add_argument( - "--max-diff-lines", type=int, default=300, help="Max diff lines total" - ) - parser.add_argument( - "--per-file-lines", type=int, default=120, help="Max diff lines per file" - ) - parser.add_argument( - "--max-line-chars", type=int, default=400, help="Max characters per line" - ) - - args = parser.parse_args() - - if not args.repo_dir: - print("Error: repo_dir is required", file=sys.stderr) - sys.exit(1) - - repo_dir = Path(args.repo_dir).resolve() - if not repo_dir.exists(): - print(f"Error: {repo_dir} does not exist", file=sys.stderr) - sys.exit(1) - - # Handle --show mode - if args.show: - try: - output = show_commit( - repo_dir, - args.show, - only=args.only, - max_diff_lines=args.max_diff_lines, - per_file_lines=args.per_file_lines, - max_line_chars=args.max_line_chars, - skip_globs=args.skip_globs, - ) - sys.stdout.write(output) - sys.exit(0) - except ValueError as e: - # Bad sha - print(str(e), file=sys.stderr) - sys.exit(1) - except KeyError as e: - # --only path not found - print(e.args[0], file=sys.stderr) - sys.exit(2) - - # Normal extraction mode - # Extract events - git_events = git_log(repo_dir, args.branch) - session_events = [] if args.no_sessions else sessions_from_transcripts( - repo_dir, args.sessions_dir - ) - - # Merge and sort - events = merge_and_sort(git_events, session_events) - - # Apply skip rules and truncate - skip_globs = args.skip_globs - skip_counts = {} - - for event in events: - skip_id = apply_skip_rules(event, skip_globs) - if skip_id: - event["skip"] = skip_id - skip_counts[skip_id] = skip_counts.get(skip_id, 0) + 1 - - # Truncate text fields - for key in ["subject", "body", "outcome", "user"]: - if key in event and isinstance(event[key], str): - event[key] = truncate_text(event[key], args.max_text) - - # Emit ts as an ISO8601 UTC string; the epoch float was only needed - # for sorting. - event["ts"] = epoch_to_iso_z(event["ts"]) - - # Write JSONL (deterministic order) - out_path = Path(args.out) - out_path.parent.mkdir(parents=True, exist_ok=True) - - with open(out_path, "w", encoding="utf-8") as f: - for event in events: - # Sort keys for determinism - sorted_event = {k: event[k] for k in sorted(event.keys())} - f.write(json.dumps(sorted_event, separators=(',', ':'), sort_keys=True) + "\n") - - # Report - print(f"Extracted {len(events)} events") - print(f" Git: {len(git_events)}") - print(f" Sessions: {len(session_events)}") - if skip_counts: - print("Skipped:") - for rule_id, count in sorted(skip_counts.items()): - print(f" {rule_id}: {count}") - - -if __name__ == "__main__": - main() diff --git a/external/okf/skills/okf/SKILL.md b/external/okf/skills/okf/SKILL.md deleted file mode 100644 index d874251..0000000 --- a/external/okf/skills/okf/SKILL.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -name: okf -description: >- - Author, maintain, and consume Open Knowledge Format (OKF) knowledge bundles — - portable markdown + YAML frontmatter that both humans and agents read. Use when - capturing project knowledge (services, APIs, schemas, metrics, runbooks, - decisions) into an OKF bundle, when updating one after code or docs change, or - when a repository contains an `.okf/` (or other OKF) bundle that should inform - the task. Triggers on: "document this in OKF", "update the knowledge bundle", - "capture this as a concept", or any work in a repo that has an OKF bundle. -user-invocable: true -argument-hint: "[produce|maintain|consume] [path]" -allowed-tools: Read Write Edit Grep Glob Bash ---- - -# Open Knowledge Format (OKF) skill - -OKF represents knowledge as a directory of markdown files with YAML frontmatter. -It is minimal by design: no schema registry, no runtime, no SDK. Your job is to -produce, maintain, and consume OKF bundles **conformant with the spec**, not your -memory of it. - -**Always read the canonical spec before non-trivial work:** -[reference/SPEC.md](reference/SPEC.md). It is the verbatim OKF v0.2 specification -and the source of truth for every rule below. - -## The one hard rule - -A bundle is conformant (§11) iff: every non-reserved `.md` file has a parseable -YAML frontmatter block, and every such block has a **non-empty `type`** field. -Everything else is soft guidance. Consumers MUST tolerate missing optional -fields, unknown types, and broken links — never reject a bundle over them. - -## Conventions to apply - -- **One concept = one file.** The file path (minus `.md`) is the concept ID. -- **Frontmatter:** `type` is required. Add `title`, `description`, `tags` when - they aid consumption; add `resource` (a canonical URI) only for concepts bound - to a real asset — omit it for abstract concepts. -- **Body:** prefer structural markdown (headings, tables, lists, fenced code). - Conventional headings: `# Schema`, `# Examples`, `# Computation`. -- **Cross-links:** standard markdown links; prefer absolute bundle-relative - form (`/services/auth-api.md`). A link asserts a relationship; its *kind* lives - in the surrounding prose, not the link. -- **Reserved files:** `index.md` (directory listing, no frontmatter, except the - bundle-root index, which may carry `okf_version` and this plugin's - `upkeep: enforced` opt-in flag) and `log.md` (ISO-dated history, newest - first). Never use these names for concepts. -- **What goes in `log.md`:** lifecycle events only, written in the bundle-root - `log.md`: a bundle **created**, a concept **deprecated** or retired, one - concept **superseding** another, concepts **regenerated** after their source - changed, a **verification** pass. Routine edits to a concept do not get an - entry: its `generated` and git already record them. A log is append-only, so - never rewrite or drop an existing entry. - -## The v0.2 families (all optional, all worth filling) - -- **Trust (§5.2):** `generated: { by, at }` — who produced the current content - and when. `verified: [{ by, at }]` — who confirmed it since (a bare mapping is - one entry). Write `by` in the **actor convention** (§7): `/` - for an agent, `human:` for a person, `process:` for an automated job. - Use `human:` whenever a person authored or signed off — consumers key trust - tiers off that prefix. -- **Lifecycle (§5.4–5.5):** `status: draft|stable|deprecated` (absent means - stable) and `stale_after`, an absolute ISO 8601 datetime (a bare date is - tolerated), not a TTL. -- **Provenance (§5.1):** `sources: [{ id, resource, title, author, - usage_count, last_modified }]` plus a `usage_window: { from, to }` sibling of - `sources` framing every `usage_count` (an entry may carry its own to override - it); a `usage_count` without a window warns. `resource` is required per entry - and may be a URL, a bundle path, or a scope descriptor. Attribute a specific - claim with a markdown footnote whose label is the source's `id`: - `…sharded daily.[^ga4-schema]` plus a `[^ga4-schema]: …` definition. The label - is the join key, it must match a `sources[].id`. -- **Attestation (§10):** a sanctioned computation is its own concept, - `type: Attested Computation`, carrying `runtime` (required), `parameters`, - `executor`, `attester`, and the computation itself under `# Computation` (or a - `computation:` path). Concepts that need the value link to it. Never inline a - number's SQL into the concept that narrates it. - -**Reading a v0.1 bundle?** Two constructs were superseded (§13.1): `timestamp` -is now `generated.at`, and a body `# Citations` list is now `sources`. Read both, -write v0.2 — and when you touch a legacy concept in **maintain** mode, migrate -its frontmatter as part of the edit. The validator warns on both. - -Templates to copy: [concept](templates/concept.md), [index](templates/index.md), -[log](templates/log.md). - -## Default bundle location - -Use `.okf/` at the repository root unless the project already uses another -location. Commit it alongside the code it describes — knowledge as code. - -## Modes - -### produce — create or extend a bundle - -**Starting a brand-new bundle?** Use the init fast-path instead of hand-writing -the first files — it scaffolds a conformant `index.md`, `log.md`, and a -`getting-started.md` concept with full recommended frontmatter in one shot: - -```bash -uv run "${CLAUDE_SKILL_DIR}/scripts/okf_init.py" [--title "..."] -``` - -It refuses to touch a directory that already has `.md` files unless `--force` -is given. Then extend it: - -1. Read [reference/SPEC.md](reference/SPEC.md). -2. Pick the source(s): **code** (derive concepts from source, READMEs, - docstrings, config), **docs/wiki** (distill pages into concepts, record the - originals in `sources`), **manual** (decisions, playbooks, metrics). -3. Choose a directory layout by domain (e.g. `services/`, `datasets/`, - `decisions/`). One concept per file. -4. Write each concept from [templates/concept.md](templates/concept.md): set a - descriptive `type`, fill recommended fields, record `generated` and the - `sources` you actually read, cross-link related concepts. -5. Add/refresh `index.md` per directory (and `okf_version: "0.2"` in the root - index). A new bundle gets a dated **Creation** entry in `log.md`. Extending - an existing one does not. -6. Validate (see below). Fix every error before finishing. - -### maintain — keep a bundle in sync with reality -1. Identify which concepts the change affects (search by `resource`, path, or - topic). This bookkeeping is exactly what agents are good at — touch every - affected file in one pass. -2. Update the body and `generated.at` (with your own actor in `generated.by`); - fix or add cross-links; create new concepts for new assets; mark removed - assets `status: deprecated` and note the deprecation in `log.md` rather than - silently deleting context. Facing a whole v0.1 bundle rather than a stray - field? Do not hand-edit it — run the validator's `--migrate` once. -3. Update the relevant `index.md` files. Append a dated `log.md` entry only if - the change is a lifecycle event (see *What goes in `log.md`*). -4. Validate. - -### consume — use a bundle as context -1. Read the bundle-root `index.md` first for progressive disclosure, then follow - links only into the concepts relevant to the task. -2. Weigh what you read: `status: draft`/`deprecated`, a `stale_after` already - past, or no `verified` entry all mean "check before relying on this". Treat - broken links as not-yet-written knowledge, not errors. -3. Need a number an `Attested Computation` covers? Run *its* computation with - values bound to the declared `parameters` — never write your own query. -4. If you learn something durable while working, switch to **maintain** and - write it back. - -## Validation (do this before declaring done) - -Never eyeball conformance — run the deterministic checker. Invoke the companion -**`validate`** skill (`/okf:validate --strict`), which ships the -checker. If that skill is not installed, run it directly: - -```bash -uv run "${CLAUDE_SKILL_DIR}/../validate/scripts/okf_validate.py" --strict -``` - -Resolve every `ERROR` (hard §11 failures). Under `--strict` warnings also fail -the run (exit 1), so clear them too; drop `--strict` (or use `--max-warnings N`) -if you only want §11 conformance gated. diff --git a/external/okf/skills/okf/reference/APACHE-2.0.txt b/external/okf/skills/okf/reference/APACHE-2.0.txt deleted file mode 100644 index d645695..0000000 --- a/external/okf/skills/okf/reference/APACHE-2.0.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/external/okf/skills/okf/reference/SPEC.md b/external/okf/skills/okf/reference/SPEC.md deleted file mode 100644 index 68629bf..0000000 --- a/external/okf/skills/okf/reference/SPEC.md +++ /dev/null @@ -1,1016 +0,0 @@ - - -# Open Knowledge Format (OKF) - -**Version 0.2** - -OKF is an open, human- and agent-friendly format for representing -*knowledge*: the metadata, context, and curated insight that surrounds -data and systems. It is designed to be authored by people, generated by -agents, exchanged across organizations, and consumed by both. - -The format is intentionally minimal: a directory of markdown files with -YAML frontmatter. There is no schema registry, no central authority, and -no required tooling. If you can `cat` a file, you can read OKF; if you -can `git clone` a repo, you can ship it. - -This document is self-contained: it specifies everything needed to -produce and consume OKF v0.2. A summary of what changed from v0.1 is in -§13. - ---- - -## 1. Motivation - -The space of knowledge representation for AI agents is evolving quickly, -and many incompatible conventions are emerging. OKF takes the position -that knowledge is best represented in commonly accessible, established -formats that are: - -- **Readable** by humans without tooling. -- **Parseable** by agents without bespoke SDKs. -- **Diffable** in version control. -- **Portable** across tools, organizations, and time. - -Increasingly, a knowledge corpus is not authored once and then read: it -is **continuously written and maintained by agents**. When most concepts -are machine-generated, a consumer needs answers that a plain -markdown-plus-frontmatter convention does not make first-class: - -1. What was this created from, and how was it verified? (**provenance**) -2. How much should I trust it? (**trust**) -3. Is it still true? (**freshness**) -4. Is it the current version? (**lifecycle**) -5. Was this number produced the way we said it must be? (**attestation**) - -OKF v0.2 makes provenance, trust, lifecycle, and attestation first-class -while keeping the format minimally opinionated. The format is minimally -opinionated. It standardizes only the small set of structural conventions -needed to make a knowledge corpus self-describing — anything beyond that -is left to the producer. - -### Goals - -1. Define a universal format that **producers** (people, agents, export - pipelines) can write into. -2. Inform how **consumers** (agents, UIs, search indexes, deterministic - code) should read and traverse it. -3. Facilitate **exchange** of knowledge across systems and organizations. -4. Standardize the small set of frontmatter fields that make an - agent-maintained corpus **trustable**, without prescribing any runtime. - -### Non-goals - -- Defining a fixed taxonomy of concept types. -- Prescribing storage, serving, or query infrastructure. -- Replacing domain-specific schemas (Avro, Protobuf, OpenAPI, and so on). - OKF *references* them; it does not subsume them. -- Specifying a packaging or invocation standard for the code an executor - or attester points at. OKF fixes the interface, not the packaging. - ---- - -## 2. Terminology - -- **Knowledge Bundle** (or **bundle**): A self-contained, hierarchical - collection of knowledge documents. The unit of distribution. -- **Concept**: A single unit of knowledge within a bundle, represented as - one markdown document. It may describe a tangible asset (a table, an - API), an abstract idea (a metric, a business process), or anything in - between. -- **Concept ID**: The path of the concept's file within the bundle, with - the `.md` suffix removed. -- **Frontmatter**: A YAML metadata block delimited by `---` at the top of - a markdown file. -- **Body**: Everything in the file after the frontmatter. -- **Link**: A standard markdown link from one concept to another, used to - express relationships beyond the implicit parent/child hierarchy. -- **Source**: A material a concept derives from, external or internal to - the bundle, recorded in the `sources` frontmatter field. -- **Provenance**: The set of sources a concept derives from. -- **Credibility signal**: An objective, per-source fact (`author`, - `usage_count`, `last_modified`) used to infer trust; OKF records the - signals, not a verdict (see §5.1). -- **Actor**: A string identifying who or what performed an action, using - the convention `/` for agents, `human:` for - people, and `process:` for automated processes (see §7). -- **Trust tier**: A level derived from a concept's `verified` field: - unverified, machine-confirmed, or human-reviewed (see §5.3). -- **Attested Computation**: A concept (`type: Attested Computation`) - carrying a sanctioned way to compute a value, so a consumer can confirm - the value was produced by running it (see §10). -- **Executor**: Run instructions or code that executes a computation and - returns a receipt (see §10.2). -- **Receipt**: The evidence a run returns, shaped by `executor.receipt`; a - runtime artifact, not stored in the bundle (see §10). -- **Attester**: Deterministic (no-LLM) code that inspects a receipt and - returns a verdict (see §10.2). - ---- - -## 3. Bundle structure - -A bundle is a directory tree of markdown files. The directory structure -is independent of the domain: producers organize concepts however makes -sense for the knowledge being captured. - -``` -path/to/bundle/ - index.md # Optional. Directory listing for progressive disclosure. - log.md # Optional. Chronological history of updates. - .md # A concept at the bundle root. - / # Subdirectories organize concepts into groups. - index.md - .md - / - ... -``` - -A bundle MAY be distributed as: - -- A git repository (recommended, since it provides history, attribution, - and diffs). -- A tarball or zip archive of the directory. -- A subdirectory within a larger repository. - -### 3.1 Reserved filenames - -The following filenames have defined meaning at any level of the -hierarchy and MUST NOT be used for concept documents: - -| Filename | Purpose | -|------------|----------------------------------| -| `index.md` | Directory listing. See §8. | -| `log.md` | Update history. See §9. | - -All other `.md` files are concept documents. - -Tags remain a first-class concept through the `tags` frontmatter field -(§4.1). OKF does not specify a separate file format for aggregating -documents by tag; a consumer that wants a tag-browsing view can -synthesize one at consumption time by scanning frontmatter. - ---- - -## 4. Concept documents - -Every concept is a UTF-8 markdown file with two parts: - -1. A **YAML frontmatter block**, delimited by `---` on its own line at the - start of the file and a closing `---` on its own line. -2. A **markdown body**, containing free-form content. - -### 4.1 Frontmatter - -```yaml ---- -type: # REQUIRED -title: -description: -resource: -tags: [, , ...] # Optional -# ... trust, lifecycle, provenance, and computation families (see §5, §10) -# ... other producer-defined key/value pairs ---- -``` - -**Required:** - -- `type`: A short string identifying the kind of concept. Consumers use it - for routing, filtering, and presentation. Example values: - `BigQuery Table`, `BigQuery Dataset`, `API Endpoint`, `Metric`, - `Playbook`, `Reference`, `Attested Computation`. - - Type values are **not** registered centrally. Producers SHOULD pick - values that are descriptive and self-explanatory; consumers MUST - tolerate unknown types gracefully, typically by treating them as generic - concepts. - -`type` is the only always-required key; a concept carrying just `type` is -fully conformant (§11). - -**Recommended:** - -- `title`: Human-readable display name. If omitted, consumers MAY derive a - title from the filename. -- `description`: A single sentence summarizing the concept. Used by - `index.md` generators, search snippets, and previews. -- `resource`: A URI that uniquely identifies the underlying asset the - concept describes. Absent for concepts that describe abstract ideas - rather than physical resources. -- `tags`: A YAML list of short strings for cross-cutting categorization. - -The optional **provenance**, **trust**, and **lifecycle** families (§5) and -the **computation** fields for Attested Computation concepts (§10) may also -appear. - -**Extensions:** Producers MAY include any additional keys. Consumers -SHOULD preserve unknown keys when round-tripping and MUST NOT reject -documents with unrecognized fields. - -### 4.2 Body - -The body is standard markdown. Producers SHOULD favor structural markdown -(headings, lists, tables, fenced code blocks) over freeform prose, since -structure aids both human reading and agent retrieval. - -There are no required body sections. The following headings have -**conventional** meaning and SHOULD be used when applicable: - -| Heading | Purpose | -|-----------------|--------------------------------------------------------| -| `# Schema` | Structured description of an asset's columns/fields. | -| `# Examples` | Concrete usage examples, often as fenced code blocks. | -| `# Computation` | The sanctioned computation of an Attested Computation. See §10. | - -Per-claim attribution to external sources uses markdown footnotes keyed to -`sources` entries rather than a body citations list (§5.1). - -### 4.3 Example: a concept bound to a resource - -```markdown ---- -type: BigQuery Table -title: Customer Orders -description: One row per completed customer order across all channels. -resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=orders -tags: [sales, orders, revenue] -generated: { by: reference_agent/gemini-2.5-pro, at: 2026-05-28T14:30:00Z } ---- - -# Schema - -| Column | Type | Description | -|---------------|-----------|------------------------------------------| -| `order_id` | STRING | Globally unique order identifier. | -| `customer_id` | STRING | Foreign key into [customers](/tables/customers.md). | -| `total_usd` | NUMERIC | Order total in US dollars. | -| `placed_at` | TIMESTAMP | When the customer submitted the order. | - -# Joins - -Joined with [customers](/tables/customers.md) on `customer_id`. -``` - -### 4.4 Example: a concept not bound to a resource - -```markdown ---- -type: Playbook -title: "Incident response: data freshness alert" -description: Steps to triage a freshness alert on the orders pipeline. -tags: [oncall, incident] -generated: { by: human:ahormati, at: 2026-04-12T09:00:00Z } ---- - -# Trigger - -A freshness alert fires when `orders` lags more than 30 minutes behind its -expected SLA. See the [orders table](/tables/orders.md). - -# Steps - -1. Check the [ingestion job dashboard](https://example.com/dash). -2. ... -``` - ---- - -## 5. Provenance, trust, and lifecycle - -These frontmatter families make "where did this come from," "how much -should I trust it," and "is it still current" answerable from frontmatter. -All are optional. Their absence carries meaning: an unverified concept is -distinguishable from a verified one, but is never rejected (§11). - -Every timestamp-valued key in OKF is an ISO 8601 datetime with an explicit -UTC offset, for example `2026-06-30T14:00:00Z`. - -### 5.1 Provenance: `sources` - -`sources` records the materials a concept derives from, external or -internal to the bundle. - -```yaml -sources: - - id: ga4-schema - resource: https://developers.google.com/analytics/bigquery/export-schema - title: GA4 BigQuery Export schema - author: team:ga4-docs - usage_count: 5000 - last_modified: 2026-05-30T00:00:00Z -usage_window: { from: 2026-06-01T00:00:00Z, to: 2026-06-30T00:00:00Z } -``` - -Each `sources` entry: - -- `resource`: REQUIRED within an entry. Names either a concrete artifact a - consumer can follow (an absolute URL, a bundle-relative path, or a path - into a `references/` subdirectory, §6) or a population or scope descriptor - it cannot (for example `all queries in BigQuery project X`). -- `id`: Optional. A stable key used to attribute individual claims (see - below). SHOULD be present when the body cites the source. -- `title`: Optional. Human-readable label for the source. -- The optional credibility signals `author`, `usage_count`, and - `last_modified`, described next. - -**Source credibility signals.** OKF records objective, per-source signals -so a consumer can judge how much to trust a concept by judging the sources -it was extracted from. It does not store a credibility score: a score is -subjective, unportable across consumers, and goes stale. Credibility is -*inferred* from the signals, the same way trust tiers are (§5.3), not -stored. Each signal is optional and lives on a `sources` entry: - -- `author`: Who or what produced the source, in the actor convention (§7). - An authority signal. -- `usage_count`: How often `resource` was exercised (dashboard views, query - executions, page reads) over `usage_window`. An adoption and liveness - signal. For a single artifact it is that artifact's own exercise count; - for a scope descriptor it is the number of exercises within the scope that - touch the concept. -- `last_modified`: When the source itself last changed. A recency signal, - distinct from `generated.at` (§5.2), which records when the concept was - written. -- `usage_window`: Written once as a sibling of `sources`, it frames every - `usage_count` with a `{ from, to }` datetime range. A single entry MAY - carry its own `usage_window` to override the shared one. - -`usage_count` is a coarse signal. It is comparable at the -alive-versus-dead and order-of-magnitude level, and against a source's own -history over time, but not as a precise cross-kind ranking: a scheduled -query's executions and a human's deliberate dashboard views do not carry -equal weight. Consumers SHOULD read it as liveness and trend, not as a -score. - -Lineage is expressed through links, not a dedicated field. When a -`resource` points at another OKF concept, the derivation edge already -exists in the bundle graph (§6), so a consumer MAY recurse into that -source's own `sources` and let credibility propagate. External leaf sources -carry only their intrinsic signals. Deeper lineage (an explicit external -`derived_from`, or data lineage) is out of scope for v0.2. - -**Per-claim attribution.** To attribute a specific claim, use a markdown -footnote whose label is a `sources[].id`: - -```markdown -The `events_` table is sharded daily as `events_YYYYMMDD`.[^ga4-schema] - -[^ga4-schema]: GA4 BigQuery Export schema -``` - -The footnote label is the join key into `sources`; consumers resolve -attribution through the matching entry, not by parsing the footnote prose. -Labels are keyed rather than positional (`sources[0]`) because agents -constantly rewrite these documents: a positional index misattributes -silently the moment the list is reordered, whereas a stable `id` survives -reordering. - -### 5.2 Trust: `generated` and `verified` - -`generated` records how the current content was produced. `verified` -records who or what has confirmed the content against its sources or -`resource`. They are kept distinct because who *wrote* a concept need not -be who *confirmed* it. - -```yaml -generated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-20T22:53:05Z } -``` - -- `generated.by`: REQUIRED within `generated`. An actor (§7). -- `generated.at`: An ISO 8601 datetime marking the content's last - meaningful change. Consumers use it to tell a recent edit from a stale - fact. - -```yaml -verified: - - { by: human:ahormati, at: 2026-06-25T09:00:00Z } - - { by: process:finance-nightly, at: 2026-06-26T02:00:00Z } -``` - -- `verified`: A list of verification events, each with `by` (an actor) and - `at` (an ISO 8601 datetime). Multiple entries capture independent - checks, for example a human sign-off plus a nightly process. "How - recently" is the latest `at`. -- `verified` is independent of `generated.at`: content can change without - re-confirmation, and facts can be re-confirmed without regeneration. -- A single verifier MAY be written as one `{ by, at }` mapping without the - list dash. Consumers MUST treat a bare mapping as a one-element list: - -```yaml -verified: { by: human:ahormati, at: 2026-06-25T09:00:00Z } -``` - -### 5.3 Trust tiers - -Consumers derive a trust tier from `verified`, lowest to highest: - -- No `verified` key ⇒ **unverified**. -- `verified` by non-`human:` actors only ⇒ **machine-confirmed**. -- `verified` by a `human:` actor ⇒ **human-reviewed**. - -A concept with no trust frontmatter is still consumable; consumers MUST -NOT reject it (§11). Trust tiers are advisory signals, not access control. - -### 5.4 Lifecycle: `status` - -```yaml -status: stable # draft | stable | deprecated -``` - -- `draft`: not yet reviewed; possibly incomplete. -- `stable`: default; ready for consumption. -- `deprecated`: kept for links and history; no longer current. - -Absent `status` ⇒ `stable`. - -### 5.5 Lifecycle: `stale_after` - -```yaml -stale_after: 2026-09-23T00:00:00Z # content is stale on/after this instant -``` - -Optional. An absolute instant. A concept is stale when -`now >= stale_after`. An absolute instant, not a relative TTL, keeps the -staleness decision a plain comparison with no reference to when the -concept was read. - ---- - -## 6. Cross-linking and paths - -### 6.1 Links between concepts - -Concepts MAY link to other concepts using standard markdown links. Two -forms are supported: - -- **Absolute (bundle-relative):** begins with `/`, interpreted relative to - the bundle root. This is the **recommended** form because it is stable - when documents are moved within their subdirectory. - - ```markdown - See the [customers table](/tables/customers.md) for the join key. - ``` - -- **Relative:** a standard markdown relative path. - - ```markdown - See the [neighboring concept](./other.md). - ``` - -A link from concept A to concept B asserts a *relationship*. The specific -kind (parent/child, references, joins-with, depends-on) is conveyed by the -surrounding prose, not by the link itself. Consumers that build a graph -view typically treat all links as directed edges of an untyped -relationship. - -Consumers MUST tolerate broken links: a link whose target does not exist -in the bundle is not malformed; it may simply represent not-yet-written -knowledge. - -### 6.2 Path-valued fields - -Several fields name a path or URI: `resource`, `sources[].resource`, -`computation`, `executor.resource`, and `attester.resource` (§10). A -`sources[].resource` may instead be a scope descriptor (§5.1), in which -case it is not a path. Each path-valued field accepts: - -- an absolute URL (for example `https://...`), -- a bundle-relative path beginning with `/`, or -- a relative path (for example `../computations/revenue.md`). - -### 6.3 The `references/` convention - -A `references/` subdirectory conventionally mirrors external material, run -instructions, or code as first-class concepts within the bundle. Sources, -executors, and attesters commonly point into it (for example -`references/attesters/revenue.py`). It is a naming convention, not a -requirement. - ---- - -## 7. Actor convention - -Fields that record an identity (`generated.by`, `verified[].by`) use a -single actor convention: - -- `/` for agents and tools, for example - `reference_agent/gemini-2.5-pro`. -- `human:` for a person, for example `human:ahormati`. -- `process:` for an automated process, for example - `process:finance-nightly`. - -Consumers that classify trust (§5.3) key off the `human:` prefix, so -producers MUST use it for hand-authored or human-confirmed content. - ---- - -## 8. Index files - -An `index.md` file MAY appear in any directory, including the bundle root. -It enumerates the directory's contents to support **progressive -disclosure**: letting a human or agent see what is available before -opening individual documents. - -Index files contain no frontmatter, with one exception: a bundle-root -`index.md` MAY carry an `okf_version` key (§12). The body uses one or more -sections, each grouping concepts under a heading: - -```markdown -# Section / Group Heading - -* [Title 1](relative-url-1) - short description of item 1 -* [Title 2](relative-url-2) - short description of item 2 - -# Another Section - -* [Subdirectory](subdir/) - short description of the subdirectory -``` - -Entries SHOULD include the description from the linked concept's -frontmatter. Producers MAY generate `index.md` automatically; consumers -MAY synthesize one on the fly when none is present. - ---- - -## 9. Log files - -A `log.md` file MAY appear at any level of the hierarchy to record the -history of changes to that scope. The format is a flat list of -date-grouped entries, newest first: - -```markdown -# Directory Update Log - -## 2026-05-22 -* **Update**: Added a BigQuery table reference for [Customer Metrics](/tables/customer-metrics.md). -* **Creation**: Established the [Dataplex Playbook](/playbooks/dataplex.md). - -## 2026-05-15 -* **Initialization**: Created foundational directory structure. -``` - -Date headings MUST use ISO 8601 `YYYY-MM-DD` form. Log entries are prose; -the leading bold word (`**Update**`, `**Creation**`, `**Deprecation**`) is -a convention, not a requirement. - ---- - -## 10. Attested computations concept - -An Attested Computation concept carries not just what a value *means* but a -sanctioned way to *compute* it, so a consumer can confirm the agent ran the -blessed computation instead of improvising its own. Provenance (§5.1) -answers "where did this claim come from"; attestation answers "was this -number produced the way we said it must be." OKF records the computation -and the means to check it; it does not execute anything itself. - -### 10.1 A computation is its own concept - -A sanctioned computation is a standalone concept of -`type: Attested Computation`. A concept that needs the value (a `Metric`, a -`BigQuery Table`) links to it with a normal markdown link (§6). Three -properties motivate the standalone concept: - -- **`runtime` defines what `parameters` mean.** A parameter is a SQL bind - variable, a dbt var, or a Python argument depending on the runtime. - Keeping `runtime` and `parameters` in one frontmatter makes the binding - semantics self-evident. -- **One computation, many consumers.** The same computation can back a - metric, a dashboard concept, and a report; as a concept it is referenced - once and reused. -- **Trust state is per computation.** `verified`, `stale_after`, and a - single `attester` describe one thing. Revenue, profit, and margin each - verify and attest independently, which is three concepts, not three - entries in one frontmatter. - -### 10.2 Contract fields - -The contract is the concept's top-level frontmatter. In addition to the -provenance, trust, and lifecycle families (§5), an Attested Computation -concept carries: - -- `runtime`: REQUIRED for this type. The single field that says how to run - the computation, and so how the executor and attester interpret it and - what `parameters` mean. Example values: `bigquery`, `postgres`, `dbt`, - `python`, `Looker`. -- `parameters`: A list of the typed, named holes the agent may fill. Each - entry: `{ name, type, required }`. Binding semantics follow `runtime`. -- `computation`: Optional. A path (§6.2) to a file holding the - computation, used instead of an inline body fence (see §10.3). Absent ⇒ - the body `# Computation` fence is the computation. -- `executor`: How the computation is run. `resource` names run - instructions or code; a runner (an agent, or deterministic consumer - code) follows it. `receipt` declares the fields a run must return, the - evidence the attester inspects (for example a BigQuery `job_id` and the - SQL the job actually executed). -- `attester`: The deterministic check. `resource` names code (no LLM) that - takes a receipt and returns a verdict. It is meant to run consumer-side. - -What sits behind a `resource` (a Skill, a script, a container) is a -packaging choice; OKF fixes the interface, not the packaging (§1). - -```markdown ---- -type: Attested Computation -title: Revenue for fiscal year -description: Recognized revenue for a fiscal year, per Finance's definition. -status: stable -runtime: bigquery -parameters: - - { name: year, type: integer, required: true } -executor: - resource: references/skills/run-on-bq.md - receipt: [job_id, executed_sql, result] -attester: - resource: references/attesters/revenue.py -generated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-20T22:53:05Z } -verified: { by: human:ahormati, at: 2026-06-25T09:00:00Z } -stale_after: 2026-09-23T00:00:00Z -sources: - - id: rev-policy - resource: https://wiki.acme/finance/revenue-recognition - title: Revenue recognition policy ---- - -# Computation - - SELECT SUM(amount) AS revenue - FROM finance.recognized_revenue - WHERE fiscal_year = @year - -The computation binds only the declared `parameters`, per the recognition -policy.[^rev-policy] - -[^rev-policy]: Revenue recognition policy -``` - -### 10.3 The computation - -Provide the computation in one of two ways: - -- **Inline:** a single fenced code block in the body under `# Computation`. - Best for a short computation reviewed alongside the contract. -- **File:** set `computation` to a path (§6.2) and omit the body fence. - Best for a long or generated computation, or one already kept as a real - file shared with non-OKF tooling. - -```yaml -runtime: bigquery -computation: references/computations/lib/revenue.sql -parameters: - - { name: year, type: integer, required: true } -``` - -The agent MAY only supply *values* for the declared `parameters`; it MUST -NOT author or edit the computation. Binding `computation` with the -parameter values into the executable artifact is the consumer's job, and -the attester independently re-derives that same binding to compare against -what actually ran. Because the comparison is on the expanded, compiled -artifact the receipt carries (`executed_sql`, `compiled_sql`), a rewritten -query, a swapped computation file, or a mutated dependency fails the check. -A typed, parameter-only surface is what makes "did the sanctioned thing -run" a mechanical comparison rather than a judgement call. - -### 10.4 Concepts that use a computation - -A document is rarely a single computation. An income-statement overview -that discusses revenue, profit, and margin stays one readable concept and -links to one Attested Computation per figure: - -```markdown ---- -type: Metric -title: Revenue -description: Recognized revenue for a fiscal year. -tags: [finance, revenue] -status: stable -generated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-20T22:53:05Z } ---- - -# Definition - -Recognized revenue sums `amount` over rows booked to the fiscal year, -computed by [the revenue computation](../computations/revenue.md). -``` - -Because each computation is its own concept, revenue can be fresh while -profit is past its `stale_after`, and each attests on its own run. -Co-locating them is a directory choice (a `computations/` folder with an -`index.md`), not a frontmatter one. - -### 10.5 How a consumer uses it (informative) - -This subsection is informative, not normative. The runtime artifacts below -are **not** stored in the bundle. - -1. **Discover** via `type: Attested Computation`, a frontmatter signal - liftable into `index.md`; a consumer reaches one directly or by - following a link from a concept that uses it. -2. **Load** the contract from frontmatter and the computation from the - body (or the file named by `computation`). -3. **Parameterize**: the agent supplies values for the declared parameters. -4. **Execute**: the executor runs the bound computation and returns a - receipt shaped by `executor.receipt`. -5. **Attest**: the consumer runs the attester over the receipt. It - confirms provenance (the computation that ran equals `computation` bound - with the claimed parameters, not agent-authored SQL) and fidelity (the - displayed value matches the receipt's authoritative source, re-read by - job id rather than taken from the agent's text). -6. **Gate**: refuse to display a failing attestation; warn or refuse when - `now >= stale_after`. On success, surface the verdict (for example a - link to the job log) so trust is visible. - -### 10.6 Verification versus attestation - -`verified` (§5.2) and attestation are distinct, and both exist: - -- `verified` confirms the *definition* still matches policy. It is - doc-level, slow, and recorded in the bundle. -- Attestation confirms a single *run* produced the value the sanctioned - way. It is per-call, runtime, and not stored in the bundle. - -A concept with a stale definition can still attest cleanly, and a -freshly-verified definition still requires attestation on each run, which -is why both are needed. - ---- - -## 11. Conformance - -A bundle is **conformant** with OKF v0.2 if: - -1. Every non-reserved `.md` file in the tree contains a parseable YAML - frontmatter block. -2. Every frontmatter block contains a non-empty `type` field. -3. Every reserved filename (`index.md`, `log.md`) follows the structure in - §8 and §9 respectively when present. - -When the trust, lifecycle, provenance, or computation families are -present, producers SHOULD follow §5 through §10, and consumers: - -- MUST treat a bare `verified` mapping as a one-element list (§5.2). -- MUST NOT reject a concept for missing any optional family (§5.3). -- SHOULD derive trust tiers and staleness only from the fields specified - here, and SHOULD surface, not silently drop, a failing attestation - (§10.5). - -Consumers SHOULD treat all other constraints as soft guidance. In -particular, consumers MUST NOT reject a bundle because of: - -- Missing optional frontmatter fields. -- Unknown `type` values. -- Unknown additional frontmatter keys. -- Broken cross-links. -- Missing `index.md` files. - ---- - -## 12. Versioning - -This document specifies OKF version **0.2**. Revisions are versioned as -`.`: - -- A **minor** version bump introduces backward-compatible additions (new - optional fields, new conventional section headings). -- A **major** version bump may make breaking changes (renaming required - fields, changing reserved filenames). - -Bundles MAY declare the version they target with `okf_version: "0.2"` in a -bundle-root `index.md` frontmatter block (the only place frontmatter is -permitted in an `index.md`). Consumers that do not understand the declared -version SHOULD attempt best-effort consumption rather than refusing the -bundle. - -### Considered and deferred - -The following are intentionally left to a future revision: - -- The full runtime protocol: receipt and verdict wire formats, and the - attestation lifecycle around a run. -- The attester ABI, portability, and sandboxing, likely bundled with - future work on serving and Skills. -- Attestation caching. -- Semantic-layer templates (Looker, dbt) where the attester comparison - shifts from SQL equality to model-and-binding equality. - ---- - -## 13. Changes from v0.1 - -v0.2 supersedes OKF v0.1 and is a minor version bump under §12, except for -two deliberate breaking changes called out below because they rename or -retire v0.1 fields. A v0.1 bundle is consumable by a v0.2 consumer under -the fallbacks noted here. - -### 13.1 Breaking changes - -- **`timestamp` is superseded by `generated.at`.** A concept's last - content change is now recorded as `generated: { by, at }` (§5.2). - Consumers MAY fall back to a legacy `timestamp` when `generated` is - absent. -- **The body `# Citations` list is superseded by `sources`.** Provenance - moves to frontmatter (§5.1). Consumers SHOULD read `sources` and MAY - still parse a legacy `# Citations` body list for v0.1 documents. - -### 13.2 Additive changes - -All of the following are additive: new optional keys, one new concept -type, and one new conventional heading. Their absence yields a plain v0.1 -concept. - -- New frontmatter families: `sources` with its per-source credibility - signals (`author`, `usage_count`, `last_modified`) and the `usage_window` - sibling; `generated`, `verified`; `status`, `stale_after` (§5). -- New concept type `Attested Computation` and its computation keys - `runtime`, `parameters`, `computation`, `executor`, `attester` (§10). -- New conventional body heading `# Computation` (§4.2). -- The actor convention for `generated.by` and `verified[].by` (§7). - -Everything else (bundle structure, reserved filenames, the required -`type`, recommended `title`/`description`/`resource`/`tags`, cross-linking, -index files, log files, permissive conformance) is carried forward -unchanged. - ---- - -## Appendix A: Worked example, an income statement - -One bundle exercising every family, shown as a v0.1 to v0.2 migration of an -income statement with two figures, revenue and gross profit. - -### v0.1 form - -A single doc: both figures in one concept, the SQL in prose an agent can -read, ignore, or rewrite, citations a flat list, and the only timestamp is -`timestamp`. - -```markdown ---- -type: Metric -title: Income statement (fiscal year) -description: Headline income-statement figures for a fiscal year. -tags: [finance, income-statement] -timestamp: '2026-05-28T22:53:05+00:00' ---- - -# Definition -The income statement reports revenue and gross profit for a fiscal year. - -# Revenue -Recognized revenue sums `amount` over rows booked to the fiscal year: - - SELECT SUM(amount) AS revenue - FROM finance.recognized_revenue - WHERE fiscal_year = - -# Gross profit -Gross profit by segment, per the cost-allocation standard: - - SELECT gross_profit FROM fct_income_statement - WHERE fiscal_year = AND segment = - -# Citations -- https://wiki.acme/finance/fpa-handbook -- https://wiki.acme/finance/revenue-recognition -- https://wiki.acme/finance/cost-allocation -``` - -### v0.2 form - -The two figures split into attested computations linked from a narrative -concept. Every family is populated, and the two computations sit in -deliberately different states so one consumer reaches two verdicts. - -``` -bundles/finance/ - metrics/income-statement.md type: Metric (narrates, links both) - computations/revenue.md type: Attested Computation (runtime: bigquery) - computations/profit.md type: Attested Computation (runtime: dbt) - references/skills/run-on-bq.md, run-dbt.md - references/attesters/sql-equality.py, dbt-binding.py -``` - -`metrics/income-statement.md`, the readable doc; trust lives on what it -links, not here: - -```markdown ---- -type: Metric -title: Income statement (fiscal year) -description: Headline income-statement figures for a fiscal year. -tags: [finance, income-statement] -status: stable -generated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-20T22:53:05Z } -verified: { by: human:ahormati, at: 2026-06-25T09:00:00Z } -stale_after: 2026-12-31T00:00:00Z -sources: - - id: fpa-handbook - resource: https://wiki.acme/finance/fpa-handbook - title: FP&A reporting handbook ---- - -# Definition -The income statement reports [revenue](../computations/revenue.md) and -[gross profit](../computations/profit.md) for a fiscal year, per the FP&A -reporting handbook.[^fpa-handbook] Each figure is produced by a sanctioned, -attestable computation; this concept only narrates them. - -[^fpa-handbook]: FP&A reporting handbook -``` - -`computations/revenue.md`, BigQuery SQL, human-verified, fresh, and -corroborated by a live dashboard source carrying credibility signals: - -```markdown ---- -type: Attested Computation -title: Revenue for fiscal year -description: Recognized revenue for a fiscal year, per Finance's definition. -tags: [finance, revenue] -status: stable -runtime: bigquery -parameters: - - { name: year, type: integer, required: true } -executor: - resource: references/skills/run-on-bq.md - receipt: [job_id, executed_sql, result] -attester: - resource: references/attesters/sql-equality.py -generated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-28T14:00:00Z } -verified: { by: human:ahormati, at: 2026-06-25T09:00:00Z } -stale_after: 2026-12-31T00:00:00Z -sources: - - id: rev-policy - resource: https://wiki.acme/finance/revenue-recognition - title: Revenue recognition policy - author: team:finance-fpa - last_modified: 2026-04-02T00:00:00Z - - id: exec-rev-dash - resource: dashboards/exec-revenue - title: Executive revenue dashboard - author: team:finance-fpa - usage_count: 5000 - last_modified: 2026-06-18T00:00:00Z -usage_window: { from: 2026-06-01T00:00:00Z, to: 2026-06-30T00:00:00Z } ---- - -# Computation - - SELECT SUM(amount) AS revenue - FROM finance.recognized_revenue - WHERE fiscal_year = @year - -Recognized revenue per the recognition policy,[^rev-policy] corroborated by -the executive revenue dashboard.[^exec-rev-dash] - -[^rev-policy]: Revenue recognition policy -[^exec-rev-dash]: Executive revenue dashboard -``` - -`computations/profit.md`, a dbt model, process-verified, and past its -`stale_after`: - -```markdown ---- -type: Attested Computation -title: Gross profit for fiscal year -description: Gross profit by segment for a fiscal year, per the cost-allocation standard. -tags: [finance, profit] -status: stable -runtime: dbt -parameters: - - { name: year, type: integer, required: true } - - { name: segment, type: string, required: true } -executor: - resource: references/skills/run-dbt.md - receipt: [run_id, compiled_sql, result] -attester: - resource: references/attesters/dbt-binding.py -generated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-14T14:00:00Z } -verified: { by: process:finance-nightly, at: 2026-06-12T08:00:00Z } -stale_after: 2026-06-15T00:00:00Z -sources: - - id: cost-alloc - resource: https://wiki.acme/finance/cost-allocation - title: Cost allocation standard ---- - -# Computation - - SELECT gross_profit - FROM {{ ref('fct_income_statement') }} - WHERE fiscal_year = {{ var('year') }} - AND segment = {{ var('segment') }} - -Gross profit by segment per the cost-allocation standard.[^cost-alloc] - -[^cost-alloc]: Cost allocation standard -``` diff --git a/external/okf/skills/okf/scripts/okf_init.py b/external/okf/skills/okf/scripts/okf_init.py deleted file mode 100644 index ec8dcb2..0000000 --- a/external/okf/skills/okf/scripts/okf_init.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.11" -# dependencies = ["pyyaml>=6"] -# /// -"""Scaffold a conformant starter Open Knowledge Format (OKF) v0.2 bundle. - -Creates a root `index.md` (frontmatter carries only `okf_version`), a `log.md` -with a creation entry under today's ISO date heading, and one starter concept -(`getting-started.md`) with the full set of recommended frontmatter fields. -The scaffold is meant to be exemplary: `okf_validate.py --strict` should pass -it with zero warnings. - -Run: uv run scripts/okf_init.py [--title "..."] [--force] -""" -from __future__ import annotations - -import argparse -import sys -from datetime import datetime, timezone -from pathlib import Path - -import yaml - -OKF_VERSION = "0.2" -STARTER_CONCEPT = "getting-started.md" -# Actor convention (§7): an automated process, not an agent with a version. -ACTOR = "process:okf_init" - - -def humanize(name: str) -> str: - words = name.replace("-", " ").replace("_", " ").strip() - return words.title() if words else "Untitled" - - -def frontmatter(meta: dict) -> str: - return "---\n" + yaml.safe_dump(meta, sort_keys=False, allow_unicode=True) + "---\n" - - -def build_index(title: str) -> str: - meta = frontmatter({"okf_version": OKF_VERSION}) - return ( - f"{meta}\n" - f"# {title}\n\n" - f"* [Getting started]({STARTER_CONCEPT}) - starting point for this bundle.\n" - ) - - -def build_log(today: str, title: str) -> str: - return ( - "# Update Log\n\n" - f"## {today}\n" - f"* **Creation**: Scaffolded the {title} bundle with `okf_init.py` — " - f"see [getting started]({STARTER_CONCEPT}).\n" - ) - - -def build_concept(title: str, now_iso: str) -> str: - meta = frontmatter({ - "type": "Reference", - "title": f"Getting started — {title}", - "description": f"Starting point for the {title} OKF bundle.", - "tags": ["getting-started"], - "status": "stable", - "generated": {"by": ACTOR, "at": now_iso}, - }) - return ( - f"{meta}\n" - "# Overview\n\n" - "This is the first concept in a freshly scaffolded OKF bundle. Replace " - "it with real knowledge — one concept per file, cross-linked with " - "standard markdown links — and keep `index.md` updated as you go. " - "`log.md` records lifecycle events only: a concept created, " - "deprecated, superseded, regenerated or verified.\n" - ) - - -def main() -> int: - ap = argparse.ArgumentParser(description=f"Scaffold a starter OKF v{OKF_VERSION} bundle.") - ap.add_argument("target", type=Path, help="directory to create the bundle in") - ap.add_argument("--title", default=None, help="bundle title (default: humanized target dir name)") - ap.add_argument("--force", action="store_true", help="scaffold even if the target already has .md files") - args = ap.parse_args() - - if args.target.exists() and not args.target.is_dir(): - print(f"error: {args.target} exists and is not a directory", file=sys.stderr) - return 2 - - existing_md = list(args.target.rglob("*.md")) if args.target.is_dir() else [] - if existing_md and not args.force: - print(f"refusing: {args.target} already contains .md files (pass --force to scaffold anyway)", file=sys.stderr) - for p in sorted(existing_md)[:5]: - print(f" {p.relative_to(args.target)}", file=sys.stderr) - return 1 - - title = args.title or humanize(args.target.resolve().name) - today = datetime.now(timezone.utc).date().isoformat() - now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - args.target.mkdir(parents=True, exist_ok=True) - (args.target / "index.md").write_text(build_index(title), encoding="utf-8") - (args.target / "log.md").write_text(build_log(today, title), encoding="utf-8") - (args.target / STARTER_CONCEPT).write_text(build_concept(title, now_iso), encoding="utf-8") - - validator = Path(__file__).resolve().parents[2] / "validate" / "scripts" / "okf_validate.py" - print(f"created OKF bundle scaffold at {args.target}") - print(f"hint: validate it — uv run {validator} {args.target} --strict (or the validate skill)") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/external/okf/skills/okf/templates/concept.md b/external/okf/skills/okf/templates/concept.md deleted file mode 100644 index 20bfde6..0000000 --- a/external/okf/skills/okf/templates/concept.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -type: -title: -description: -resource: -tags: [, ] -status: stable # draft | stable | deprecated; absent means stable -generated: { by: , at: } -verified: { by: human:, at: } # omit until someone confirms it -stale_after: # omit when the content does not expire -sources: # what this was derived from; omit if nothing - - id: - resource: - title: - author: # optional credibility signal - last_modified: # when the source itself last changed ---- - - - -# Overview - - - -# Schema - - - -| Field | Type | Description | -|-------|------|-------------| -| | | | - -[^short-key]: diff --git a/external/okf/skills/okf/templates/index.md b/external/okf/skills/okf/templates/index.md deleted file mode 100644 index 56ba3be..0000000 --- a/external/okf/skills/okf/templates/index.md +++ /dev/null @@ -1,8 +0,0 @@ -# - -* [](<relative-url>) - <short description from the concept's frontmatter> -* [<Title>](<relative-url>) - <short description> - -# <Another Group> - -* [<Subdirectory>](<subdir>/) - <short description of the subdirectory> diff --git a/external/okf/skills/okf/templates/log.md b/external/okf/skills/okf/templates/log.md deleted file mode 100644 index e857119..0000000 --- a/external/okf/skills/okf/templates/log.md +++ /dev/null @@ -1,8 +0,0 @@ -# Update Log - -## <YYYY-MM-DD> -* **Creation**: <what was created> — [<concept>](<bundle-relative-path>). -* **Deprecation**: <what was retired, and what replaces it> — [<concept>](<bundle-relative-path>). -* **Supersession**: [<new concept>](<bundle-relative-path>) supersedes [<old concept>](<bundle-relative-path>). Affirmed by <actor>. -* **Regeneration**: <which concepts, after what source change> — [<concept>](<bundle-relative-path>). -* **Verification**: <what was verified, by whom> — [<concept>](<bundle-relative-path>). diff --git a/external/okf/skills/validate/SKILL.md b/external/okf/skills/validate/SKILL.md deleted file mode 100644 index 881ecbf..0000000 --- a/external/okf/skills/validate/SKILL.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -name: validate -description: >- - Check that an Open Knowledge Format (OKF) bundle is conformant with the v0.2 - spec (§11). Use when asked to validate, lint, or check an OKF bundle, or before - committing changes to one. Runs a deterministic Python checker — not an - eyeball pass. Also migrates a v0.1 bundle to v0.2 in place with `--migrate`. -user-invocable: true -argument-hint: "[bundle-dir] [--strict | --max-warnings N] [--migrate]" -allowed-tools: Bash ---- - -# Validate an OKF bundle - -Run the deterministic conformance checker against the target bundle. Default to -the project's `.okf/` directory when no path is given. - -```bash -uv run "${CLAUDE_SKILL_DIR}/scripts/okf_validate.py" $ARGUMENTS -``` - -If `uv` is unavailable, fall back to: - -```bash -python3 -m pip install --quiet pyyaml && \ -python3 "${CLAUDE_SKILL_DIR}/scripts/okf_validate.py" $ARGUMENTS -``` - -`${CLAUDE_SKILL_DIR}` resolves whether this skill runs as part of the `okf` -plugin or is installed standalone (e.g. via `npx skills add`), so the checker is -always found alongside the skill. - -Interpret the result: - -- **ERROR** → a hard §11 conformance failure (no parseable frontmatter, a - missing/empty `type`, or a file the checker cannot read as UTF-8). The bundle - is non-conformant. Fix every one. - -Only `index.md` and `log.md` are reserved: §11.1 makes *every* other `.md` in -the tree a concept, convention files included. An `AGENTS.md` or `README.md` -dropped next to a bundle is therefore an ERROR until it carries frontmatter with -a `type` (`Document` reads well), which is the fix; a skip-list would put the -checker out of conformance. Move the file above the bundle root if it is not -knowledge. -- **warn** → soft guidance (missing recommended field, non-ISO log date, broken - cross-link, a malformed v0.2 family, a footnote naming no source, an actor - that misses the §7 shapes, an Attested Computation missing `runtime`, a - computation path that resolves nowhere, frontmatter in a reserved file (§8, - §9), a root `index.md` carrying extra keys or declaring a different - `okf_version` (§12)). Never blocks; broken links in particular are explicitly - tolerated by the spec (§6.1). Fix when cheap. - -One warning is worth more than the others: a §7 near-miss such as `Human:dana` -or `human/dana`. §5.3 keys trust tiers off the exact lowercase `human:` prefix, -so the concept silently reads as machine-confirmed when a person did review it. -Fix that one on sight. - -v0.1 bundles validate too: a legacy `timestamp` or `# Citations` section is -reported as a warning naming its v0.2 replacement (`generated.at`, `sources`), -never as an error (§13.1). Under `--strict` those warnings do fail the run — -that is the migration nudge, and `--migrate` is the door. - -## Migrating a v0.1 bundle - -`--migrate` **rewrites the bundle in place** before validating — the one mode of -this skill that is not read-only, so say what it will touch before running it on -a bundle the user has not asked to migrate. It is textual (comments, key order -and quoting survive) and idempotent. - -```bash -uv run "${CLAUDE_SKILL_DIR}/scripts/okf_validate.py" .okf --migrate --strict -``` - -It hoists `timestamp` to `generated: { by: process:okf-migrate, at }`, lifts a -`# Citations` list into `sources`, and bumps `okf_version`. Two limits worth -repeating to the user: `generated.by` cannot be recovered for pre-v0.2 content -(hence the `process:` actor — the concept stays correctly `unverified` under -§5.3), and per-claim `[^id]` attribution was never encoded in v0.1, so only the -source list moves up. - -## Exit codes - -Non-zero if any error is present, or if warnings exceed the gate: -`--strict` allows none, `--max-warnings N` allows N, the default allows any. -Add `--json` for machine-readable output (useful in CI). diff --git a/external/okf/skills/validate/scripts/okf_validate.py b/external/okf/skills/validate/scripts/okf_validate.py deleted file mode 100644 index 8498452..0000000 --- a/external/okf/skills/validate/scripts/okf_validate.py +++ /dev/null @@ -1,573 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.11" -# dependencies = ["pyyaml>=6"] -# /// -"""Deterministic conformance checker for Open Knowledge Format (OKF) v0.2. - -Implements the §11 conformance rules verbatim: - 1. Every non-reserved `.md` file contains a parseable YAML frontmatter block. - 2. Every frontmatter block contains a non-empty `type` field. - 3. Reserved filenames (`index.md`, `log.md`) follow §8 / §9 when present. - -Rules 1 and 2 are hard errors (a bundle that fails them is non-conformant). -Everything else the spec marks as soft guidance: reported as warnings, never -fatal unless `--strict` is given. In particular broken cross-links are NOT -errors — the spec requires consumers to tolerate them (§6.1). - -v0.1 bundles still validate: the two superseded constructs (`timestamp`, a body -`# Citations` list) are read and reported as warnings pointing at their v0.2 -replacements (`generated.at`, `sources`), never as errors (§13.1). `--migrate` -performs that rewrite in place, textually, so `--strict` has a door and not just -a wall. - -Run: uv run scripts/okf_validate.py <bundle-dir> - [--strict | --max-warnings N] [--migrate] [--json] -""" -from __future__ import annotations - -import argparse -import json -import re -import sys -from dataclasses import dataclass, field -from pathlib import Path - -import yaml - -OKF_VERSION = "0.2" -RESERVED = {"index.md", "log.md"} -# `resource` is intentionally excluded: the spec (§4.1) states it is absent for -# concepts describing abstract ideas, so flagging it produces false noise. -RECOMMENDED = ("title", "description", "tags") -STATUS_VALUES = {"draft", "stable", "deprecated"} -ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") -# RFC 3339, as the spec writes `generated.at` / `verified[].at` throughout, and -# (since 0.9.6) `stale_after`, `sources[].last_modified` and `usage_window`. A -# date-only value is tolerated: it is common in the wild and loses only precision. -# PyYAML resolves an unquoted timestamp to a datetime whose str() separates with a -# space, so both spellings have to pass. -RFC3339 = re.compile( - r"^\d{4}-\d{2}-\d{2}" - r"(?:[Tt ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:[Zz]|[+-]\d{2}:?\d{2})?)?$") -# §7 gives three actor shapes, and §5.1 reuses them for `sources[].author`. The -# spec's own example (`author: team:ga4-docs`) shows the `<prefix>:<id>` family is -# open, so this is deliberately not a whitelist — it catches the one failure that -# is silent and costly: a near-miss of `human:`, which §5.3 keys trust tiers off. -# `human/dana` or `Human:dana` reads as an agent and downgrades the tier with no -# other symptom. -ACTOR_SHAPE = re.compile(r"^(?:[^\s:/]+:\S+|\S+/\S+)$") -# Anything that opens with human/process in any case — `Human:dana` is the worst -# of these, since it satisfies the generic `<prefix>:<id>` shape and so looks -# entirely well-formed while §5.3 reads it as an agent. The lookahead keeps -# `humanoid_agent/v1` out: only a non-identifier boundary makes it a near-miss. -ACTOR_HUMANISH = re.compile(r"^(?:human|process)(?![A-Za-z0-9_])", re.I) -FENCE = re.compile(r"^(```|~~~)") -# markdown link target capture: [text](target) — ignores images ![...] -LINK = re.compile(r"(?<!\!)\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)") -CITATIONS = re.compile(r"^#{1,6}[ \t]+Citations[ \t]*$", re.M) -# a footnote *reference* in the body; the definition line `[^id]: …` matches too, -# which is harmless — both carry the same label, the join key into `sources`. -FOOTNOTE = re.compile(r"\[\^([^\]\s]+)\]") - -# --migrate only. `timestamp` is a top-level key (§4.1), so anchor at column 0 — -# an indented one belongs to something else and must not be hoisted. -TIMESTAMP = re.compile(r"^timestamp:[ \t]*(.+?)[ \t]*$", re.M) -HEADING = re.compile(r"^#{1,6}[ \t]", re.M) -# same shape as LINK above, plus the link text — a citation written with a title -# attribute, `[Src](url "Title")`, must not be dropped: the section is deleted -# wholesale once anything parses, so a miss here is silent data loss. -MD_LINK = re.compile(r"(?<!\!)\[([^\]]+)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)") -OKF_VERSION_LINE = re.compile(r"^(okf_version:[ \t]*)[\"']?0\.1[\"']?[ \t]*$", re.M) -# True, and it keeps the concept correctly `unverified` under §5.3: who wrote the -# content before `generated` existed is not recoverable, and inventing a `human:` -# actor would fake a review that never happened. -MIGRATE_ACTOR = "process:okf-migrate" - - -@dataclass -class Report: - errors: list[str] = field(default_factory=list) - warnings: list[str] = field(default_factory=list) - concepts: int = 0 - indexes: int = 0 - logs: int = 0 - - def err(self, rel: str, msg: str) -> None: - self.errors.append(f"{rel}: {msg}") - - def warn(self, rel: str, msg: str) -> None: - self.warnings.append(f"{rel}: {msg}") - - -def split_frontmatter(text: str) -> tuple[str | None, str]: - """Return (raw_yaml, body). raw_yaml is None when no frontmatter block.""" - if not text.startswith("---"): - return None, text - # first line must be exactly '---' (allow trailing whitespace / BOM already stripped) - lines = text.splitlines(keepends=True) - if lines[0].strip() != "---": - return None, text - for i in range(1, len(lines)): - if lines[i].strip() == "---": - return "".join(lines[1:i]), "".join(lines[i + 1 :]) - return None, text # unterminated block → treated as absent - - -def _read_text(path: Path, rel: str, report: Report) -> str | None: - """Read a concept/index/log file, reporting (not raising) on a bad file. - - A binary or non-UTF-8 `.md` file must fail as a per-file error, not crash - the whole validation run. - """ - try: - return path.read_text(encoding="utf-8").lstrip("") - except (UnicodeDecodeError, OSError) as exc: - report.err(rel, f"cannot read file: {exc}") - return None - - -def check_computation(meta: dict, path: Path, bundle: Path, rel: str, report: Report) -> None: - """§10.2 — the Attested Computation contract, including its path-valued fields. - - `computation`, `executor.resource` and `attester.resource` name files (§6.2). - They are exactly the pointer that rots: the concept keeps validating while the - script it names moves or disappears. A target outside the bundle is legitimate - and unverifiable from here, so only bundle-relative paths are resolved. - """ - if not str(meta.get("runtime", "")).strip(): - report.warn(rel, "§10.2 an Attested Computation concept requires `runtime`") - fields = [("computation", meta.get("computation"))] - for key in ("executor", "attester"): - block = meta.get(key) - if isinstance(block, dict): - fields.append((f"{key}.resource", block.get("resource"))) - for where, target in fields: - if target is None: - continue - t = str(target).strip() - if not t or re.match(r"^[a-z][a-z0-9+.-]*://", t) or t.startswith("mailto:"): - continue - candidate = (bundle / t.lstrip("/")) if t.startswith("/") else (path.parent / t) - try: - inside = candidate.resolve().is_relative_to(bundle.resolve()) - except OSError: - inside = False - if inside and not candidate.exists(): - report.warn(rel, f"§6.2 `{where}` points at `{t}`, which does not exist") - - -def check_concept(path: Path, rel: str, bundle: Path, report: Report) -> None: - report.concepts += 1 - text = _read_text(path, rel, report) - if text is None: - return - raw, body = split_frontmatter(text) - if raw is None: - report.err(rel, "§11.1 no parseable YAML frontmatter block " - "(every non-reserved `.md` in a bundle is a concept: add " - "frontmatter with a `type` field, or move the file outside " - "the bundle)") - return - try: - meta = yaml.safe_load(raw) - except yaml.YAMLError as exc: - report.err(rel, f"§11.1 frontmatter is not valid YAML: {exc}".replace("\n", " ")) - return - if not isinstance(meta, dict): - report.err(rel, "§11.1 frontmatter must be a YAML mapping") - return - type_val = meta.get("type") - if not (isinstance(type_val, str) and type_val.strip()): - report.err(rel, "§11.2 missing or empty required `type` field") - for key in RECOMMENDED: - if key not in meta: - report.warn(rel, f"recommended field `{key}` is absent (§4.1)") - check_trust(meta, rel, report) - check_lifecycle(meta, rel, report) - check_sources(meta, body, rel, report) - if isinstance(type_val, str) and type_val.strip() == "Attested Computation": - check_computation(meta, path, bundle, rel, report) - - -def check_actor(value, where: str, rel: str, report: Report) -> None: - """§7 — the actor convention shared by `generated.by`, `verified[].by` and - `sources[].author`.""" - actor = str(value).strip() - if not actor: - return - if ACTOR_HUMANISH.match(actor) and not actor.startswith(("human:", "process:")): - report.warn(rel, f"§7 `{where}` `{actor}` is a near-miss of " - f"`human:`/`process:` — §5.3 keys trust tiers off that exact " - f"lowercase prefix, so this silently reads as an agent") - elif not ACTOR_SHAPE.match(actor): - report.warn(rel, f"§7 `{where}` `{actor}` matches no actor shape " - f"(`human:<id>`, `process:<id>`, or `<producer>/<version>`)") - - -def check_instant(value, where: str, rel: str, report: Report) -> None: - """§5.2 — `at` values are RFC 3339; a date-only value is tolerated.""" - if value is not None and not RFC3339.match(str(value).strip()): - report.warn(rel, f"§5.2 `{where}` `{value}` is not an RFC 3339 timestamp") - - -def check_trust(meta: dict, rel: str, report: Report) -> None: - """§5.2 — `generated` / `verified`, with the v0.1 `timestamp` fallback.""" - gen = meta.get("generated") - if gen is None: - if "timestamp" in meta: - report.warn(rel, "legacy v0.1 `timestamp`: v0.2 records the last content " - "change as `generated: {by, at}` (§13.1)") - else: - report.warn(rel, "recommended field `generated` is absent (§5.2)") - elif not isinstance(gen, dict): - report.warn(rel, "§5.2 `generated` must be a mapping with `by` and `at`") - elif not str(gen.get("by", "")).strip(): - report.warn(rel, "§5.2 `generated.by` is required within `generated`") - else: - check_actor(gen["by"], "generated.by", rel, report) - check_instant(gen.get("at"), "generated.at", rel, report) - - ver = meta.get("verified") - if ver is None: - return - # A bare mapping is one verification event — consumers MUST treat it as a - # one-element list (§5.2), so normalize before checking the entries. - entries = [ver] if isinstance(ver, dict) else ver - if not isinstance(entries, list): - report.warn(rel, "§5.2 `verified` must be a `{by, at}` mapping or a list of them") - return - for i, entry in enumerate(entries): - if not (isinstance(entry, dict) and str(entry.get("by", "")).strip()): - report.warn(rel, "§5.2 every `verified` entry needs a `by` actor") - continue - check_actor(entry["by"], f"verified[{i}].by", rel, report) - check_instant(entry.get("at"), f"verified[{i}].at", rel, report) - - -def check_lifecycle(meta: dict, rel: str, report: Report) -> None: - """§5.4 / §5.5 — `status` and `stale_after`. A concept is stale when - `now >= stale_after`.""" - status = meta.get("status") - if status is not None and status not in STATUS_VALUES: - report.warn(rel, f"§5.4 unknown `status` `{status}` (expected " - f"{'|'.join(sorted(STATUS_VALUES))})") - stale = meta.get("stale_after") - if stale is not None and not RFC3339.match(str(stale)): - report.warn(rel, f"§5.5 `stale_after` `{stale}` is not an ISO 8601 datetime " - f"(a bare date is tolerated)") - - -def check_window(window, where: str, rel: str, report: Report) -> None: - """§5.1 — a `usage_window` is a `{from, to}` pair of ISO 8601 datetimes.""" - if window is None: - return - if not isinstance(window, dict): - report.warn(rel, f"§5.1 `{where}` `usage_window` must be a `{{from, to}}` mapping") - return - for bound in ("from", "to"): - value = window.get(bound) - if value is None: - report.warn(rel, f"§5.1 `{where}` `usage_window` is missing `{bound}`") - elif not RFC3339.match(str(value)): - report.warn(rel, f"§5.1 `{where}` `usage_window.{bound}` `{value}` " - f"is not an ISO 8601 datetime (a bare date is tolerated)") - - -def check_sources(meta: dict, body: str, rel: str, report: Report) -> None: - """§5.1 — the `sources` provenance family and its footnote join keys.""" - if CITATIONS.search(body): - report.warn(rel, "legacy v0.1 `# Citations` body list: v0.2 records provenance " - "in the `sources` frontmatter (§13.1)") - srcs = meta.get("sources") - if srcs is None: - return - if not isinstance(srcs, list): - report.warn(rel, "§5.1 `sources` must be a list of entries") - return - ids = set() - for i, src in enumerate(srcs): - if not isinstance(src, dict): - report.warn(rel, f"§5.1 `sources[{i}]` must be a mapping") - continue - if not str(src.get("resource", "")).strip(): - report.warn(rel, f"§5.1 `sources[{i}]` is missing the required `resource`") - if "id" in src: - ids.add(str(src["id"])) - if src.get("author") is not None: - check_actor(src["author"], f"sources[{i}].author", rel, report) - # §5.1 — `usage_count` is framed by a `usage_window`, written once as a - # sibling of `sources` or overridden on the entry. Without one the count - # is a number with no units, and the spec asks consumers to read it as - # liveness and trend rather than a score. - window = src.get("usage_window", meta.get("usage_window")) - if src.get("usage_count") is not None and window is None: - report.warn(rel, f"§5.1 `sources[{i}].usage_count` has no `usage_window` " - f"framing it (a sibling of `sources`, or on the entry)") - check_window(window, f"sources[{i}]", rel, report) - last_mod = src.get("last_modified") - if last_mod is not None and not RFC3339.match(str(last_mod)): - report.warn(rel, f"§5.1 `sources[{i}].last_modified` `{last_mod}` is not an " - f"ISO 8601 datetime (a bare date is tolerated)") - # Attribution joins on the label, not on position (§5.1) — a footnote whose - # label names no source silently attributes a claim to nothing. - for label in sorted(set(FOOTNOTE.findall(body)) - ids): - report.warn(rel, f"§5.1 footnote `[^{label}]` matches no `sources[].id`") - - -def check_index(path: Path, rel: str, is_root: bool, report: Report) -> None: - report.indexes += 1 - text = _read_text(path, rel, report) - if text is None: - return - raw, _ = split_frontmatter(text) - if raw is not None: - if not is_root: - report.warn(rel, "§8 index.md should contain no frontmatter") - else: - try: - meta = yaml.safe_load(raw) or {} - except yaml.YAMLError: - report.warn(rel, "§12 root index.md frontmatter is not valid YAML") - meta = {} - # `upkeep` rides along: the plugin's Stop hook reads its opt-in flag - # from here, so an enforced-mode bundle would otherwise fail its own - # `--strict` run. §11 forbids rejecting a bundle for unknown keys. - extra = set(meta) - {"okf_version", "upkeep"} - if extra: - report.warn(rel, f"§12 root index.md frontmatter may only carry `okf_version` (found {sorted(extra)})") - declared = meta.get("okf_version") - if declared is not None and str(declared) != OKF_VERSION: - report.warn(rel, f"§12 bundle declares `okf_version: \"{declared}\"`; " - f"checked against v{OKF_VERSION}") - - -def check_log(path: Path, rel: str, report: Report) -> None: - report.logs += 1 - text = _read_text(path, rel, report) - if text is None: - return - raw, _ = split_frontmatter(text) - if raw is not None: - report.warn(rel, "§9 log.md should contain no frontmatter") - for line in text.splitlines(): - if line.startswith("## "): - heading = line[3:].strip() - if not ISO_DATE.match(heading): - report.warn(rel, f"§9 date heading `{heading}` is not ISO 8601 YYYY-MM-DD") - - -def collect_link_targets(path: Path) -> list[str]: - # Unreadable files are already reported as a per-file error by - # check_concept/check_index/check_log; fail soft here to avoid a duplicate - # error and a crash on the second read pass. - try: - text = path.read_text(encoding="utf-8") - except (UnicodeDecodeError, OSError): - return [] - targets: list[str] = [] - in_fence = False - for line in text.splitlines(): - if FENCE.match(line.strip()): - in_fence = not in_fence - continue - if in_fence: - continue - targets.extend(LINK.findall(line)) - return targets - - -def check_links(bundle: Path, md_files: list[Path], report: Report) -> None: - """Broken bundle-internal links are warnings only (§6.1).""" - existing = {p.relative_to(bundle).as_posix() for p in md_files} - for path in md_files: - rel = path.relative_to(bundle).as_posix() - for target in collect_link_targets(path): - t = target.split("#", 1)[0] - if not t or t.endswith("/"): - continue - if re.match(r"^[a-z][a-z0-9+.-]*://", t) or t.startswith("mailto:"): - continue - if not t.endswith(".md"): - continue - if t.startswith("/"): - resolved = t.lstrip("/") - else: - resolved = (path.parent / t).resolve().relative_to(bundle.resolve()).as_posix() \ - if (path.parent / t).resolve().is_relative_to(bundle.resolve()) else t - if resolved not in existing: - report.warn(rel, f"cross-link target not found: `{target}` (tolerated under §6.1)") - - -def citations_to_sources(raw: str, body: str) -> tuple[str, str]: - """Hoist a v0.1 `# Citations` list into the `sources` family (§5.1, §13.1).""" - heading = CITATIONS.search(body) - if heading is None: - return raw, body - rest = body[heading.end():] - nxt = HEADING.search(rest) - section, tail = (rest[:nxt.start()], rest[nxt.start():]) if nxt else (rest, "") - entries = MD_LINK.findall(section) - if not entries: - return raw, body # nothing extractable — leave it and keep warning - # json.dumps gives a correctly escaped YAML double-quoted scalar for free. - block = "sources:\n" + "".join( - f" - resource: {json.dumps(url)}\n title: {json.dumps(title)}\n" - for title, url in entries) - kept = body[:heading.start()].rstrip("\n") - return raw + block, kept + ("\n\n" + tail if tail else "\n") - - -def migrate_text(text: str, is_root_index: bool) -> str | None: - """Rewrite one file's v0.1 constructs to v0.2. None when nothing changes. - - Textual on purpose: dumping the frontmatter back through PyYAML would flatten - comments, key order and quoting across every file in the bundle. Each branch - is guarded on the v0.2 key already being present, so a half-migrated bundle - converges instead of growing a duplicate key that PyYAML then refuses to load. - """ - raw, body = split_frontmatter(text) - if raw is None: - return None - if is_root_index: - new_raw = OKF_VERSION_LINE.sub(rf'\g<1>"{OKF_VERSION}"', raw, count=1) - return None if new_raw == raw else f"---\n{new_raw}---\n{body}" - try: - meta = yaml.safe_load(raw) - except yaml.YAMLError: - return None - if not isinstance(meta, dict): - return None - new_raw, new_body = raw, body - if "generated" not in meta: - new_raw = TIMESTAMP.sub( - lambda m: f"generated:\n by: {MIGRATE_ACTOR}\n at: {m.group(1)}", - new_raw, count=1) - if "sources" not in meta: - new_raw, new_body = citations_to_sources(new_raw, new_body) - if (new_raw, new_body) == (raw, body): - return None - return f"---\n{new_raw}---\n{new_body}" - - -def migrate(bundle: Path) -> list[str]: - """Migrate a bundle in place. Returns the relative paths actually rewritten.""" - changed = [] - for path in sorted(p for p in bundle.rglob("*.md") if p.is_file()): - rel = path.relative_to(bundle).as_posix() - if path.name == "log.md": - continue - try: - # Universal newlines on purpose: a CRLF bundle is rewritten with the - # platform ending, consistently. Passing newline="" to preserve them - # exactly would leave a stray \r inside every `$`-anchored capture - # below, so those patterns need hardening first if byte fidelity is - # ever required. - text = path.read_text(encoding="utf-8") - except (UnicodeDecodeError, OSError): - continue - is_root_index = path.name == "index.md" and path.parent == bundle - if path.name == "index.md" and not is_root_index: - continue - new = migrate_text(text.lstrip(""), is_root_index) - if new is not None and new != text: - path.write_text(new, encoding="utf-8") - changed.append(rel) - return changed - - -def validate(bundle: Path) -> Report: - report = Report() - md_files = sorted(p for p in bundle.rglob("*.md") if p.is_file()) - for path in md_files: - rel = path.relative_to(bundle).as_posix() - name = path.name - if name == "index.md": - check_index(path, rel, is_root=(path.parent == bundle), report=report) - elif name == "log.md": - check_log(path, rel, report) - else: - check_concept(path, rel, bundle, report) - check_links(bundle, md_files, report) - return report - - -def _force_utf8_stdio() -> None: - """Ensure stdout/stderr can emit the ✓/✗/— glyphs this tool prints. - - Windows consoles default to cp1252, which raises UnicodeEncodeError on those - characters and crashes an otherwise-successful run. Reconfiguring to UTF-8 - (Python ≥3.7) fixes it; `errors="replace"` is a belt-and-suspenders fallback - for any stream that still cannot encode a glyph. - """ - for stream in (sys.stdout, sys.stderr): - reconfigure = getattr(stream, "reconfigure", None) - if reconfigure is not None: - try: - reconfigure(encoding="utf-8", errors="replace") - except (ValueError, OSError): - pass - - -def main() -> int: - _force_utf8_stdio() - ap = argparse.ArgumentParser(description=f"Validate an OKF v{OKF_VERSION} bundle.") - ap.add_argument("bundle", type=Path, help="path to the bundle directory") - ap.add_argument("--strict", action="store_true", help="treat warnings as errors") - ap.add_argument("--max-warnings", type=int, default=None, metavar="N", - help="fail if warnings exceed N (--strict is the N=0 case)") - ap.add_argument("--migrate", action="store_true", - help="rewrite v0.1 constructs to v0.2 in place, then validate") - ap.add_argument("--json", action="store_true", help="emit a JSON report") - args = ap.parse_args() - - if not args.bundle.is_dir(): - print(f"error: {args.bundle} is not a directory", file=sys.stderr) - return 2 - - migrated = migrate(args.bundle) if args.migrate else [] - - r = validate(args.bundle) - conformant = not r.errors - max_warn = 0 if args.strict else args.max_warnings - failed = bool(r.errors) or (max_warn is not None and len(r.warnings) > max_warn) - - if args.json: - print(json.dumps({ - "bundle": str(args.bundle), - "conformant": conformant, - "passed": not failed, - "counts": {"concepts": r.concepts, "indexes": r.indexes, "logs": r.logs}, - "errors": r.errors, - "warnings": r.warnings, - "migrated": migrated, - }, indent=2)) - return 0 if not failed else 1 - - if args.migrate: - for rel in migrated: - print(f" \033[36m→ migrated\033[0m {rel}") - print(f" migrated {len(migrated)} file(s) to v{OKF_VERSION}" - + (f", `generated.by` recorded as `{MIGRATE_ACTOR}`" if migrated else "")) - if migrated: - print(" note: per-claim attribution (the `[^id]` footnotes of §5.1) is not" - " recoverable from a v0.1 `# Citations` list — sources moved up, but" - " which claim cited which was never encoded.") - - print(f"OKF v{OKF_VERSION} conformance — {args.bundle}") - print(f" concepts: {r.concepts} index.md: {r.indexes} log.md: {r.logs}") - for e in r.errors: - print(f" \033[31m✗ ERROR\033[0m {e}") - for w in r.warnings: - print(f" \033[33m! warn \033[0m {w}") - if conformant and not r.warnings: - print(" \033[32m✓ conformant — no issues\033[0m") - elif conformant: - print(f" \033[32m✓ conformant\033[0m ({len(r.warnings)} warning(s))") - else: - print(f" \033[31m✗ non-conformant\033[0m ({len(r.errors)} error(s))") - return 0 if not failed else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/external/okf/skills/visualize/SKILL.md b/external/okf/skills/visualize/SKILL.md deleted file mode 100644 index 1611ac0..0000000 --- a/external/okf/skills/visualize/SKILL.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -name: visualize -description: >- - Render an Open Knowledge Format (OKF) bundle as a single self-contained, - interactive HTML graph (viz.html) — concepts as nodes coloured by type and - sized by body length, markdown links and bundle-internal `sources` as edges, - a wiki-style detail panel with rendered markdown, v0.2 trust/lifecycle/provenance - metadata, and "Links to" / "Cited by" backlinks, layout switching, per-type - filter and search. - Use when asked to visualize, graph, preview, or explore an OKF bundle. -user-invocable: true -argument-hint: "<bundle-dir> [-o viz.html]" -allowed-tools: Bash ---- - -# Visualize an OKF bundle - -Generate a self-contained HTML graph of the target bundle. The bundle directory -is required, so pass `.okf` explicitly when no path is given. No backend, no -install, no bundle content leaves the page. The viewer does need network -access: cytoscape, marked and dompurify load from cdn.jsdelivr.net. - -```bash -uv run "${CLAUDE_SKILL_DIR}/scripts/okf_visualize.py" $ARGUMENTS -``` - -If `uv` is unavailable: - -```bash -python3 -m pip install --quiet pyyaml && \ -python3 "${CLAUDE_SKILL_DIR}/scripts/okf_visualize.py" $ARGUMENTS -``` - -The detail panel shows each concept's `type`, `description`, `tags`, `status`, -`generated`, `verified`, `stale_after`, `sources` (with their credibility -signals, a `usage_count` alongside the `usage_window` that frames it), and -rendered body; a `sources` entry pointing at another concept in the bundle also -becomes a graph edge. A v0.1 `timestamp` is read as `generated.at`, so legacy -bundles still render fully. - -Two badges are **derived**, not read: the §5.3 trust tier -(*unverified* / *machine-confirmed* / *human-reviewed*, keyed off the `human:` -prefix in `verified[].by`) and staleness (`now >= stale_after`). They are -advisory signals — when reporting on a bundle, say which tier a concept is in -rather than treating any of them as a gate. - -| Flag | Effect | -|------|--------| -| `-o <path>` | Output file (default `<bundle>/viz.html`). | -| `-t, --title` | Graph title (default `parent/bundle` dir name). | -| `-l, --link` | Source URL shown in the header. | -| `--layout` | `cose`, `concentric`, `breadthfirst`, `circle`, `grid`. | -| `--max-nodes N` | Exit 1 above N concepts (useful in CI). | -| `--og-image URL` | Absolute URL for the social preview image. | - -Above 1,000 concepts the default switches to `concentric` (force freezes the -page; `--layout cose` overrides, and the in-page picker asks first). Above -5,000 it warns that the page loads slowly and reads as a hairball, and -suggests rendering a subdirectory. - -Open it in any browser; `${CLAUDE_SKILL_DIR}` resolves whether this runs as part -of the `okf` plugin or as a standalone skills.sh skill. The page is -deep-linkable: `?select=<concept-id>` or `#<concept-id>` opens that concept, -`?layout=<name>` presets the layout. diff --git a/external/okf/skills/visualize/scripts/okf_visualize.py b/external/okf/skills/visualize/scripts/okf_visualize.py deleted file mode 100644 index 5265cd8..0000000 --- a/external/okf/skills/visualize/scripts/okf_visualize.py +++ /dev/null @@ -1,432 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.11" -# dependencies = ["pyyaml>=6"] -# /// -"""Render an Open Knowledge Format (OKF) bundle as a single self-contained, -interactive HTML graph (`viz.html`). No backend, no install on the viewing side, -no data leaves the page — concepts become nodes (coloured by `type`, sized by -body length), markdown links and bundle-internal `sources` become edges, and -clicking a node opens a wiki-style panel with its rendered markdown, OKF v0.2 -provenance/trust/lifecycle metadata, outgoing links, and "Cited by" backlinks. - -Features: force/concentric/breadth-first/circle/grid layouts, per-type filter, -free-text search, neighbour highlight, clickable cross-links and backlinks. - -The default layout is force (cose) up to AUTO_COSE_MAX concepts, then the linear -concentric layout — force-directed cost grows roughly quadratically with node -count and freezes the page on large bundles. An explicit --layout always wins. - -Run: uv run okf_visualize.py <bundle-dir> [-o viz.html] -""" -from __future__ import annotations - -import argparse -import json -import re -import sys -from pathlib import Path - -import yaml - -RESERVED = {"index.md", "log.md"} -# Force (cose) layout froze the page for ~32 s at ~2k concepts (measured in -# Chrome); the linear layouts load the same bundle in under 2 s. Past this size -# the default switches to concentric, and the in-page layout picker asks before -# running force. An explicit --layout (or ?layout=) still wins. -AUTO_COSE_MAX = 1000 -# Above this the page is slow on any layout (23k concepts measured: ~27 s load, -# ~650 MB heap) and reads as a hairball — warn and suggest rendering a subtree. -SCALE_WARN = 5000 -FENCE = re.compile(r"^(```|~~~)") -LINK = re.compile(r"(?<!\!)\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)") - - -def json_for_script(obj) -> str: - """JSON-encode ``obj`` for safe embedding inside an inline ``<script>``. - - The HTML tokenizer knows nothing about JS string context: a literal - ``</script>`` in a concept body ends the inline script early (truncating the - embedded ``NODES``/``EDGES`` and breaking the whole page), and ``<!--`` can - shift it into the escaped script-data states where even the template's real - closing tag stops working. Escaping every ``<`` as ``\\u003c`` neutralizes - ``</script>``, ``<!--`` and ``<script`` in one stroke, and the result stays - valid JSON *and* JavaScript (``json.dumps`` keeps everything else ASCII via - the default ``ensure_ascii=True``). - """ - return json.dumps(obj, default=str).replace("<", "\\u003c") - - -def split_frontmatter(text: str): - if not text.startswith("---"): - return {}, text - lines = text.splitlines(keepends=True) - if lines[0].strip() != "---": - return {}, text - for i in range(1, len(lines)): - if lines[i].strip() == "---": - try: - meta = yaml.safe_load("".join(lines[1:i])) or {} - except yaml.YAMLError: - meta = {} - return (meta if isinstance(meta, dict) else {}), "".join(lines[i + 1:]) - return {}, text - - -def link_targets(text: str): - out, in_fence = [], False - for line in text.splitlines(): - if FENCE.match(line.strip()): - in_fence = not in_fence - continue - if not in_fence: - out.extend(LINK.findall(line)) - return out - - -def resolve(target: str, path: Path, bundle: Path): - """Resolve a link/`sources[].resource` to a concept id, or None if it is not - one (an external URL, an asset, a scope descriptor, an escape from the tree).""" - t = str(target).split("#", 1)[0] - if not t.endswith(".md"): - return None - if t.startswith("/"): - return t.lstrip("/")[:-3] - cand = (path.parent / t).resolve() - return cand.relative_to(bundle.resolve()).as_posix()[:-3] \ - if cand.is_relative_to(bundle.resolve()) else None - - -def read_sources(meta: dict, path: Path, bundle: Path): - """§5.1 `sources`, flattened for display. `cid` is set when the source is - itself a concept in this bundle — that derivation is a real graph edge.""" - raw = meta.get("sources") - out = [] - if not isinstance(raw, list): - return out - for src in raw: - if not isinstance(src, dict): - continue - resource = str(src.get("resource", "")).strip() - # §5.1 — an entry's own `usage_window` overrides the one written once as a - # sibling of `sources`. A count without its window has no units. - window = src.get("usage_window", meta.get("usage_window")) - out.append({ - "title": str(src.get("title") or src.get("id") or resource), - "resource": resource, - "cid": resolve(resource, path, bundle) if resource else None, - "author": str(src.get("author", "")), - "usage_count": src.get("usage_count"), - "usage_window": (f"{window.get('from', '?')}→{window.get('to', '?')}" - if isinstance(window, dict) else ""), - "last_modified": str(src.get("last_modified") or ""), - }) - return out - - -def read_trust(meta: dict): - """§5.2 `generated` / `verified`, falling back to a v0.1 `timestamp` (§13.1).""" - gen = meta.get("generated") - if isinstance(gen, dict): - generated = {"by": str(gen.get("by", "")), "at": str(gen.get("at", ""))} - elif meta.get("timestamp"): - generated = {"by": "", "at": str(meta["timestamp"])} - else: - generated = None - ver = meta.get("verified") - # a bare mapping is one verification event (§5.2) - entries = [ver] if isinstance(ver, dict) else (ver if isinstance(ver, list) else []) - verified = [{"by": str(e.get("by", "")), "at": str(e.get("at", ""))} - for e in entries if isinstance(e, dict)] - return generated, verified - - -def build(bundle: Path): - nodes, edges, seen = [], [], set() - files = sorted(p for p in bundle.rglob("*.md") if p.is_file() and p.name not in RESERVED) - ids = {p.relative_to(bundle).with_suffix("").as_posix() for p in files} - for p in files: - cid = p.relative_to(bundle).with_suffix("").as_posix() - try: - raw = p.read_text(encoding="utf-8").lstrip("") - except (UnicodeDecodeError, OSError) as exc: - print(f"warning: skipping {p.relative_to(bundle)}: cannot read file: {exc}", file=sys.stderr) - ids.discard(cid) - continue - meta, body = split_frontmatter(raw) - body = body.strip() - generated, verified = read_trust(meta) - sources = read_sources(meta, p, bundle) - nodes.append({ - "id": cid, - "type": str(meta.get("type", "Untyped")), - "title": str(meta.get("title", p.stem)), - "description": str(meta.get("description", "")), - "tags": meta.get("tags", []) if isinstance(meta.get("tags"), list) else [], - "group": cid.split("/")[0] if "/" in cid else "(root)", - "sz": max(24, min(70, 24 + len(body) // 200)), - "status": str(meta.get("status", "")), - "stale_after": str(meta.get("stale_after") or ""), - "generated": generated, - "verified": verified, - "sources": sources, - "body": body[:8000], - }) - targets = link_targets(body) + [s["resource"] for s in sources if s["cid"]] - for t in targets: - tgt = resolve(t, p, bundle) - if tgt and tgt in ids and tgt != cid and (cid, tgt) not in seen: - seen.add((cid, tgt)) - edges.append({"source": cid, "target": tgt}) - return nodes, edges - - -HTML = r"""<!doctype html><html lang="en"><head><meta charset="utf-8"> -<meta name="viewport" content="width=device-width,initial-scale=1"> -<title>OKF — __NAME__ - - - - -__OGIMAGE__ - - - - -

Click a concept to inspect it.

-

__NAME__

__N__ concepts · __E__ links · OKF v0.2__LINK__
-
- - - -
-
-""" - - -def render(bundle: Path, out: Path, title: str | None = None, link: str | None = None, - layout: str | None = None, og_image: str | None = None, - max_nodes: int | None = None): - nodes, edges = build(bundle) - if max_nodes is not None and len(nodes) > max_nodes: - sys.exit(f"error: {len(nodes)} concepts exceeds --max-nodes {max_nodes}") - if layout is None: - layout = "cose" if len(nodes) <= AUTO_COSE_MAX else "concentric" - if layout != "cose": - print(f"note: {len(nodes)} concepts > {AUTO_COSE_MAX} — using the linear 'concentric' " - "layout (force freezes the page at this size; pass --layout cose to override)", - file=sys.stderr) - if len(nodes) > SCALE_WARN: - print(f"warning: {len(nodes)} concepts — the page will load slowly and read as a hairball; " - f"consider rendering a subtree, e.g. okf_visualize.py {bundle}/", - file=sys.stderr) - name = title or f"{bundle.resolve().parent.name}/{bundle.name}" - src = f' source ↗' if link else "" - aesc = lambda s: (s or "").replace("&", "&").replace('"', """).replace("<", "<").replace(">", ">") - og_title = aesc(f"OKF — {name}") - og_desc = aesc(f"{len(nodes)} concepts · interactive Open Knowledge Format knowledge graph") - og_img = (f'\n' - f'') if og_image else "" - subs = {"__NAME__": name, "__LINK__": src, "__LAYOUT__": layout, - "__COSEMAX__": str(AUTO_COSE_MAX), - "__OGTITLE__": og_title, "__OGDESC__": og_desc, "__OGIMAGE__": og_img, - "__N__": str(len(nodes)), "__E__": str(len(edges)), - "__NODES__": json_for_script(nodes), "__EDGES__": json_for_script(edges)} - # One pass, longest marker first: substituted content (e.g. a concept body that - # mentions "__EDGES__") must never itself be rescanned for other markers. - marker = re.compile("|".join(sorted(map(re.escape, subs), key=len, reverse=True))) - html = marker.sub(lambda m: subs[m.group(0)], HTML) - out.write_text(html, encoding="utf-8") - return len(nodes), len(edges) - - -def _force_utf8_stdio() -> None: - """Reconfigure stdout/stderr to UTF-8 so status lines never crash on a - cp1252 (default Windows) console. `errors="replace"` is a fallback.""" - for stream in (sys.stdout, sys.stderr): - reconfigure = getattr(stream, "reconfigure", None) - if reconfigure is not None: - try: - reconfigure(encoding="utf-8", errors="replace") - except (ValueError, OSError): - pass - - -def main() -> int: - _force_utf8_stdio() - ap = argparse.ArgumentParser(description="Render an OKF bundle as a self-contained HTML graph.") - ap.add_argument("bundle", type=Path) - ap.add_argument("-o", "--out", type=Path, default=None) - ap.add_argument("-t", "--title", default=None, help="graph title (default: parent/bundle dir name)") - ap.add_argument("-l", "--link", default=None, help="optional source URL shown in the header") - ap.add_argument("--layout", default=None, - choices=["cose", "concentric", "breadthfirst", "circle", "grid"], - help=f"initial graph layout (default: cose, or concentric above {AUTO_COSE_MAX} " - "concepts — force layout freezes the page on large bundles)") - ap.add_argument("--max-nodes", type=int, default=None, - help="refuse to render bundles with more concepts than this (useful in CI)") - ap.add_argument("--og-image", default=None, - help="absolute URL for the social-preview image (og:image / twitter:image)") - args = ap.parse_args() - if not args.bundle.is_dir(): - print(f"error: {args.bundle} is not a directory", file=sys.stderr) - return 2 - out = args.out or (args.bundle / "viz.html") - n, e = render(args.bundle, out, title=args.title, link=args.link, layout=args.layout, - og_image=args.og_image, max_nodes=args.max_nodes) - print(f"rendered {n} concepts, {e} links -> {out}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())