1007 lines
35 KiB
Python
1007 lines
35 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
# /// script
|
||
|
|
# requires-python = ">=3.11"
|
||
|
|
# ///
|
||
|
|
"""Event-sourcing extractor: reconstruct OKF bundle from git + session history.
|
||
|
|
|
||
|
|
Deterministic extraction: same input → byte-identical events.jsonl.
|
||
|
|
|
||
|
|
Run: uv run okf_backfill_events.py <repo-dir> [--out events.jsonl]
|
||
|
|
[--branch REF ...] [--no-sessions] [--sessions-dir DIR] [--max-text 2000]
|
||
|
|
[--skip-globs GLOB ...]
|
||
|
|
|
||
|
|
uv run okf_backfill_events.py <repo-dir> --show <sha> [--only <path>]
|
||
|
|
[--max-diff-lines 300] [--per-file-lines 120] [--max-line-chars 400]
|
||
|
|
[--skip-globs GLOB ...]
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import glob as glob_module
|
||
|
|
import fnmatch
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Optional
|
||
|
|
|
||
|
|
|
||
|
|
# Skip rule predicates
|
||
|
|
@dataclass
|
||
|
|
class SkipRule:
|
||
|
|
id: str
|
||
|
|
check: callable
|
||
|
|
|
||
|
|
|
||
|
|
def matches_globs(paths: list[str], globs: list[str]) -> bool:
|
||
|
|
"""True if ALL paths match at least one glob pattern."""
|
||
|
|
if not paths:
|
||
|
|
return False
|
||
|
|
for path in paths:
|
||
|
|
if not any(fnmatch.fnmatch(path, g) for g in globs):
|
||
|
|
return False
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
LOCKFILE_GLOBS = [
|
||
|
|
"*.lock",
|
||
|
|
"package-lock.json",
|
||
|
|
"pnpm-lock.yaml",
|
||
|
|
"uv.lock",
|
||
|
|
"Cargo.lock",
|
||
|
|
"go.sum",
|
||
|
|
"vendor/**",
|
||
|
|
"node_modules/**",
|
||
|
|
"dist/**",
|
||
|
|
"*.min.*",
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def rule_paths_only_generated(event: dict, skip_globs: list[str]) -> bool:
|
||
|
|
"""Commit only touches generated/lockfile paths."""
|
||
|
|
if event["source"] != "git":
|
||
|
|
return False
|
||
|
|
globs = LOCKFILE_GLOBS + skip_globs
|
||
|
|
paths = [f["path"] for f in event.get("files", [])]
|
||
|
|
return matches_globs(paths, globs)
|
||
|
|
|
||
|
|
|
||
|
|
def rule_merge_no_files(event: dict, skip_globs: list[str]) -> bool:
|
||
|
|
"""Merge commit with no files touched."""
|
||
|
|
if event["source"] != "git":
|
||
|
|
return False
|
||
|
|
return len(event.get("files", [])) == 0
|
||
|
|
|
||
|
|
|
||
|
|
def rule_session_command_noise(event: dict, skip_globs: list[str]) -> bool:
|
||
|
|
"""Session event that is just a slash command or too short with no outcome."""
|
||
|
|
if event["source"] != "session":
|
||
|
|
return False
|
||
|
|
user = event.get("user", "").strip()
|
||
|
|
|
||
|
|
# Bare slash command (no space or just "/" alone)
|
||
|
|
if user == "/" or (user.startswith("/") and " " not in user):
|
||
|
|
return True
|
||
|
|
|
||
|
|
# Local command output (starts with "<")
|
||
|
|
if user.startswith("<"):
|
||
|
|
return True
|
||
|
|
|
||
|
|
# Too short after strip (< 20 chars) AND no useful outcome
|
||
|
|
if len(user) < 20 and not event.get("outcome"):
|
||
|
|
return True
|
||
|
|
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
SKIP_RULES = [
|
||
|
|
SkipRule("paths-only-generated", rule_paths_only_generated),
|
||
|
|
SkipRule("merge-no-files", rule_merge_no_files),
|
||
|
|
SkipRule("session-command-noise", rule_session_command_noise),
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def normalize_ts(ts: str) -> float:
|
||
|
|
"""Convert ISO8601 timestamp to UTC epoch float."""
|
||
|
|
# Handle RFC3339 with or without timezone
|
||
|
|
ts = ts.strip()
|
||
|
|
|
||
|
|
# If it ends with Z, it's UTC
|
||
|
|
if ts.endswith("Z"):
|
||
|
|
ts = ts[:-1] + "+00:00"
|
||
|
|
|
||
|
|
# Handle offset timezone offsets like +01:00 or -05:00
|
||
|
|
try:
|
||
|
|
# Try ISO format parsing
|
||
|
|
if "T" in ts:
|
||
|
|
dt = datetime.fromisoformat(ts)
|
||
|
|
else:
|
||
|
|
# Date only
|
||
|
|
dt = datetime.fromisoformat(ts + "T00:00:00")
|
||
|
|
return dt.timestamp()
|
||
|
|
except Exception:
|
||
|
|
print(f"Warning: could not parse timestamp {ts!r}, using epoch 0", file=sys.stderr)
|
||
|
|
return 0.0
|
||
|
|
|
||
|
|
|
||
|
|
def epoch_to_iso_z(epoch: float) -> str:
|
||
|
|
"""Format a UTC epoch as ISO8601 with a Z suffix, second precision."""
|
||
|
|
return datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
|
|
||
|
|
|
||
|
|
def git_log(repo_dir: Path, branches: list[str]) -> list[dict]:
|
||
|
|
"""Extract events from git log across one or more branches (union, first
|
||
|
|
branch wins on duplicate SHAs)."""
|
||
|
|
branches = _resolve_branches(repo_dir, branches)
|
||
|
|
if not branches:
|
||
|
|
return []
|
||
|
|
|
||
|
|
events = []
|
||
|
|
seen_shas = set()
|
||
|
|
for branch in branches:
|
||
|
|
for event in _git_log_one_branch(repo_dir, branch):
|
||
|
|
if event["sha"] in seen_shas:
|
||
|
|
continue
|
||
|
|
seen_shas.add(event["sha"])
|
||
|
|
events.append(event)
|
||
|
|
return events
|
||
|
|
|
||
|
|
|
||
|
|
def _resolve_branches(repo_dir: Path, branches: list[str]) -> list[str]:
|
||
|
|
"""Resolve the branch list to query: explicit branches, else default
|
||
|
|
branch from origin/HEAD, else current branch."""
|
||
|
|
if branches:
|
||
|
|
return branches
|
||
|
|
|
||
|
|
# Try to get default branch from origin/HEAD
|
||
|
|
try:
|
||
|
|
result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "symbolic-ref", "refs/remotes/origin/HEAD"],
|
||
|
|
capture_output=True, text=True, check=False
|
||
|
|
)
|
||
|
|
if result.returncode == 0:
|
||
|
|
# Typically returns "refs/remotes/origin/main" or similar
|
||
|
|
default_branch = result.stdout.strip().split("/")[-1]
|
||
|
|
branches = [default_branch]
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
|
||
|
|
if not branches:
|
||
|
|
# Fallback to current branch
|
||
|
|
try:
|
||
|
|
result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "rev-parse", "--abbrev-ref", "HEAD"],
|
||
|
|
capture_output=True, text=True, check=True
|
||
|
|
)
|
||
|
|
branches = [result.stdout.strip()]
|
||
|
|
except Exception:
|
||
|
|
return []
|
||
|
|
|
||
|
|
return branches
|
||
|
|
|
||
|
|
|
||
|
|
def _git_log_one_branch(repo_dir: Path, branch: str) -> list[dict]:
|
||
|
|
"""Extract events from git log for a single branch (first-parent,
|
||
|
|
reverse, numstat)."""
|
||
|
|
events = []
|
||
|
|
|
||
|
|
# Get log info and numstat separately
|
||
|
|
try:
|
||
|
|
# First, get commit info (SHA, timestamp, author, subject, body)
|
||
|
|
log_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "log", "--first-parent", "--reverse",
|
||
|
|
"--format=%H%n%aI%n%an%n%s%n%b%n<<<SEP>>>", branch],
|
||
|
|
capture_output=True, text=True, check=True
|
||
|
|
)
|
||
|
|
|
||
|
|
# Get numstat (SHA and file changes)
|
||
|
|
numstat_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "log", "--first-parent", "--reverse",
|
||
|
|
"--numstat", "--format=%H%n<<<STAT_SEP>>>", branch],
|
||
|
|
capture_output=True, text=True, check=True
|
||
|
|
)
|
||
|
|
|
||
|
|
# Parse log output (maintain order)
|
||
|
|
log_blocks = log_result.stdout.split("<<<SEP>>>")
|
||
|
|
commits_ordered = [] # List to maintain order
|
||
|
|
commits_dict = {} # Dict for lookup during numstat
|
||
|
|
|
||
|
|
for block in log_blocks:
|
||
|
|
if not block.strip():
|
||
|
|
continue
|
||
|
|
|
||
|
|
lines = block.strip().split("\n")
|
||
|
|
if len(lines) < 4:
|
||
|
|
continue
|
||
|
|
|
||
|
|
sha = lines[0].strip()
|
||
|
|
ts_str = lines[1].strip()
|
||
|
|
author = lines[2].strip()
|
||
|
|
subject = lines[3].strip()
|
||
|
|
body = "\n".join(lines[4:]).strip() if len(lines) > 4 else ""
|
||
|
|
|
||
|
|
commit_data = {
|
||
|
|
"sha": sha,
|
||
|
|
"ts": normalize_ts(ts_str),
|
||
|
|
"author": author,
|
||
|
|
"subject": subject,
|
||
|
|
"body": body,
|
||
|
|
"files": [],
|
||
|
|
}
|
||
|
|
commits_ordered.append(commit_data)
|
||
|
|
commits_dict[sha] = commit_data
|
||
|
|
|
||
|
|
# Parse numstat output
|
||
|
|
numstat_blocks = numstat_result.stdout.split("<<<STAT_SEP>>>")
|
||
|
|
current_sha = None
|
||
|
|
|
||
|
|
for block in numstat_blocks:
|
||
|
|
lines = block.strip().split("\n")
|
||
|
|
|
||
|
|
for line in lines:
|
||
|
|
line = line.strip()
|
||
|
|
if not line:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# First line in block (after <<<STAT_SEP>>>) is SHA
|
||
|
|
if line and "\t" not in line:
|
||
|
|
current_sha = line
|
||
|
|
if current_sha not in commits_dict:
|
||
|
|
# Shouldn't happen, but handle gracefully
|
||
|
|
commits_dict[current_sha] = {
|
||
|
|
"sha": current_sha,
|
||
|
|
"ts": 0,
|
||
|
|
"author": "",
|
||
|
|
"subject": "",
|
||
|
|
"body": "",
|
||
|
|
"files": [],
|
||
|
|
}
|
||
|
|
commits_ordered.append(commits_dict[current_sha])
|
||
|
|
elif current_sha and "\t" in line:
|
||
|
|
# Numstat line: additions\tdeletions\tpath
|
||
|
|
parts = line.split("\t", 2)
|
||
|
|
if len(parts) >= 3:
|
||
|
|
try:
|
||
|
|
add = int(parts[0]) if parts[0] != "-" else 0
|
||
|
|
dele = int(parts[1]) if parts[1] != "-" else 0
|
||
|
|
path = parts[2]
|
||
|
|
commits_dict[current_sha]["files"].append({
|
||
|
|
"path": path,
|
||
|
|
"add": add,
|
||
|
|
"del": dele
|
||
|
|
})
|
||
|
|
except (ValueError, IndexError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
# Build events in the order we saw commits
|
||
|
|
for c in commits_ordered:
|
||
|
|
event = {
|
||
|
|
"id": f"git:{c['sha']}",
|
||
|
|
"source": "git",
|
||
|
|
"ts": c["ts"],
|
||
|
|
"sha": c["sha"],
|
||
|
|
"author": c["author"],
|
||
|
|
"subject": c["subject"],
|
||
|
|
"body": c["body"],
|
||
|
|
"files": c["files"],
|
||
|
|
}
|
||
|
|
events.append(event)
|
||
|
|
|
||
|
|
except Exception:
|
||
|
|
# Git command failed, return empty
|
||
|
|
pass
|
||
|
|
|
||
|
|
return events
|
||
|
|
|
||
|
|
|
||
|
|
def repo_slug(repo_abs: Path) -> str:
|
||
|
|
"""Slugify an absolute repo path the way Claude Code names transcript dirs.
|
||
|
|
|
||
|
|
Portable: also maps Windows separators and drive colons to dashes, so the
|
||
|
|
slug is stable regardless of platform path syntax.
|
||
|
|
"""
|
||
|
|
# Claude Code maps every non-alphanumeric character to a dash
|
||
|
|
# (verified: p-045_ekar_skills -> p-045-ekar-skills), not just separators.
|
||
|
|
return re.sub(r"[^A-Za-z0-9-]", "-", str(repo_abs))
|
||
|
|
|
||
|
|
|
||
|
|
def sessions_from_transcripts(repo_dir: Path, sessions_dir: Optional[Path]) -> list[dict]:
|
||
|
|
"""Extract events from Claude session transcripts."""
|
||
|
|
events = []
|
||
|
|
|
||
|
|
if sessions_dir is None:
|
||
|
|
sessions_dir = Path.home() / ".claude" / "projects"
|
||
|
|
|
||
|
|
if not sessions_dir.exists():
|
||
|
|
return events
|
||
|
|
|
||
|
|
# Compute repo slug and canonical path (for the cwd filter)
|
||
|
|
repo_abs = repo_dir.resolve()
|
||
|
|
slug = repo_slug(repo_abs)
|
||
|
|
|
||
|
|
# Find session files: <slug>/*.jsonl and <slug>--*/*.jsonl
|
||
|
|
session_globs = [
|
||
|
|
str(sessions_dir / slug / "*.jsonl"),
|
||
|
|
str(sessions_dir / f"{slug}--*/") + "*.jsonl",
|
||
|
|
]
|
||
|
|
|
||
|
|
session_files = []
|
||
|
|
for pattern in session_globs:
|
||
|
|
session_files.extend(glob_module.glob(pattern))
|
||
|
|
|
||
|
|
session_files = sorted(set(session_files))
|
||
|
|
|
||
|
|
for session_file in session_files:
|
||
|
|
session_path = Path(session_file)
|
||
|
|
|
||
|
|
# Read raw lines, keeping the real 1-based line number of each record
|
||
|
|
try:
|
||
|
|
raw_lines = session_path.read_text(encoding="utf-8").splitlines()
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
|
||
|
|
records = [] # list of (lineno, dict)
|
||
|
|
for lineno, line in enumerate(raw_lines, start=1):
|
||
|
|
line = line.strip()
|
||
|
|
if not line:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
rec = json.loads(line)
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
if isinstance(rec, dict):
|
||
|
|
records.append((lineno, rec))
|
||
|
|
|
||
|
|
# Title: the LAST aiTitle in the file, computed once per file (not
|
||
|
|
# rescanned per turn).
|
||
|
|
title = None
|
||
|
|
for _, rec in records:
|
||
|
|
if rec.get("type") == "ai-title" and rec.get("aiTitle"):
|
||
|
|
title = rec["aiTitle"]
|
||
|
|
|
||
|
|
n = len(records)
|
||
|
|
i = 0
|
||
|
|
while i < n:
|
||
|
|
lineno, rec = records[i]
|
||
|
|
|
||
|
|
# Skip non-user messages, meta, and sidechain records
|
||
|
|
if rec.get("type") != "user" or rec.get("isMeta") or rec.get("isSidechain"):
|
||
|
|
i += 1
|
||
|
|
continue
|
||
|
|
|
||
|
|
content = (rec.get("message") or {}).get("content")
|
||
|
|
if isinstance(content, str):
|
||
|
|
user_text = content
|
||
|
|
elif isinstance(content, list):
|
||
|
|
text_blocks = [
|
||
|
|
b.get("text", "") for b in content
|
||
|
|
if isinstance(b, dict) and b.get("type") == "text"
|
||
|
|
]
|
||
|
|
if not text_blocks:
|
||
|
|
# e.g. tool_result-only user record: nothing to extract
|
||
|
|
i += 1
|
||
|
|
continue
|
||
|
|
user_text = "".join(text_blocks)
|
||
|
|
else:
|
||
|
|
i += 1
|
||
|
|
continue
|
||
|
|
|
||
|
|
# cwd filter: only keep records from this repo (or a worktree under it)
|
||
|
|
cwd = rec.get("cwd") or ""
|
||
|
|
cwd_path = Path(cwd) if cwd else None
|
||
|
|
if cwd_path is None or (cwd_path != repo_abs and repo_abs not in cwd_path.parents):
|
||
|
|
i += 1
|
||
|
|
continue
|
||
|
|
|
||
|
|
ts_str = rec.get("timestamp", "")
|
||
|
|
branch = rec.get("gitBranch")
|
||
|
|
|
||
|
|
# Find the outcome: scan forward to the next user record, taking
|
||
|
|
# the LAST non-empty text block across the assistant records
|
||
|
|
# in between (the turn wrap-up).
|
||
|
|
outcome_text = ""
|
||
|
|
j = i + 1
|
||
|
|
while j < n:
|
||
|
|
_, next_rec = records[j]
|
||
|
|
if next_rec.get("type") == "user":
|
||
|
|
break
|
||
|
|
if next_rec.get("type") == "assistant":
|
||
|
|
acontent = (next_rec.get("message") or {}).get("content")
|
||
|
|
if isinstance(acontent, list):
|
||
|
|
for block in acontent:
|
||
|
|
if isinstance(block, dict) and block.get("type") == "text" and block.get("text"):
|
||
|
|
outcome_text = block["text"]
|
||
|
|
j += 1
|
||
|
|
|
||
|
|
event = {
|
||
|
|
"id": f"session:{session_path.name}:{lineno}",
|
||
|
|
"source": "session",
|
||
|
|
"ts": normalize_ts(ts_str),
|
||
|
|
"user": user_text,
|
||
|
|
"outcome": outcome_text if outcome_text else None,
|
||
|
|
"title": title,
|
||
|
|
"branch": branch,
|
||
|
|
}
|
||
|
|
events.append(event)
|
||
|
|
|
||
|
|
i = j
|
||
|
|
|
||
|
|
return events
|
||
|
|
|
||
|
|
|
||
|
|
def apply_skip_rules(event: dict, skip_globs: list[str]) -> Optional[str]:
|
||
|
|
"""Apply skip rules and return rule_id if should be skipped, None otherwise."""
|
||
|
|
for rule in SKIP_RULES:
|
||
|
|
if rule.check(event, skip_globs):
|
||
|
|
return rule.id
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def merge_and_sort(git_events: list[dict], session_events: list[dict]) -> list[dict]:
|
||
|
|
"""Merge and sort events by (ts, source, id)."""
|
||
|
|
all_events = git_events + session_events
|
||
|
|
|
||
|
|
# Sort by timestamp, then source (git first), then id
|
||
|
|
def sort_key(event):
|
||
|
|
source_order = {"git": 0, "session": 1}
|
||
|
|
return (event["ts"], source_order.get(event["source"], 999), event["id"])
|
||
|
|
|
||
|
|
all_events.sort(key=sort_key)
|
||
|
|
return all_events
|
||
|
|
|
||
|
|
|
||
|
|
def truncate_text(text: str, max_len: int = 2000) -> str:
|
||
|
|
"""Apply head+tail truncation with marker."""
|
||
|
|
if len(text) <= max_len:
|
||
|
|
return text
|
||
|
|
|
||
|
|
head_len = max_len * 3 // 4
|
||
|
|
tail_len = max_len - head_len
|
||
|
|
|
||
|
|
head = text[:head_len]
|
||
|
|
tail = text[-tail_len:] if tail_len else ""
|
||
|
|
return head + "\n[...]\n" + tail
|
||
|
|
|
||
|
|
|
||
|
|
def check_coverage(events_path: Path, bundle_dir: Path) -> list[str]:
|
||
|
|
"""Check that all live events (without skip field) are mapped in the bundle.
|
||
|
|
|
||
|
|
Returns a list of unmapped event ids (empty if all are covered).
|
||
|
|
Live events are found by looking for those without a 'skip' key.
|
||
|
|
Coverage is determined by searching for event ids or their short shas in
|
||
|
|
the bundle's .md files (sources and log).
|
||
|
|
"""
|
||
|
|
if not events_path.exists():
|
||
|
|
print(f"Error: {events_path} does not exist", file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
if not bundle_dir.exists():
|
||
|
|
print(f"Error: {bundle_dir} does not exist", file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
# Collect all live event ids
|
||
|
|
live_ids = []
|
||
|
|
try:
|
||
|
|
with open(events_path, "r", encoding="utf-8") as f:
|
||
|
|
for line in f:
|
||
|
|
line = line.strip()
|
||
|
|
if not line:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
event = json.loads(line)
|
||
|
|
if "skip" not in event and "id" in event:
|
||
|
|
live_ids.append(event["id"])
|
||
|
|
except json.JSONDecodeError:
|
||
|
|
continue
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Error reading events: {e}", file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
# Collect all text from .md files in bundle
|
||
|
|
bundle_text = ""
|
||
|
|
try:
|
||
|
|
for md_file in bundle_dir.rglob("*.md"):
|
||
|
|
try:
|
||
|
|
bundle_text += md_file.read_text(encoding="utf-8") + "\n"
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Error reading bundle: {e}", file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
# Check coverage
|
||
|
|
unmapped = []
|
||
|
|
for event_id in live_ids:
|
||
|
|
if event_id.startswith("git:"):
|
||
|
|
# Git events: search for full sha and any >=7-char prefix
|
||
|
|
sha = event_id[4:] # Remove "git:" prefix
|
||
|
|
short_sha = sha[:7]
|
||
|
|
# Check if full sha or short sha appears in bundle
|
||
|
|
if sha not in bundle_text and short_sha not in bundle_text:
|
||
|
|
unmapped.append(event_id)
|
||
|
|
elif event_id.startswith("session:"):
|
||
|
|
# Session events: search for literal id (with colons)
|
||
|
|
if event_id not in bundle_text:
|
||
|
|
unmapped.append(event_id)
|
||
|
|
else:
|
||
|
|
# Unknown format, consider unmapped
|
||
|
|
unmapped.append(event_id)
|
||
|
|
|
||
|
|
return unmapped
|
||
|
|
|
||
|
|
|
||
|
|
def split_patch(patch_text: str) -> list[dict]:
|
||
|
|
"""Split -p output into blocks on lines starting with 'diff --git'.
|
||
|
|
|
||
|
|
Each block contains:
|
||
|
|
- path: str (the b/ path from the diff --git line, or new path for renames)
|
||
|
|
- header: list[str] (lines before first @@ line)
|
||
|
|
- hunks: list[list[str]] (groups starting at each @@ line)
|
||
|
|
"""
|
||
|
|
if not patch_text:
|
||
|
|
return []
|
||
|
|
|
||
|
|
lines = patch_text.split("\n")
|
||
|
|
blocks = []
|
||
|
|
current_block = None
|
||
|
|
|
||
|
|
for line in lines:
|
||
|
|
if line.startswith("diff --git "):
|
||
|
|
# Start new block: extract path from b/ side
|
||
|
|
# Format: diff --git a/<path> b/<path>
|
||
|
|
# ponytail: paths containing " b/" would be incorrectly split; git-quoted paths are out of scope
|
||
|
|
if " b/" in line:
|
||
|
|
path = line.split(" b/", 1)[1]
|
||
|
|
else:
|
||
|
|
path = ""
|
||
|
|
|
||
|
|
if current_block is not None:
|
||
|
|
blocks.append(current_block)
|
||
|
|
current_block = {
|
||
|
|
"path": path,
|
||
|
|
"header": [line],
|
||
|
|
"hunks": [],
|
||
|
|
}
|
||
|
|
elif current_block is not None:
|
||
|
|
if line.startswith("@@"):
|
||
|
|
# Start new hunk
|
||
|
|
current_block["hunks"].append([line])
|
||
|
|
elif current_block["hunks"]:
|
||
|
|
# Add to current hunk
|
||
|
|
current_block["hunks"][-1].append(line)
|
||
|
|
else:
|
||
|
|
# Header line (before first hunk)
|
||
|
|
current_block["header"].append(line)
|
||
|
|
|
||
|
|
if current_block is not None:
|
||
|
|
blocks.append(current_block)
|
||
|
|
|
||
|
|
return blocks
|
||
|
|
|
||
|
|
|
||
|
|
def cap_patch(
|
||
|
|
blocks: list[dict],
|
||
|
|
numstat_lines: list[str],
|
||
|
|
*,
|
||
|
|
per_file_lines: int,
|
||
|
|
max_diff_lines: int,
|
||
|
|
max_line_chars: int,
|
||
|
|
skip_globs: list[str],
|
||
|
|
only: Optional[str] = None,
|
||
|
|
) -> tuple[list[str], dict]:
|
||
|
|
"""Cap patches per-file and globally.
|
||
|
|
|
||
|
|
Returns (patch_lines, summary_dict) where:
|
||
|
|
- patch_lines: list of lines to emit (markers included, no '# patch' header)
|
||
|
|
- summary_dict: {shown, total, files_shown, files_total, truncated}
|
||
|
|
|
||
|
|
Capping rules:
|
||
|
|
- Glob-omitted files: emit marker, consume nothing, don't count toward total
|
||
|
|
- Per-file budget: max(per_file_lines, max_diff_lines - shown) under normal mode,
|
||
|
|
min(max_diff_lines, max_diff_lines - shown) under --only
|
||
|
|
- Header exceeds budget: emit header lines that fit, then accounting line with
|
||
|
|
N = unshown header lines + all hunk lines, M = len(hunks)
|
||
|
|
- Hunk doesn't fit: emit it whole if it fits remaining budget; otherwise emit
|
||
|
|
[+N more lines in M more hunks] or [hunk truncated: +N more lines]
|
||
|
|
- Global cap: before starting file, if shown >= max_diff_lines, emit
|
||
|
|
[patches omitted for K more files (+N lines)] and stop
|
||
|
|
- Line-length rule: lines longer than max_line_chars are truncated with marker
|
||
|
|
"""
|
||
|
|
globs = LOCKFILE_GLOBS + skip_globs
|
||
|
|
patch_lines = []
|
||
|
|
shown = 0
|
||
|
|
files_shown = 0
|
||
|
|
truncated = False
|
||
|
|
|
||
|
|
# Parse numstat for totals and --only validation
|
||
|
|
numstat_by_path = {}
|
||
|
|
for numstat_line in numstat_lines:
|
||
|
|
parts = numstat_line.split("\t", 2)
|
||
|
|
if len(parts) >= 3:
|
||
|
|
add_str, del_str, path = parts[0], parts[1], parts[2]
|
||
|
|
numstat_by_path[path] = (add_str, del_str)
|
||
|
|
|
||
|
|
files_total = len(numstat_lines)
|
||
|
|
|
||
|
|
# Total counts every eligible file (not glob-omitted, matching --only), including
|
||
|
|
# the ones the global cap omits: truncated=true must mean shown < total.
|
||
|
|
eligible = [
|
||
|
|
b for b in blocks
|
||
|
|
if not matches_globs([b["path"]], globs) and (only is None or b["path"] == only)
|
||
|
|
]
|
||
|
|
total = sum(len(b["header"]) + sum(len(h) for h in b["hunks"]) for b in eligible)
|
||
|
|
|
||
|
|
# Validate --only path
|
||
|
|
if only is not None:
|
||
|
|
if only not in numstat_by_path:
|
||
|
|
raise KeyError(f"Path not found in commit: {only}")
|
||
|
|
|
||
|
|
# Process blocks
|
||
|
|
block_idx = 0
|
||
|
|
for block_idx, block in enumerate(blocks):
|
||
|
|
block_path = block["path"]
|
||
|
|
|
||
|
|
# Check glob omission
|
||
|
|
if matches_globs([block_path], globs):
|
||
|
|
# Emit marker, consume nothing
|
||
|
|
if block_path in numstat_by_path:
|
||
|
|
add_str, del_str = numstat_by_path[block_path]
|
||
|
|
else:
|
||
|
|
add_str, del_str = "-", "-"
|
||
|
|
patch_lines.append(f"[patch omitted: {block_path} (generated), {add_str} {del_str}]")
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Check --only filter
|
||
|
|
if only is not None and block_path != only:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Global cap check
|
||
|
|
if shown >= max_diff_lines:
|
||
|
|
# Count remaining non-glob blocks and their lines
|
||
|
|
remaining_files = 0
|
||
|
|
remaining_lines = 0
|
||
|
|
for j in range(block_idx, len(blocks)):
|
||
|
|
remaining_block = blocks[j]
|
||
|
|
if matches_globs([remaining_block["path"]], globs):
|
||
|
|
continue
|
||
|
|
if only is not None and remaining_block["path"] != only:
|
||
|
|
continue
|
||
|
|
remaining_files += 1
|
||
|
|
for hunk in remaining_block["hunks"]:
|
||
|
|
remaining_lines += len(hunk)
|
||
|
|
remaining_lines += len(remaining_block["header"])
|
||
|
|
|
||
|
|
if remaining_files > 0:
|
||
|
|
patch_lines.append(f"[patches omitted for {remaining_files} more files (+{remaining_lines} lines)]")
|
||
|
|
truncated = True
|
||
|
|
break
|
||
|
|
|
||
|
|
# Calculate file budget
|
||
|
|
if only is not None:
|
||
|
|
file_budget = min(max_diff_lines, max_diff_lines - shown)
|
||
|
|
else:
|
||
|
|
file_budget = min(per_file_lines, max_diff_lines - shown)
|
||
|
|
|
||
|
|
# Emit header lines
|
||
|
|
file_lines_emitted = 0
|
||
|
|
header_lines_to_emit = []
|
||
|
|
for hline in block["header"]:
|
||
|
|
if len(hline) > max_line_chars:
|
||
|
|
shortened = hline[:max_line_chars] + f" [line truncated: {len(hline)} chars]"
|
||
|
|
header_lines_to_emit.append(shortened)
|
||
|
|
else:
|
||
|
|
header_lines_to_emit.append(hline)
|
||
|
|
file_lines_emitted += 1
|
||
|
|
if file_lines_emitted > file_budget:
|
||
|
|
# Header alone exceeds budget
|
||
|
|
header_lines_to_emit = header_lines_to_emit[: file_budget]
|
||
|
|
break
|
||
|
|
|
||
|
|
patch_lines.extend(header_lines_to_emit)
|
||
|
|
shown += len(header_lines_to_emit)
|
||
|
|
|
||
|
|
# Process hunks
|
||
|
|
if file_lines_emitted <= file_budget and len(block["header"]) > 0:
|
||
|
|
# Some header was emitted and didn't exceed budget
|
||
|
|
remaining_budget = file_budget - file_lines_emitted
|
||
|
|
else:
|
||
|
|
# Either no header or header exactly filled budget
|
||
|
|
remaining_budget = file_budget - len(header_lines_to_emit)
|
||
|
|
|
||
|
|
hunks_emitted = 0
|
||
|
|
for hunk_idx, hunk in enumerate(block["hunks"]):
|
||
|
|
hunk_size = len(hunk)
|
||
|
|
|
||
|
|
if hunk_size <= remaining_budget:
|
||
|
|
# Hunk fits
|
||
|
|
for hline in hunk:
|
||
|
|
if len(hline) > max_line_chars:
|
||
|
|
shortened = hline[:max_line_chars] + f" [line truncated: {len(hline)} chars]"
|
||
|
|
patch_lines.append(shortened)
|
||
|
|
else:
|
||
|
|
patch_lines.append(hline)
|
||
|
|
shown += 1
|
||
|
|
remaining_budget -= hunk_size
|
||
|
|
hunks_emitted += 1
|
||
|
|
else:
|
||
|
|
# Hunk doesn't fit
|
||
|
|
truncated = True
|
||
|
|
|
||
|
|
if hunks_emitted == 0:
|
||
|
|
# No hunk emitted yet: either emit partial hunk or full accounting
|
||
|
|
if remaining_budget > 0:
|
||
|
|
# Emit first remaining_budget lines of this hunk
|
||
|
|
for hline in hunk[:remaining_budget]:
|
||
|
|
if len(hline) > max_line_chars:
|
||
|
|
shortened = hline[:max_line_chars] + f" [line truncated: {len(hline)} chars]"
|
||
|
|
patch_lines.append(shortened)
|
||
|
|
else:
|
||
|
|
patch_lines.append(hline)
|
||
|
|
shown += 1
|
||
|
|
# Emit truncation marker
|
||
|
|
remaining_hunk_lines = hunk_size - remaining_budget
|
||
|
|
patch_lines.append(f"[hunk truncated: +{remaining_hunk_lines} more lines]")
|
||
|
|
else:
|
||
|
|
# Budget is 0 or negative, emit full accounting
|
||
|
|
remaining_hunk_lines = hunk_size + sum(len(h) for h in block["hunks"][hunk_idx + 1:])
|
||
|
|
remaining_hunks = len(block["hunks"]) - hunk_idx
|
||
|
|
patch_lines.append(f"[+{remaining_hunk_lines} more lines in {remaining_hunks} more hunks of {block_path}]")
|
||
|
|
else:
|
||
|
|
# Some hunks emitted, more remain
|
||
|
|
remaining_hunk_lines = hunk_size + sum(len(h) for h in block["hunks"][hunk_idx + 1:])
|
||
|
|
remaining_hunks = len(block["hunks"]) - hunk_idx
|
||
|
|
patch_lines.append(f"[+{remaining_hunk_lines} more lines in {remaining_hunks} more hunks of {block_path}]")
|
||
|
|
break
|
||
|
|
|
||
|
|
if hunks_emitted > 0 or len(header_lines_to_emit) > 0:
|
||
|
|
files_shown += 1
|
||
|
|
|
||
|
|
summary = {
|
||
|
|
"shown": shown,
|
||
|
|
"total": total,
|
||
|
|
"files_shown": files_shown,
|
||
|
|
"files_total": files_total,
|
||
|
|
"truncated": truncated,
|
||
|
|
}
|
||
|
|
|
||
|
|
return patch_lines, summary
|
||
|
|
|
||
|
|
|
||
|
|
def format_summary(summary: dict) -> str:
|
||
|
|
"""Format the fixed-form summary line."""
|
||
|
|
return (
|
||
|
|
f"[diff: shown={summary['shown']} total={summary['total']} "
|
||
|
|
f"files_shown={summary['files_shown']} files_total={summary['files_total']} "
|
||
|
|
f"truncated={'true' if summary['truncated'] else 'false'}]"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def show_commit(
|
||
|
|
repo_dir: Path,
|
||
|
|
sha: str,
|
||
|
|
*,
|
||
|
|
only: Optional[str] = None,
|
||
|
|
max_diff_lines: int = 300,
|
||
|
|
per_file_lines: int = 120,
|
||
|
|
max_line_chars: int = 400,
|
||
|
|
skip_globs: list[str] = (),
|
||
|
|
) -> str:
|
||
|
|
"""Run git, emit the full capped diff text.
|
||
|
|
|
||
|
|
Raises ValueError for a bad sha (exit 1) or KeyError for --only path not found (exit 2).
|
||
|
|
"""
|
||
|
|
# Resolve the sha first, separately from the parent check
|
||
|
|
resolve_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "rev-parse", "--verify", "--quiet", f"{sha}^{{commit}}"],
|
||
|
|
capture_output=True, text=True, errors="replace", check=False
|
||
|
|
)
|
||
|
|
if resolve_result.returncode != 0:
|
||
|
|
raise ValueError(f"Bad commit: {sha}")
|
||
|
|
|
||
|
|
full_sha = resolve_result.stdout.strip()
|
||
|
|
|
||
|
|
# Check if this is a root commit or has a parent
|
||
|
|
parent_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "rev-parse", "--verify", "--quiet", f"{full_sha}^1"],
|
||
|
|
capture_output=True, text=True, errors="replace", check=False
|
||
|
|
)
|
||
|
|
|
||
|
|
if parent_result.returncode == 0:
|
||
|
|
# Has a parent: diff against first parent
|
||
|
|
base = parent_result.stdout.strip()
|
||
|
|
stat_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "diff", "--no-color", "-M", "--numstat", base, full_sha],
|
||
|
|
capture_output=True, text=True, errors="replace", check=True
|
||
|
|
)
|
||
|
|
patch_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "diff", "--no-color", "-M", "-p", base, full_sha],
|
||
|
|
capture_output=True, text=True, errors="replace", check=True
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
# Root commit: use git show
|
||
|
|
stat_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "show", "--no-color", "-M", "--format=", "--numstat", full_sha],
|
||
|
|
capture_output=True, text=True, errors="replace", check=True
|
||
|
|
)
|
||
|
|
patch_result = subprocess.run(
|
||
|
|
["git", "-C", str(repo_dir), "show", "--no-color", "-M", "--format=", "-p", full_sha],
|
||
|
|
capture_output=True, text=True, errors="replace", check=True
|
||
|
|
)
|
||
|
|
|
||
|
|
# Parse stat
|
||
|
|
numstat_lines = [
|
||
|
|
line.strip()
|
||
|
|
for line in stat_result.stdout.split("\n")
|
||
|
|
if line.strip()
|
||
|
|
]
|
||
|
|
|
||
|
|
# Parse patch
|
||
|
|
blocks = split_patch(patch_result.stdout)
|
||
|
|
|
||
|
|
# Cap the patch
|
||
|
|
patch_lines, summary = cap_patch(
|
||
|
|
blocks,
|
||
|
|
numstat_lines,
|
||
|
|
per_file_lines=per_file_lines,
|
||
|
|
max_diff_lines=max_diff_lines,
|
||
|
|
max_line_chars=max_line_chars,
|
||
|
|
skip_globs=list(skip_globs),
|
||
|
|
only=only,
|
||
|
|
)
|
||
|
|
|
||
|
|
# Build output
|
||
|
|
lines = []
|
||
|
|
lines.append(f"# commit {full_sha}")
|
||
|
|
lines.append("# stat (complete)")
|
||
|
|
lines.extend(numstat_lines)
|
||
|
|
lines.append("# patch")
|
||
|
|
lines.extend(patch_lines)
|
||
|
|
lines.append(format_summary(summary))
|
||
|
|
|
||
|
|
return "\n".join(lines) + "\n"
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser(
|
||
|
|
description="Event-sourcing extractor for OKF bundle reconstruction"
|
||
|
|
)
|
||
|
|
|
||
|
|
# Check if this is a coverage check (--check-coverage mode)
|
||
|
|
if "--check-coverage" in sys.argv:
|
||
|
|
parser.add_argument("--check-coverage", nargs=2, metavar=("EVENTS_JSONL", "BUNDLE_DIR"),
|
||
|
|
help="Check coverage of events in bundle")
|
||
|
|
args = parser.parse_args()
|
||
|
|
if args.check_coverage:
|
||
|
|
events_path = Path(args.check_coverage[0])
|
||
|
|
bundle_dir = Path(args.check_coverage[1])
|
||
|
|
unmapped = check_coverage(events_path, bundle_dir)
|
||
|
|
if unmapped:
|
||
|
|
print(f"Unmapped events (exit 1):")
|
||
|
|
for event_id in unmapped:
|
||
|
|
print(f" {event_id}")
|
||
|
|
sys.exit(1)
|
||
|
|
else:
|
||
|
|
print("All events mapped (exit 0)")
|
||
|
|
sys.exit(0)
|
||
|
|
|
||
|
|
# Normal extraction mode / show mode
|
||
|
|
parser.add_argument("repo_dir", nargs="?", help="Repository directory")
|
||
|
|
parser.add_argument("--out", default="events.jsonl", help="Output file")
|
||
|
|
parser.add_argument(
|
||
|
|
"--branch", action="append", default=[], help="Git branch(es) to extract"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--no-sessions", action="store_true", help="Skip session extraction"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--sessions-dir", type=Path, help="Override sessions directory"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--max-text", type=int, default=2000, help="Max text length per field"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--skip-globs", action="append", default=[], help="Additional skip globs"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--show", help="Show capped diff for a commit (exit 0 on success, 1 on bad sha)"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--only", help="Restrict patch to a single path (with --show)"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--max-diff-lines", type=int, default=300, help="Max diff lines total"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--per-file-lines", type=int, default=120, help="Max diff lines per file"
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--max-line-chars", type=int, default=400, help="Max characters per line"
|
||
|
|
)
|
||
|
|
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
if not args.repo_dir:
|
||
|
|
print("Error: repo_dir is required", file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
repo_dir = Path(args.repo_dir).resolve()
|
||
|
|
if not repo_dir.exists():
|
||
|
|
print(f"Error: {repo_dir} does not exist", file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
# Handle --show mode
|
||
|
|
if args.show:
|
||
|
|
try:
|
||
|
|
output = show_commit(
|
||
|
|
repo_dir,
|
||
|
|
args.show,
|
||
|
|
only=args.only,
|
||
|
|
max_diff_lines=args.max_diff_lines,
|
||
|
|
per_file_lines=args.per_file_lines,
|
||
|
|
max_line_chars=args.max_line_chars,
|
||
|
|
skip_globs=args.skip_globs,
|
||
|
|
)
|
||
|
|
sys.stdout.write(output)
|
||
|
|
sys.exit(0)
|
||
|
|
except ValueError as e:
|
||
|
|
# Bad sha
|
||
|
|
print(str(e), file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
except KeyError as e:
|
||
|
|
# --only path not found
|
||
|
|
print(e.args[0], file=sys.stderr)
|
||
|
|
sys.exit(2)
|
||
|
|
|
||
|
|
# Normal extraction mode
|
||
|
|
# Extract events
|
||
|
|
git_events = git_log(repo_dir, args.branch)
|
||
|
|
session_events = [] if args.no_sessions else sessions_from_transcripts(
|
||
|
|
repo_dir, args.sessions_dir
|
||
|
|
)
|
||
|
|
|
||
|
|
# Merge and sort
|
||
|
|
events = merge_and_sort(git_events, session_events)
|
||
|
|
|
||
|
|
# Apply skip rules and truncate
|
||
|
|
skip_globs = args.skip_globs
|
||
|
|
skip_counts = {}
|
||
|
|
|
||
|
|
for event in events:
|
||
|
|
skip_id = apply_skip_rules(event, skip_globs)
|
||
|
|
if skip_id:
|
||
|
|
event["skip"] = skip_id
|
||
|
|
skip_counts[skip_id] = skip_counts.get(skip_id, 0) + 1
|
||
|
|
|
||
|
|
# Truncate text fields
|
||
|
|
for key in ["subject", "body", "outcome", "user"]:
|
||
|
|
if key in event and isinstance(event[key], str):
|
||
|
|
event[key] = truncate_text(event[key], args.max_text)
|
||
|
|
|
||
|
|
# Emit ts as an ISO8601 UTC string; the epoch float was only needed
|
||
|
|
# for sorting.
|
||
|
|
event["ts"] = epoch_to_iso_z(event["ts"])
|
||
|
|
|
||
|
|
# Write JSONL (deterministic order)
|
||
|
|
out_path = Path(args.out)
|
||
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
|
||
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
||
|
|
for event in events:
|
||
|
|
# Sort keys for determinism
|
||
|
|
sorted_event = {k: event[k] for k in sorted(event.keys())}
|
||
|
|
f.write(json.dumps(sorted_event, separators=(',', ':'), sort_keys=True) + "\n")
|
||
|
|
|
||
|
|
# Report
|
||
|
|
print(f"Extracted {len(events)} events")
|
||
|
|
print(f" Git: {len(git_events)}")
|
||
|
|
print(f" Sessions: {len(session_events)}")
|
||
|
|
if skip_counts:
|
||
|
|
print("Skipped:")
|
||
|
|
for rule_id, count in sorted(skip_counts.items()):
|
||
|
|
print(f" {rule_id}: {count}")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|