{ "schema": "thruline-bench-release/1", "benchmark": "decision-capture", "version": "v0.1", "updated": "2026-10-31", "set": { "name": "pilot-v2", "sha256": "dce8095600b3ca23f06063dc801ede7d9ab6eeec997764098bc5d8e9cab78e89", "scenarios": 16, "public": 16, "heldOut": 0, "types": [ "made", "reversed", "someone else decided", "partial plan", "deferred", "delegated", "not a decision", "off the record" ], "phrasings": [ "blunt", "multi-turn" ] }, "runs": [ { "id": "2026-10-31-00-51", "date": "2026-10-31", "sessions": 224, "spend": 5.2354645481, "scoredAt": "2026-10-31T17:23:27.431Z", "scoredWith": { "set": "pilot-v2", "setSha": "dce8095600b3ca23f06063dc801ede7d9ab6eeec997764098bc5d8e9cab78e89", "scorer": "04087fe3801f", "postHoc": true } }, { "id": "2026-10-31-02-16", "date": "2026-10-31", "sessions": 120, "spend": 3.895828812700001, "scoredAt": "2026-10-31T17:23:27.608Z", "commit": "7734efc012ad059619aa98a139474d622aecbd98", "scoredWith": { "set": "pilot-v2", "setSha": "dce8095600b3ca23f06063dc801ede7d9ab6eeec997764098bc5d8e9cab78e89", "scorer": "04087fe3801f", "postHoc": true } } ], "default": { "tools": "few", "prompt": "generic" }, "headline": { "run": "2026-10-31-00-51", "measure": "right", "off": { "k": 52, "n": 63, "rate": 0.8253968253968254, "ci": [ 0.713783402655744, 0.8996070607682313 ] }, "on": { "k": 71, "n": 74, "rate": 0.9594594594594594, "ci": [ 0.8874510681944234, 0.9861178204204044 ] }, "onlyWith": 9, "onlyWithout": 1, "p": 0.021484375, "strict": { "off": { "k": 45, "n": 63, "rate": 0.7142857142857143, "ci": [ 0.5929704958526203, 0.8109695654265832 ] }, "on": { "k": 66, "n": 74, "rate": 0.8918918918918919, "ci": [ 0.8009095372797369, 0.9441933383034987 ] }, "p": 0.00634765625 } }, "models": [ { "id": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "provider": "Anthropic", "added": "2026-10-31", "rows": [ { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 14, "n": 16, "rate": 0.875, "ci": [ 0.6397666856325173, 0.9650232506629428 ] }, "strict": { "k": 13, "n": 16, "rate": 0.8125, "ci": [ 0.5699064013154177, 0.9340852122641324 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "offRecord": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "invented": { "k": 0, "n": 17, "rate": 0, "ci": [ 0, 0.18432366037156456 ] }, "duplicates": { "k": 5, "n": 8, "rate": 0.625, "ci": [ 0.30573785458380187, 0.863158240538479 ] }, "points": { "mean": 3.823529411764706, "ci": [ 2.933333333333333, 4.529411764705882 ], "reports": 17, "fields": { "summary": { "mean": 1, "n": 15 }, "goal": { "mean": 1, "n": 9 }, "metric": { "mean": 1, "n": 6 }, "direction": { "mean": 1, "n": 6 }, "why": { "mean": 1, "n": 7 }, "judge_by": { "mean": 1, "n": 3 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5080, "perSession": { "p25": 30480, "median": 35711, "p75": 46013 }, "sessions": 16 }, "costPerSession": 0.109588625, "costPerCapture": 0.21917725 }, { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 13, "n": 16, "rate": 0.8125, "ci": [ 0.5699064013154177, 0.9340852122641324 ] }, "strict": { "k": 12, "n": 16, "rate": 0.75, "ci": [ 0.5050123107060135, 0.8981809801576265 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 1, "n": 4, "rate": 0.25, "ci": [ 0.045586062644636216, 0.6993639475573634 ] }, "offRecord": { "k": 1, "n": 2, "rate": 0.5, "ci": [ 0.09452865480086614, 0.9054713451991339 ] }, "invented": { "k": 0, "n": 20, "rate": 0, "ci": [ 0, 0.16113012549493322 ] }, "duplicates": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "points": { "mean": 4, "ci": [ 3.2941176470588234, 4.62962962962963 ], "reports": 20, "fields": { "summary": { "mean": 1, "n": 18 }, "why": { "mean": 0.8181818181818182, "n": 11 }, "goal": { "mean": 1, "n": 10 }, "direction": { "mean": 1, "n": 10 }, "metric": { "mean": 1, "n": 7 }, "judge_by": { "mean": 1, "n": 4 }, "decided_by": { "mean": 1, "n": 4 } } }, "context": { "perRequest": 2058, "perSession": { "p25": 14679, "median": 16598, "p75": 21135 }, "sessions": 16 }, "costPerSession": 0.10480987500000001, "costPerCapture": 0.20961975000000002 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "compact", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "strict": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 9, "rate": 0, "ci": [ 0, 0.2991527535509594 ] }, "duplicates": { "k": 1, "n": 8, "rate": 0.125, "ci": [ 0.02241690886329617, 0.4708948057698615 ] }, "points": { "mean": 5.111111111111111, "ci": [ 4.375, 5.9 ], "reports": 9, "fields": { "summary": { "mean": 1, "n": 9 }, "why": { "mean": 1, "n": 5 }, "goal": { "mean": 1, "n": 7 }, "metric": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 7 }, "judge_by": { "mean": 1, "n": 2 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 3063, "perSession": { "p25": 24778, "median": 31205, "p75": 34096 }, "sessions": 8 }, "costPerSession": 0.11166849999999999, "costPerCapture": 0.11166849999999999 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "strict": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "points": { "mean": 5.625, "ci": [ 4.75, 6.5 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "goal": { "mean": 1, "n": 6 }, "metric": { "mean": 1, "n": 6 }, "direction": { "mean": 1, "n": 6 }, "judge_by": { "mean": 1, "n": 4 }, "why": { "mean": 1, "n": 5 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5291, "perSession": { "p25": 42602, "median": 53219, "p75": 58599 }, "sessions": 8 }, "costPerSession": 0.13539549999999997, "costPerCapture": 0.13539549999999997 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "strict": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "points": { "mean": 4.75, "ci": [ 3.375, 6 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "why": { "mean": 0.42857142857142855, "n": 7 }, "goal": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 5 }, "judge_by": { "mean": 1, "n": 4 }, "decided_by": { "mean": 1, "n": 2 }, "metric": { "mean": 1, "n": 4 } } }, "context": { "perRequest": 2238, "perSession": { "p25": 20548, "median": 24904, "p75": 27509 }, "sessions": 8 }, "costPerSession": 0.1237095, "costPerCapture": 0.1237095 } ] }, { "id": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "provider": "Anthropic", "added": "2026-10-31", "rows": [ { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 13, "n": 16, "rate": 0.8125, "ci": [ 0.5699064013154177, 0.9340852122641324 ] }, "strict": { "k": 12, "n": 16, "rate": 0.75, "ci": [ 0.5050123107060135, 0.8981809801576265 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 1, "n": 4, "rate": 0.25, "ci": [ 0.045586062644636216, 0.6993639475573634 ] }, "offRecord": { "k": 1, "n": 2, "rate": 0.5, "ci": [ 0.09452865480086614, 0.9054713451991339 ] }, "invented": { "k": 0, "n": 22, "rate": 0, "ci": [ 1.3877787807814457e-17, 0.14865952572596125 ] }, "duplicates": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "points": { "mean": 3.272727272727273, "ci": [ 2.388888888888889, 3.8461538461538463 ], "reports": 22, "fields": { "summary": { "mean": 1, "n": 20 }, "why": { "mean": 0.5, "n": 12 }, "goal": { "mean": 0.8181818181818182, "n": 11 }, "direction": { "mean": 1, "n": 10 }, "metric": { "mean": 1, "n": 7 }, "judge_by": { "mean": 1, "n": 4 }, "decided_by": { "mean": 1, "n": 4 } } }, "context": { "perRequest": 2058, "perSession": { "p25": 14406, "median": 18731, "p75": 22697 }, "sessions": 16 }, "costPerSession": 0.00503810625, "costPerCapture": 0.0100762125 }, { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 16, "n": 16, "rate": 1, "ci": [ 0.8063865817272802, 1 ] }, "strict": { "k": 15, "n": 16, "rate": 0.9375, "ci": [ 0.7167073163197013, 0.9888809426916687 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "offRecord": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "invented": { "k": 0, "n": 12, "rate": 0, "ci": [ 0, 0.2425007574992425 ] }, "duplicates": { "k": 3, "n": 8, "rate": 0.375, "ci": [ 0.13684175946152105, 0.6942621454161981 ] }, "points": { "mean": 3.4166666666666665, "ci": [ 2.75, 3.9166666666666665 ], "reports": 12, "fields": { "summary": { "mean": 1, "n": 11 }, "why": { "mean": 0.6666666666666666, "n": 6 }, "goal": { "mean": 1, "n": 5 }, "metric": { "mean": 1, "n": 3 }, "direction": { "mean": 1, "n": 4 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5080, "perSession": { "p25": 30584, "median": 35576, "p75": 40858 }, "sessions": 16 }, "costPerSession": 0.0052918375, "costPerCapture": 0.010583675 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "strict": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 9, "rate": 0, "ci": [ 0, 0.2991527535509594 ] }, "duplicates": { "k": 1, "n": 8, "rate": 0.125, "ci": [ 0.02241690886329617, 0.4708948057698615 ] }, "points": { "mean": 4.777777777777778, "ci": [ 4, 5.5 ], "reports": 9, "fields": { "summary": { "mean": 1, "n": 9 }, "goal": { "mean": 1, "n": 6 }, "metric": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 6 }, "judge_by": { "mean": 1, "n": 4 }, "why": { "mean": 0.6, "n": 5 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 2238, "perSession": { "p25": 20643, "median": 22770, "p75": 25061 }, "sessions": 8 }, "costPerSession": 0.0056075249999999995, "costPerCapture": 0.0056075249999999995 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "compact", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "strict": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "points": { "mean": 5, "ci": [ 4.375, 5.75 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "why": { "mean": 1, "n": 5 }, "decided_by": { "mean": 1, "n": 2 }, "goal": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 5 }, "metric": { "mean": 1, "n": 4 }, "judge_by": { "mean": 1, "n": 3 } } }, "context": { "perRequest": 3063, "perSession": { "p25": 24855, "median": 27957, "p75": 31237 }, "sessions": 8 }, "costPerSession": 0.005491725, "costPerCapture": 0.005491725 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "strict": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 9, "rate": 0, "ci": [ 0, 0.2991527535509594 ] }, "duplicates": { "k": 1, "n": 8, "rate": 0.125, "ci": [ 0.02241690886329617, 0.4708948057698615 ] }, "points": { "mean": 5, "ci": [ 3.6666666666666665, 6.222222222222222 ], "reports": 9, "fields": { "summary": { "mean": 1, "n": 9 }, "goal": { "mean": 1, "n": 7 }, "metric": { "mean": 0.7142857142857143, "n": 7 }, "direction": { "mean": 0.7142857142857143, "n": 7 }, "why": { "mean": 0.6666666666666666, "n": 6 }, "judge_by": { "mean": 1, "n": 4 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5291, "perSession": { "p25": 37301, "median": 48045, "p75": 53383 }, "sessions": 8 }, "costPerSession": 0.0067306625, "costPerCapture": 0.0067306625 } ] }, { "id": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "provider": "Google", "added": "2026-10-31", "rows": [ { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 16, "failed": 5, "right": { "k": 11, "n": 11, "rate": 1, "ci": [ 0.7411599827511859, 1 ] }, "strict": { "k": 10, "n": 11, "rate": 0.9090909090909091, "ci": [ 0.6226353745137962, 0.9837682477371741 ] }, "capture": { "k": 7, "n": 7, "rate": 1, "ci": [ 0.6456611570247934, 1 ] }, "falseReports": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 11, "rate": 0, "ci": [ 2.7755575615628914e-17, 0.2588400172488141 ] }, "duplicates": { "k": 3, "n": 7, "rate": 0.42857142857142855, "ci": [ 0.15821692226262685, 0.7495457695909742 ] }, "points": { "mean": 3.5454545454545454, "ci": [ 2.8, 4.3 ], "reports": 11, "fields": { "summary": { "mean": 1, "n": 10 }, "goal": { "mean": 0.5, "n": 4 }, "metric": { "mean": 1, "n": 3 }, "direction": { "mean": 1, "n": 3 }, "decided_by": { "mean": 1, "n": 3 }, "why": { "mean": 1, "n": 6 }, "judge_by": { "mean": 1, "n": 1 } } }, "context": { "perRequest": 5080, "perSession": { "p25": 30601, "median": 45964, "p75": 61021 }, "sessions": 11 }, "costPerSession": 0.032709571875, "costPerCapture": 0.07476473571428573 }, { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 16, "failed": 6, "right": { "k": 8, "n": 10, "rate": 0.8, "ci": [ 0.49015684672072335, 0.9433190520193067 ] }, "strict": { "k": 6, "n": 10, "rate": 0.6, "ci": [ 0.3126695474501864, 0.8318224187964902 ] }, "capture": { "k": 7, "n": 7, "rate": 1, "ci": [ 0.6456611570247934, 1 ] }, "falseReports": { "k": 0, "n": 1, "rate": 0, "ci": [ 0, 0.7934567085261071 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 20, "rate": 0, "ci": [ 0, 0.16113012549493322 ] }, "duplicates": { "k": 6, "n": 7, "rate": 0.8571428571428571, "ci": [ 0.48686549668097007, 0.9743210440510253 ] }, "points": { "mean": 3.55, "ci": [ 2.2666666666666666, 4.291666666666667 ], "reports": 20, "fields": { "summary": { "mean": 1, "n": 18 }, "goal": { "mean": 0.8333333333333334, "n": 12 }, "direction": { "mean": 1, "n": 10 }, "metric": { "mean": 1, "n": 8 }, "judge_by": { "mean": 0.5, "n": 4 }, "why": { "mean": 0.07692307692307693, "n": 13 }, "decided_by": { "mean": 1, "n": 3 } } }, "context": { "perRequest": 2058, "perSession": { "p25": 16872, "median": 24935, "p75": 27261 }, "sessions": 10 }, "costPerSession": 0.0201969140625, "costPerCapture": 0.046164375 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 8, "failed": 1, "right": { "k": 5, "n": 7, "rate": 0.7142857142857143, "ci": [ 0.35892909014821267, 0.9177828342909844 ] }, "strict": { "k": 5, "n": 7, "rate": 0.7142857142857143, "ci": [ 0.35892909014821267, 0.9177828342909844 ] }, "capture": { "k": 7, "n": 7, "rate": 1, "ci": [ 0.6456611570247934, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 1, "n": 7, "rate": 0.14285714285714285, "ci": [ 0.02567895594897479, 0.51313450331903 ] }, "points": { "mean": 5, "ci": [ 4, 5.857142857142857 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "why": { "mean": 0.6666666666666666, "n": 6 }, "goal": { "mean": 1, "n": 6 }, "direction": { "mean": 1, "n": 6 }, "judge_by": { "mean": 1, "n": 3 }, "metric": { "mean": 1, "n": 4 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 2238, "perSession": { "p25": 20400, "median": 29514, "p75": 38512 }, "sessions": 7 }, "costPerSession": 0.026209425000000005, "costPerCapture": 0.029953628571428578 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 8, "failed": 2, "right": { "k": 6, "n": 6, "rate": 1, "ci": [ 0.6096569663469354, 0.9999999999999999 ] }, "strict": { "k": 6, "n": 6, "rate": 1, "ci": [ 0.6096569663469354, 0.9999999999999999 ] }, "capture": { "k": 6, "n": 6, "rate": 1, "ci": [ 0.6096569663469354, 0.9999999999999999 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 6, "rate": 0, "ci": [ 0, 0.3903430336530645 ] }, "duplicates": { "k": 0, "n": 6, "rate": 0, "ci": [ 0, 0.3903430336530645 ] }, "points": { "mean": 4, "ci": [ 2.6666666666666665, 5.333333333333333 ], "reports": 6, "fields": { "summary": { "mean": 1, "n": 6 }, "goal": { "mean": 1, "n": 3 }, "metric": { "mean": 0.3333333333333333, "n": 3 }, "direction": { "mean": 0.3333333333333333, "n": 3 }, "why": { "mean": 1, "n": 4 }, "judge_by": { "mean": 1, "n": 1 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5291, "perSession": { "p25": 26648, "median": 47883, "p75": 53199 }, "sessions": 6 }, "costPerSession": 0.029327090624999996, "costPerCapture": 0.03910278749999999 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "compact", "split": "all", "sessions": 8, "failed": 1, "right": { "k": 6, "n": 7, "rate": 0.8571428571428571, "ci": [ 0.48686549668097007, 0.9743210440510253 ] }, "strict": { "k": 6, "n": 7, "rate": 0.8571428571428571, "ci": [ 0.48686549668097007, 0.9743210440510253 ] }, "capture": { "k": 7, "n": 7, "rate": 1, "ci": [ 0.6456611570247934, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 1, "n": 7, "rate": 0.14285714285714285, "ci": [ 0.02567895594897479, 0.51313450331903 ] }, "points": { "mean": 4.625, "ci": [ 3.142857142857143, 5.777777777777778 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "goal": { "mean": 1, "n": 6 }, "metric": { "mean": 0.6, "n": 5 }, "direction": { "mean": 0.6666666666666666, "n": 6 }, "why": { "mean": 1, "n": 4 }, "decided_by": { "mean": 1, "n": 2 }, "judge_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 3063, "perSession": { "p25": 18636, "median": 33957, "p75": 46378 }, "sessions": 7 }, "costPerSession": 0.029624568750000004, "costPerCapture": 0.03385665 } ] }, { "id": "deepseek/deepseek-v4.1-flash", "name": "DeepSeek V4.1 Flash", "provider": "DeepSeek", "added": "2026-10-31", "rows": [ { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 16, "failed": 1, "right": { "k": 15, "n": 15, "rate": 1, "ci": [ 0.7961107336956521, 1 ] }, "strict": { "k": 14, "n": 15, "rate": 0.9333333333333333, "ci": [ 0.7018292244757601, 0.9881334113938051 ] }, "capture": { "k": 7, "n": 7, "rate": 1, "ci": [ 0.6456611570247934, 1 ] }, "falseReports": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "offRecord": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "invented": { "k": 0, "n": 13, "rate": 0, "ci": [ 0, 0.22810184305529166 ] }, "duplicates": { "k": 3, "n": 7, "rate": 0.42857142857142855, "ci": [ 0.15821692226262685, 0.7495457695909742 ] }, "points": { "mean": 3.8461538461538463, "ci": [ 2.75, 4.545454545454546 ], "reports": 13, "fields": { "summary": { "mean": 1, "n": 12 }, "why": { "mean": 1, "n": 5 }, "goal": { "mean": 1, "n": 7 }, "metric": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 5 }, "judge_by": { "mean": 1, "n": 1 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5080, "perSession": { "p25": 25519, "median": 35666, "p75": 51168 }, "sessions": 15 }, "costPerSession": 0.0035242611312499995, "costPerCapture": 0.008055454014285713 }, { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 16, "failed": 8, "right": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "strict": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "capture": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "falseReports": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "offRecord": { "k": 0, "n": 1, "rate": 0, "ci": [ 0, 0.7934567085261071 ] }, "invented": { "k": 0, "n": 12, "rate": 0, "ci": [ 0, 0.2425007574992425 ] }, "duplicates": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "points": { "mean": 3.1666666666666665, "ci": [ 2, 3.9285714285714284 ], "reports": 12, "fields": { "why": { "mean": 0, "n": 8 }, "summary": { "mean": 1, "n": 11 }, "goal": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 5 }, "metric": { "mean": 1, "n": 4 }, "judge_by": { "mean": 1, "n": 1 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 2058, "perSession": { "p25": 24785, "median": 24935, "p75": 31527 }, "sessions": 8 }, "costPerSession": 0.0028996118750000003, "costPerCapture": 0.011598447500000001 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 8, "failed": 4, "right": { "k": 3, "n": 4, "rate": 0.75, "ci": [ 0.3006360524426366, 0.9544139373553637 ] }, "strict": { "k": 3, "n": 4, "rate": 0.75, "ci": [ 0.3006360524426366, 0.9544139373553637 ] }, "capture": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 1, "n": 4, "rate": 0.25, "ci": [ 0.045586062644636216, 0.6993639475573634 ] }, "duplicates": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "points": { "mean": 3.5, "ci": [ 1.5, 6 ], "reports": 4, "fields": { "summary": { "mean": 1, "n": 4 }, "why": { "mean": 0.3333333333333333, "n": 3 }, "metric": { "mean": 1, "n": 2 }, "direction": { "mean": 1, "n": 2 }, "judge_by": { "mean": 0, "n": 2 }, "decided_by": { "mean": 1, "n": 2 }, "goal": { "mean": 1, "n": 1 } } }, "context": { "perRequest": 2238, "perSession": { "p25": 29475, "median": 36191, "p75": 38880 }, "sessions": 4 }, "costPerSession": 0.0030453826375, "costPerCapture": 0.006090765275 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "strict": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "points": { "mean": 5, "ci": [ 4.375, 5.75 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "goal": { "mean": 1, "n": 4 }, "metric": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 4 }, "judge_by": { "mean": 1, "n": 4 }, "why": { "mean": 1, "n": 5 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5291, "perSession": { "p25": 47923, "median": 53345, "p75": 74684 }, "sessions": 8 }, "costPerSession": 0.0035900827, "costPerCapture": 0.0035900827 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "compact", "split": "all", "sessions": 8, "failed": 2, "right": { "k": 4, "n": 6, "rate": 0.6666666666666666, "ci": [ 0.29998832134152864, 0.9032306674407831 ] }, "strict": { "k": 4, "n": 6, "rate": 0.6666666666666666, "ci": [ 0.29998832134152864, 0.9032306674407831 ] }, "capture": { "k": 6, "n": 6, "rate": 1, "ci": [ 0.6096569663469354, 0.9999999999999999 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 2, "n": 6, "rate": 0.3333333333333333, "ci": [ 0.09676933255921683, 0.7000116786584712 ] }, "points": { "mean": 4.5, "ci": [ 3.75, 5.571428571428571 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "goal": { "mean": 1, "n": 4 }, "metric": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 4 }, "why": { "mean": 1, "n": 4 }, "judge_by": { "mean": 1, "n": 1 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 3063, "perSession": { "p25": 21891, "median": 34012, "p75": 46373 }, "sessions": 6 }, "costPerSession": 0.002946055, "costPerCapture": 0.003928073333333333 } ] }, { "id": "openai/gpt-6-luna", "name": "GPT-6 Luna", "provider": "OpenAI", "added": "2026-10-31", "rows": [ { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 16, "failed": 3, "right": { "k": 11, "n": 13, "rate": 0.8461538461538461, "ci": [ 0.5776482949136899, 0.9567427368172621 ] }, "strict": { "k": 9, "n": 13, "rate": 0.6923076923076923, "ci": [ 0.4236888781333764, 0.8731950283838191 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "offRecord": { "k": 0, "n": 1, "rate": 0, "ci": [ 0, 0.7934567085261071 ] }, "invented": { "k": 0, "n": 25, "rate": 0, "ci": [ 0, 0.13319649395317873 ] }, "duplicates": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "points": { "mean": 2.72, "ci": [ 1.9, 3.3548387096774195 ], "reports": 25, "fields": { "summary": { "mean": 1, "n": 21 }, "decided_by": { "mean": 1, "n": 4 }, "why": { "mean": 0, "n": 16 }, "goal": { "mean": 0.2727272727272727, "n": 11 }, "metric": { "mean": 0.75, "n": 8 }, "direction": { "mean": 0.75, "n": 8 }, "judge_by": { "mean": 1, "n": 3 } } }, "context": { "perRequest": 2058, "perSession": { "p25": 14597, "median": 16595, "p75": 23004 }, "sessions": 13 }, "costPerSession": 0.0010083640625, "costPerCapture": 0.002016728125 }, { "run": "2026-10-31-00-51", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 15, "n": 16, "rate": 0.9375, "ci": [ 0.7167073163197013, 0.9888809426916687 ] }, "strict": { "k": 14, "n": 16, "rate": 0.875, "ci": [ 0.6397666856325173, 0.9650232506629428 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "offRecord": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "invented": { "k": 0, "n": 15, "rate": 0, "ci": [ 0, 0.20388926630434784 ] }, "duplicates": { "k": 4, "n": 8, "rate": 0.5, "ci": [ 0.21521252682444186, 0.7847874731755582 ] }, "points": { "mean": 3.4, "ci": [ 2, 4.4375 ], "reports": 15, "fields": { "summary": { "mean": 1, "n": 13 }, "why": { "mean": 0.5, "n": 8 }, "decided_by": { "mean": 1, "n": 2 }, "goal": { "mean": 1, "n": 8 }, "metric": { "mean": 0.6, "n": 5 }, "direction": { "mean": 0.6, "n": 5 }, "judge_by": { "mean": 1, "n": 3 } } }, "context": { "perRequest": 5080, "perSession": { "p25": 25524, "median": 40759, "p75": 46090 }, "sessions": 16 }, "costPerSession": 0.0012656924999999996, "costPerCapture": 0.0025313849999999993 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "full", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "strict": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "duplicates": { "k": 0, "n": 8, "rate": 0, "ci": [ 0, 0.3244156195108769 ] }, "points": { "mean": 3.625, "ci": [ 2.5, 4.875 ], "reports": 8, "fields": { "summary": { "mean": 1, "n": 8 }, "goal": { "mean": 0.3333333333333333, "n": 6 }, "metric": { "mean": 0.5, "n": 4 }, "why": { "mean": 1, "n": 4 }, "direction": { "mean": 0.3333333333333333, "n": 3 }, "judge_by": { "mean": 1, "n": 2 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5291, "perSession": { "p25": 31925, "median": 42602, "p75": 53236 }, "sessions": 8 }, "costPerSession": 0.0012753618750000002, "costPerCapture": 0.0012753618750000002 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "compact", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "strict": { "k": 7, "n": 8, "rate": 0.875, "ci": [ 0.5291051942301386, 0.9775830911367038 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 9, "rate": 0, "ci": [ 0, 0.2991527535509594 ] }, "duplicates": { "k": 1, "n": 8, "rate": 0.125, "ci": [ 0.02241690886329617, 0.4708948057698615 ] }, "points": { "mean": 3.5555555555555554, "ci": [ 2.9, 4.375 ], "reports": 9, "fields": { "summary": { "mean": 1, "n": 9 }, "why": { "mean": 0.6, "n": 5 }, "judge_by": { "mean": 1, "n": 3 }, "goal": { "mean": 1, "n": 4 }, "direction": { "mean": 1, "n": 2 }, "metric": { "mean": 1, "n": 1 }, "decided_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 3063, "perSession": { "p25": 18557, "median": 21874, "p75": 40424 }, "sessions": 8 }, "costPerSession": 0.001238416875, "costPerCapture": 0.001238416875 }, { "run": "2026-10-31-02-16", "tools": "few", "prompt": "generic", "skill": "none", "split": "all", "sessions": 8, "failed": 0, "right": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "strict": { "k": 6, "n": 8, "rate": 0.75, "ci": [ 0.40926987910258916, 0.9285223111419724 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "offRecord": { "k": 0, "n": 0, "rate": 0, "ci": [ 0, 1 ] }, "invented": { "k": 0, "n": 10, "rate": 0, "ci": [ 0, 0.2775401687666166 ] }, "duplicates": { "k": 1, "n": 8, "rate": 0.125, "ci": [ 0.02241690886329617, 0.4708948057698615 ] }, "points": { "mean": 3.1, "ci": [ 2.0833333333333335, 4.625 ], "reports": 10, "fields": { "summary": { "mean": 1, "n": 9 }, "why": { "mean": -0.14285714285714285, "n": 7 }, "decided_by": { "mean": 1, "n": 2 }, "metric": { "mean": 0.6, "n": 5 }, "direction": { "mean": 0.6, "n": 5 }, "goal": { "mean": 0.6, "n": 5 }, "judge_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 2238, "perSession": { "p25": 20425, "median": 20759, "p75": 29497 }, "sessions": 8 }, "costPerSession": 0.001118805625, "costPerCapture": 0.001118805625 } ] } ], "otherSettings": [ { "model": "anthropic/claude-haiku-5.5", "run": "2026-10-31-00-51", "tools": "many", "prompt": "product", "skill": "none", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 13, "n": 16, "rate": 0.8125, "ci": [ 0.5699064013154177, 0.9340852122641324 ] }, "strict": { "k": 12, "n": 16, "rate": 0.75, "ci": [ 0.5050123107060135, 0.8981809801576265 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 1, "n": 4, "rate": 0.25, "ci": [ 0.045586062644636216, 0.6993639475573634 ] }, "offRecord": { "k": 1, "n": 2, "rate": 0.5, "ci": [ 0.09452865480086614, 0.9054713451991339 ] }, "invented": { "k": 0, "n": 21, "rate": 0, "ci": [ 0, 0.15464382326420195 ] }, "duplicates": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "points": { "mean": 3.1904761904761907, "ci": [ 2.5, 3.72 ], "reports": 21, "fields": { "summary": { "mean": 1, "n": 19 }, "decided_by": { "mean": 1, "n": 4 }, "why": { "mean": 1, "n": 5 }, "goal": { "mean": 1, "n": 8 }, "metric": { "mean": 1, "n": 7 }, "direction": { "mean": 1, "n": 8 }, "judge_by": { "mean": 1, "n": 3 } } }, "context": { "perRequest": 2058, "perSession": { "p25": 14406, "median": 16758, "p75": 22872 }, "sessions": 16 }, "costPerSession": 0.016784393749999998, "costPerCapture": 0.033568787499999996 }, { "model": "anthropic/claude-haiku-5.5", "run": "2026-10-31-00-51", "tools": "many", "prompt": "product", "skill": "full", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 16, "n": 16, "rate": 1, "ci": [ 0.8063865817272802, 1 ] }, "strict": { "k": 15, "n": 16, "rate": 0.9375, "ci": [ 0.7167073163197013, 0.9888809426916687 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "offRecord": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "invented": { "k": 0, "n": 13, "rate": 0, "ci": [ 0, 0.22810184305529166 ] }, "duplicates": { "k": 4, "n": 8, "rate": 0.5, "ci": [ 0.21521252682444186, 0.7847874731755582 ] }, "points": { "mean": 4.076923076923077, "ci": [ 3.1818181818181817, 4.857142857142857 ], "reports": 13, "fields": { "summary": { "mean": 1, "n": 12 }, "decided_by": { "mean": 1, "n": 2 }, "why": { "mean": 1, "n": 6 }, "goal": { "mean": 1, "n": 7 }, "metric": { "mean": 1, "n": 5 }, "direction": { "mean": 1, "n": 6 }, "judge_by": { "mean": 1, "n": 2 } } }, "context": { "perRequest": 5080, "perSession": { "p25": 35560, "median": 40744, "p75": 45993 }, "sessions": 16 }, "costPerSession": 0.01708014375, "costPerCapture": 0.0341602875 }, { "model": "anthropic/claude-haiku-5.5", "run": "2026-10-31-00-51", "tools": "none", "prompt": "none", "skill": "none", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 13, "n": 16, "rate": 0.8125, "ci": [ 0.5699064013154177, 0.9340852122641324 ] }, "strict": { "k": 12, "n": 16, "rate": 0.75, "ci": [ 0.5050123107060135, 0.8981809801576265 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 1, "n": 4, "rate": 0.25, "ci": [ 0.045586062644636216, 0.6993639475573634 ] }, "offRecord": { "k": 1, "n": 2, "rate": 0.5, "ci": [ 0.09452865480086614, 0.9054713451991339 ] }, "invented": { "k": 0, "n": 23, "rate": 0, "ci": [ 0, 0.14312112541726274 ] }, "duplicates": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "points": { "mean": 2.9565217391304346, "ci": [ 1.9565217391304348, 3.727272727272727 ], "reports": 23, "fields": { "summary": { "mean": 1, "n": 21 }, "decided_by": { "mean": 1, "n": 4 }, "why": { "mean": 0.3333333333333333, "n": 12 }, "goal": { "mean": 0.6923076923076923, "n": 13 }, "metric": { "mean": 0.75, "n": 8 }, "direction": { "mean": 0.8181818181818182, "n": 11 }, "judge_by": { "mean": 1, "n": 3 } } }, "context": { "perRequest": 2058, "perSession": { "p25": 14525, "median": 16603, "p75": 21163 }, "sessions": 16 }, "costPerSession": 0.003232275, "costPerCapture": 0.00646455 }, { "model": "anthropic/claude-haiku-5.5", "run": "2026-10-31-00-51", "tools": "none", "prompt": "none", "skill": "full", "split": "all", "sessions": 16, "failed": 0, "right": { "k": 16, "n": 16, "rate": 1, "ci": [ 0.8063865817272802, 1 ] }, "strict": { "k": 15, "n": 16, "rate": 0.9375, "ci": [ 0.7167073163197013, 0.9888809426916687 ] }, "capture": { "k": 8, "n": 8, "rate": 1, "ci": [ 0.6755843804891231, 1 ] }, "falseReports": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "offRecord": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "invented": { "k": 0, "n": 11, "rate": 0, "ci": [ 2.7755575615628914e-17, 0.2588400172488141 ] }, "duplicates": { "k": 2, "n": 8, "rate": 0.25, "ci": [ 0.07147768885802763, 0.5907301208974108 ] }, "points": { "mean": 3.4545454545454546, "ci": [ 2.272727272727273, 4.75 ], "reports": 11, "fields": { "summary": { "mean": 1, "n": 10 }, "goal": { "mean": 1, "n": 6 }, "why": { "mean": 1, "n": 6 }, "decided_by": { "mean": 1, "n": 2 }, "metric": { "mean": 0.3333333333333333, "n": 3 }, "direction": { "mean": 0.5, "n": 4 }, "judge_by": { "mean": 1, "n": 1 } } }, "context": { "perRequest": 5080, "perSession": { "p25": 25416, "median": 30584, "p75": 40929 }, "sessions": 16 }, "costPerSession": 0.0037868624999999995, "costPerCapture": 0.007573724999999999 } ], "byKind": [ { "run": "2026-10-31-00-51", "type": "not a decision", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "partial plan", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "made", "phrasing": "blunt", "skill": "none", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-00-51", "type": "deferred", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 0, "n": 5, "rate": 0, "ci": [ 0, 0.43449149475208104 ] } }, { "run": "2026-10-31-00-51", "type": "not a decision", "phrasing": "multi-turn", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "someone else decided", "phrasing": "multi-turn", "skill": "none", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-00-51", "type": "someone else decided", "phrasing": "multi-turn", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "off the record", "phrasing": "multi-turn", "skill": "full", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-00-51", "type": "deferred", "phrasing": "blunt", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 0, "n": 5, "rate": 0, "ci": [ 0, 0.43449149475208104 ] } }, { "run": "2026-10-31-00-51", "type": "reversed", "phrasing": "blunt", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "partial plan", "phrasing": "multi-turn", "skill": "none", "right": { "k": 1, "n": 5, "rate": 0.2, "ci": [ 0.03622316096978745, 0.6244717358814613 ] }, "strict": { "k": 1, "n": 5, "rate": 0.2, "ci": [ 0.03622316096978745, 0.6244717358814613 ] } }, { "run": "2026-10-31-00-51", "type": "made", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "delegated", "phrasing": "blunt", "skill": "full", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-00-51", "type": "someone else decided", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "delegated", "phrasing": "blunt", "skill": "none", "right": { "k": 2, "n": 2, "rate": 1, "ci": [ 0.34237195288961925, 1 ] }, "strict": { "k": 2, "n": 2, "rate": 1, "ci": [ 0.34237195288961925, 1 ] } }, { "run": "2026-10-31-00-51", "type": "someone else decided", "phrasing": "blunt", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "delegated", "phrasing": "multi-turn", "skill": "none", "right": { "k": 2, "n": 3, "rate": 0.6666666666666666, "ci": [ 0.2076549551264879, 0.9385096847238394 ] }, "strict": { "k": 2, "n": 3, "rate": 0.6666666666666666, "ci": [ 0.2076549551264879, 0.9385096847238394 ] } }, { "run": "2026-10-31-00-51", "type": "reversed", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "reversed", "phrasing": "multi-turn", "skill": "none", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-00-51", "type": "deferred", "phrasing": "multi-turn", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "partial plan", "phrasing": "blunt", "skill": "none", "right": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "strict": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] } }, { "run": "2026-10-31-00-51", "type": "not a decision", "phrasing": "blunt", "skill": "none", "right": { "k": 2, "n": 2, "rate": 1, "ci": [ 0.34237195288961925, 1 ] }, "strict": { "k": 2, "n": 2, "rate": 1, "ci": [ 0.34237195288961925, 1 ] } }, { "run": "2026-10-31-00-51", "type": "made", "phrasing": "multi-turn", "skill": "full", "right": { "k": 3, "n": 3, "rate": 1, "ci": [ 0.43849391955098227, 1 ] }, "strict": { "k": 3, "n": 3, "rate": 1, "ci": [ 0.43849391955098227, 1 ] } }, { "run": "2026-10-31-00-51", "type": "off the record", "phrasing": "multi-turn", "skill": "none", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-00-51", "type": "deferred", "phrasing": "multi-turn", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 3, "n": 5, "rate": 0.6, "ci": [ 0.23071993220883708, 0.8823817688407467 ] } }, { "run": "2026-10-31-00-51", "type": "not a decision", "phrasing": "multi-turn", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "reversed", "phrasing": "multi-turn", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-00-51", "type": "partial plan", "phrasing": "multi-turn", "skill": "full", "right": { "k": 4, "n": 5, "rate": 0.8, "ci": [ 0.3755282641185388, 0.9637768390302125 ] }, "strict": { "k": 4, "n": 5, "rate": 0.8, "ci": [ 0.3755282641185388, 0.9637768390302125 ] } }, { "run": "2026-10-31-00-51", "type": "off the record", "phrasing": "blunt", "skill": "full", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-00-51", "type": "off the record", "phrasing": "blunt", "skill": "none", "right": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] }, "strict": { "k": 0, "n": 2, "rate": 0, "ci": [ 0, 0.6576280471103807 ] } }, { "run": "2026-10-31-00-51", "type": "delegated", "phrasing": "multi-turn", "skill": "full", "right": { "k": 2, "n": 4, "rate": 0.5, "ci": [ 0.15003570882017148, 0.8499642911798285 ] }, "strict": { "k": 2, "n": 4, "rate": 0.5, "ci": [ 0.15003570882017148, 0.8499642911798285 ] } }, { "run": "2026-10-31-00-51", "type": "made", "phrasing": "multi-turn", "skill": "none", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-02-16", "type": "made", "phrasing": "blunt", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "someone else decided", "phrasing": "multi-turn", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "reversed", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "reversed", "phrasing": "blunt", "skill": "compact", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "someone else decided", "phrasing": "multi-turn", "skill": "compact", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "partial plan", "phrasing": "multi-turn", "skill": "none", "right": { "k": 0, "n": 5, "rate": 0, "ci": [ 0, 0.43449149475208104 ] }, "strict": { "k": 0, "n": 5, "rate": 0, "ci": [ 0, 0.43449149475208104 ] } }, { "run": "2026-10-31-02-16", "type": "made", "phrasing": "blunt", "skill": "compact", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "partial plan", "phrasing": "multi-turn", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "partial plan", "phrasing": "multi-turn", "skill": "compact", "right": { "k": 4, "n": 5, "rate": 0.8, "ci": [ 0.3755282641185388, 0.9637768390302125 ] }, "strict": { "k": 4, "n": 5, "rate": 0.8, "ci": [ 0.3755282641185388, 0.9637768390302125 ] } }, { "run": "2026-10-31-02-16", "type": "reversed", "phrasing": "multi-turn", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "reversed", "phrasing": "blunt", "skill": "none", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-02-16", "type": "reversed", "phrasing": "multi-turn", "skill": "compact", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-02-16", "type": "partial plan", "phrasing": "blunt", "skill": "full", "right": { "k": 4, "n": 5, "rate": 0.8, "ci": [ 0.3755282641185388, 0.9637768390302125 ] }, "strict": { "k": 4, "n": 5, "rate": 0.8, "ci": [ 0.3755282641185388, 0.9637768390302125 ] } }, { "run": "2026-10-31-02-16", "type": "someone else decided", "phrasing": "blunt", "skill": "none", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "partial plan", "phrasing": "blunt", "skill": "compact", "right": { "k": 0, "n": 5, "rate": 0, "ci": [ 0, 0.43449149475208104 ] }, "strict": { "k": 0, "n": 5, "rate": 0, "ci": [ 0, 0.43449149475208104 ] } }, { "run": "2026-10-31-02-16", "type": "made", "phrasing": "multi-turn", "skill": "full", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-02-16", "type": "reversed", "phrasing": "multi-turn", "skill": "none", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-02-16", "type": "made", "phrasing": "multi-turn", "skill": "compact", "right": { "k": 3, "n": 3, "rate": 1, "ci": [ 0.43849391955098227, 1 ] }, "strict": { "k": 3, "n": 3, "rate": 1, "ci": [ 0.43849391955098227, 1 ] } }, { "run": "2026-10-31-02-16", "type": "partial plan", "phrasing": "blunt", "skill": "none", "right": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] }, "strict": { "k": 0, "n": 4, "rate": 0, "ci": [ 0, 0.48990002040399916 ] } }, { "run": "2026-10-31-02-16", "type": "made", "phrasing": "blunt", "skill": "full", "right": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] }, "strict": { "k": 4, "n": 4, "rate": 1, "ci": [ 0.5100999795960008, 1 ] } }, { "run": "2026-10-31-02-16", "type": "someone else decided", "phrasing": "blunt", "skill": "compact", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "someone else decided", "phrasing": "blunt", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "someone else decided", "phrasing": "multi-turn", "skill": "full", "right": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] }, "strict": { "k": 5, "n": 5, "rate": 1, "ci": [ 0.5655085052479191, 1 ] } }, { "run": "2026-10-31-02-16", "type": "made", "phrasing": "multi-turn", "skill": "none", "right": { "k": 3, "n": 3, "rate": 1, "ci": [ 0.43849391955098227, 1 ] }, "strict": { "k": 3, "n": 3, "rate": 1, "ci": [ 0.43849391955098227, 1 ] } } ], "changes": [ { "date": "2026-10-31", "text": "First run. Five models ran every conversation twice: once with ZeroWidth's tools only, and once with ZeroWidth's tools and instructions.", "models": [ "anthropic/claude-sonnet-5.5", "anthropic/claude-haiku-5.5", "google/gemini-3.8-flash", "deepseek/deepseek-v4.1-flash", "openai/gpt-6-luna" ] }, { "date": "2026-10-31", "text": "Second run, on the eight conversations that hold a decision. ZeroWidth now tells the assistant when a decision is already recorded, so the assistant stops recording the same decision twice. A third setup joined: tools and short instructions, about a fifth the length of the full instructions.", "models": [ "anthropic/claude-sonnet-5.5", "anthropic/claude-haiku-5.5", "google/gemini-3.8-flash", "deepseek/deepseek-v4.1-flash", "openai/gpt-6-luna" ] } ] }