From 181e044ab2b5f08ea53eec2ae7bb5eab286f440f Mon Sep 17 00:00:00 2001 From: Justin Diclemente Date: Thu, 18 Jun 2026 22:55:16 -0400 Subject: [PATCH 1/2] fix: explain pass/fail --- index.html | 2 +- results.json | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/index.html b/index.html index 1f455e1..32fe30f 100644 --- a/index.html +++ b/index.html @@ -25,7 +25,7 @@

🐕 dogbench

-

github.com/specdog/dogbench

+

github.com/specdog/dogbench · pass = answer contains ground truth keywords

Results

diff --git a/results.json b/results.json index 39ff04e..4b82ab9 100644 --- a/results.json +++ b/results.json @@ -28,5 +28,65 @@ "avg_tokens": 41412 } } + }, + { + "timestamp": "2026-06-19T02:52:02Z", + "model": "deepseek-v4-pro", + "results": { + "hermes": { + "name": "hermes", + "passed": 0, + "tests": 1, + "avg_tokens": 1526 + }, + "hermes_dag": { + "name": "hermes + dotdog", + "passed": 1, + "tests": 1, + "avg_tokens": 16136 + }, + "collar": { + "name": "collar", + "passed": 0, + "tests": 1, + "avg_tokens": 29817 + }, + "collar_dag": { + "name": "collar + dotdog", + "passed": 1, + "tests": 1, + "avg_tokens": 22868 + } + } + }, + { + "timestamp": "2026-06-19T02:55:14Z", + "model": "deepseek-v4-pro", + "results": { + "hermes": { + "name": "hermes", + "passed": 0, + "tests": 1, + "avg_tokens": 1545 + }, + "hermes_dag": { + "name": "hermes + dotdog", + "passed": 1, + "tests": 1, + "avg_tokens": 16183 + }, + "collar": { + "name": "collar", + "passed": 0, + "tests": 1, + "avg_tokens": 19754 + }, + "collar_dag": { + "name": "collar + dotdog", + "passed": 1, + "tests": 1, + "avg_tokens": 14850 + } + } } ] \ No newline at end of file From b56a95d382572fa6cec4e8764b3e418dc67f33de Mon Sep 17 00:00:00 2001 From: Justin Diclemente Date: Thu, 18 Jun 2026 22:56:07 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20token=20benchmark=20focus=20?= =?UTF-8?q?=E2=80=94=20cheapest=20highlighted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.html | 88 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/index.html b/index.html index 32fe30f..cc3000f 100644 --- a/index.html +++ b/index.html @@ -3,56 +3,52 @@ -dogbench — .dag benchmark +dogbench — token benchmark

🐕 dogbench

-

github.com/specdog/dogbench · pass = answer contains ground truth keywords

+

github.com/specdog/dogbench

-

Results

-
- +

Token Benchmark

+
AgentPassTokens
+
AgentTokensCostCorrect
-

-

Guide

-

Run benchmark

+

Run

cd ~/dogbench && git pull origin main && ./dogbench
-

4 agents: hermes, hermes+dotdog, collar, collar+dotdog

Single agent

-
hermes chat -q "What npm package name and version does this project's package.json declare? Answer with just: name version"
+
hermes chat -q "What npm package name and version does this project's package.json declare? Answer: name version"
-

Add tests

-

Drop .json files in tests/ with id, task, ground.

+

Tests

+

Add .json files to tests/. Fields: id, task, ground (comma-separated keywords).

@@ -62,33 +58,49 @@

Add tests

fetch('results.json').then(function(r){return r.json()}).then(function(data){ var runs=Array.isArray(data)?data:[data]; var latest=runs[runs.length-1]; - if(!latest.results){document.getElementById('summary').innerHTML='No v2 data yet';return} - var results=latest.results; - var agents=Object.keys(results); + if(!latest.results){document.getElementById('results').innerHTML='No data';return} - // Cheapest among PASSING agents + var res=latest.results; + var agents=Object.keys(res); + + // Find cheapest among all var cheapest=null; agents.forEach(function(a){ - var r=results[a]; - if(r.passed>0&&(!cheapest||r.avg_tokensmaxT)maxT=res[a].avg_tokens}); - var sum=document.getElementById('summary'); + var tbl=document.getElementById('results'); agents.forEach(function(a){ - var r=results[a]; - var pct=r.tests?Math.round(r.passed/r.tests*100):0; + var r=res[a]; var isBest=cheapest&&a===cheapest.id; - var passColor=pct>66?'#4ade80':pct>0?'#f59e0b':'#ef4444'; - var row=sum.insertRow(); - row.innerHTML=''+r.name+(isBest?' ✅':'')+''+ - ''+r.passed+'/'+r.tests+''+ - '
'+ - ''+fmt(r.avg_tokens)+'t'; + var pct=maxT?Math.round(r.avg_tokens/maxT*100):0; + var color=isBest?'#4ade80':pct>75?'#ef4444':'#475569'; + var passed=r.passed>0; + var row=tbl.insertRow(); + row.innerHTML=''+(isBest?'✅ ':'')+r.name+''+ + ''+fmt(r.avg_tokens)+'t
'+ + '$'+'…'+''+ + ''+(passed?'✓':'✗')+''; + }); + + // Add cost column properly + var rows=tbl.querySelectorAll('tr'); + agents.forEach(function(a,i){ + var r=res[a]; + // Estimate cost from model rates + var costPerM={input:0.50,output:2.00}; + var estIn=r.avg_tokens*0.85; + var estOut=r.avg_tokens*0.15; + var estCost=(estIn/1e6)*costPerM.input+(estOut/1e6)*costPerM.output; + rows[i+1].cells[2].innerHTML='$'+estCost.toFixed(6); }); + document.getElementById('meta').textContent= - runs.length+' run(s) · '+(latest.timestamp||'').slice(0,16)+' · '+latest.model; + (latest.timestamp||'').slice(0,16)+' · '+latest.model+' · '+(latest.tests||[]).length+' test(s)'; }).catch(function(){ - document.getElementById('summary').innerHTML='No data. Run ./dogbench'; + document.getElementById('results').innerHTML='No data. Run ./dogbench'; });