- | Agent | Pass | Tokens |
|---|
+Token Benchmark
+
-
-Guide
-
Run benchmark
+
Run
cd ~/dogbench && git pull origin main && ./dogbench
-
4 agents: hermes, hermes+dotdog, collar, collar+dotdog
Single agent
-
hermes chat -q "What npm package name and version does this project's package.json declare? Answer with just: name version"
+
hermes chat -q "What npm package name and version does this project's package.json declare? Answer: name version"
-
Add tests
-
Drop .json files in tests/ with id, task, ground.
+
Tests
+
Add .json files to tests/. Fields: id, task, ground (comma-separated keywords).
Add tests
fetch('results.json').then(function(r){return r.json()}).then(function(data){
var runs=Array.isArray(data)?data:[data];
var latest=runs[runs.length-1];
- if(!latest.results){document.getElementById('summary').innerHTML='| No v2 data yet |
';return}
- var results=latest.results;
- var agents=Object.keys(results);
+ if(!latest.results){document.getElementById('results').innerHTML='| No data |
';return}
- // Cheapest among PASSING agents
+ var res=latest.results;
+ var agents=Object.keys(res);
+
+ // Find cheapest among all
var cheapest=null;
agents.forEach(function(a){
- var r=results[a];
- if(r.passed>0&&(!cheapest||r.avg_tokensmaxT)maxT=res[a].avg_tokens});
- var sum=document.getElementById('summary');
+ var tbl=document.getElementById('results');
agents.forEach(function(a){
- var r=results[a];
- var pct=r.tests?Math.round(r.passed/r.tests*100):0;
+ var r=res[a];
var isBest=cheapest&&a===cheapest.id;
- var passColor=pct>66?'#4ade80':pct>0?'#f59e0b':'#ef4444';
- var row=sum.insertRow();
- row.innerHTML=''+r.name+(isBest?' ✅':'')+' | '+
- ''+r.passed+'/'+r.tests+''+
- ' | '+
- ''+fmt(r.avg_tokens)+'t | ';
+ var pct=maxT?Math.round(r.avg_tokens/maxT*100):0;
+ var color=isBest?'#4ade80':pct>75?'#ef4444':'#475569';
+ var passed=r.passed>0;
+ var row=tbl.insertRow();
+ row.innerHTML=''+(isBest?'✅ ':'')+r.name+' | '+
+ ''+fmt(r.avg_tokens)+'t | '+
+ '$'+'…'+' | '+
+ ''+(passed?'✓':'✗')+' | ';
+ });
+
+ // Add cost column properly
+ var rows=tbl.querySelectorAll('tr');
+ agents.forEach(function(a,i){
+ var r=res[a];
+ // Estimate cost from model rates
+ var costPerM={input:0.50,output:2.00};
+ var estIn=r.avg_tokens*0.85;
+ var estOut=r.avg_tokens*0.15;
+ var estCost=(estIn/1e6)*costPerM.input+(estOut/1e6)*costPerM.output;
+ rows[i+1].cells[2].innerHTML='$'+estCost.toFixed(6);
});
+
document.getElementById('meta').textContent=
- runs.length+' run(s) · '+(latest.timestamp||'').slice(0,16)+' · '+latest.model;
+ (latest.timestamp||'').slice(0,16)+' · '+latest.model+' · '+(latest.tests||[]).length+' test(s)';
}).catch(function(){
- document.getElementById('summary').innerHTML='No data. Run ./dogbench |
';
+ document.getElementById('results').innerHTML='No data. Run ./dogbench |
';
});
diff --git a/results.json b/results.json
index 39ff04e..4b82ab9 100644
--- a/results.json
+++ b/results.json
@@ -28,5 +28,65 @@
"avg_tokens": 41412
}
}
+ },
+ {
+ "timestamp": "2026-06-19T02:52:02Z",
+ "model": "deepseek-v4-pro",
+ "results": {
+ "hermes": {
+ "name": "hermes",
+ "passed": 0,
+ "tests": 1,
+ "avg_tokens": 1526
+ },
+ "hermes_dag": {
+ "name": "hermes + dotdog",
+ "passed": 1,
+ "tests": 1,
+ "avg_tokens": 16136
+ },
+ "collar": {
+ "name": "collar",
+ "passed": 0,
+ "tests": 1,
+ "avg_tokens": 29817
+ },
+ "collar_dag": {
+ "name": "collar + dotdog",
+ "passed": 1,
+ "tests": 1,
+ "avg_tokens": 22868
+ }
+ }
+ },
+ {
+ "timestamp": "2026-06-19T02:55:14Z",
+ "model": "deepseek-v4-pro",
+ "results": {
+ "hermes": {
+ "name": "hermes",
+ "passed": 0,
+ "tests": 1,
+ "avg_tokens": 1545
+ },
+ "hermes_dag": {
+ "name": "hermes + dotdog",
+ "passed": 1,
+ "tests": 1,
+ "avg_tokens": 16183
+ },
+ "collar": {
+ "name": "collar",
+ "passed": 0,
+ "tests": 1,
+ "avg_tokens": 19754
+ },
+ "collar_dag": {
+ "name": "collar + dotdog",
+ "passed": 1,
+ "tests": 1,
+ "avg_tokens": 14850
+ }
+ }
}
]
\ No newline at end of file