Skip to content

Commit 1c23729

Browse files
committed
Add PTC-Bench Interactive Dashboard for Benchmark Visualization
Changes: - Introduced a new Streamlit-based dashboard in `dashboard.py` for visualizing benchmark results comparing Programmatic Tool Calling (PTC) and Function Calling (FC). - Enhanced README with instructions on running the dashboard and updated content to reflect the new features. - Added new assets for visual comparisons, including performance metrics and charts. - Implemented functionality to load benchmark results from JSON files or run benchmarks directly from the dashboard. This commit significantly improves user experience by providing an interactive way to analyze and visualize benchmark data, facilitating better insights into the performance of PTC vs FC approaches.
1 parent 1a0b2fe commit 1c23729

64 files changed

Lines changed: 17329 additions & 4044 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
name: PTC-Bench Benchmark
2+
3+
on:
4+
push:
5+
branches: [ main, master ]
6+
pull_request:
7+
branches: [ main, master ]
8+
schedule:
9+
# Run benchmark daily at 2 AM UTC
10+
- cron: '0 2 * * *'
11+
workflow_dispatch:
12+
inputs:
13+
backend:
14+
description: 'Backend to test'
15+
required: true
16+
default: 'subprocess'
17+
type: choice
18+
options:
19+
- subprocess
20+
- opensandbox
21+
approach:
22+
description: 'Approach to test'
23+
required: true
24+
default: 'both'
25+
type: choice
26+
options:
27+
- ptc
28+
- function_calling
29+
- both
30+
profile:
31+
description: 'Benchmark profile'
32+
required: true
33+
default: 'quick'
34+
type: choice
35+
options:
36+
- quick
37+
- standard
38+
- full
39+
40+
jobs:
41+
benchmark:
42+
runs-on: ubuntu-latest
43+
44+
steps:
45+
- name: Checkout repository
46+
uses: actions/checkout@v3
47+
48+
- name: Set up Python
49+
uses: actions/setup-python@v4
50+
with:
51+
python-version: '3.10'
52+
53+
- name: Install dependencies
54+
run: |
55+
python -m pip install --upgrade pip
56+
pip install -e .
57+
pip install opensandbox opensandbox-server
58+
59+
- name: Configure OpenSandbox (if selected)
60+
if: github.event.inputs.backend == 'opensandbox' || (github.event_name != 'workflow_dispatch' && github.event.inputs.backend != 'subprocess')
61+
run: |
62+
opensandbox-server init-config ~/.sandbox.toml --example docker
63+
64+
- name: Run PTC-Bench Benchmark
65+
id: benchmark
66+
env:
67+
# Use GitHub Secrets for API keys (optional)
68+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
69+
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
70+
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
71+
run: |
72+
# Determine parameters
73+
BACKEND="${{ github.event.inputs.backend || 'subprocess' }}"
74+
APPROACH="${{ github.event.inputs.approach || 'both' }}"
75+
PROFILE="${{ github.event.inputs.profile || 'quick' }}"
76+
77+
# Map profile to categories
78+
case $PROFILE in
79+
quick)
80+
CATEGORIES="compute,ptc"
81+
;;
82+
standard)
83+
CATEGORIES="compute,ptc,io,import_heavy"
84+
;;
85+
full)
86+
CATEGORIES=""
87+
;;
88+
esac
89+
90+
# Determine LLM provider
91+
if [ -n "$OPENAI_API_KEY" ] || [ -n "$AZURE_OPENAI_API_KEY" ]; then
92+
LLM_PROVIDER="openai"
93+
echo "🔑 Using LLM mode with API key"
94+
else
95+
LLM_PROVIDER="none"
96+
echo "ℹ️ Running in baseline mode (no LLM)"
97+
fi
98+
99+
# Build command
100+
CMD="python -m benchmarks run --backend $BACKEND --approach $APPROACH --llm-provider $LLM_PROVIDER --output results.md"
101+
102+
if [ -n "$CATEGORIES" ]; then
103+
CMD="$CMD --categories $CATEGORIES"
104+
fi
105+
106+
echo "🚀 Running: $CMD"
107+
eval $CMD
108+
109+
# Extract summary metrics from results
110+
echo "📊 Benchmark Results:"
111+
cat results.md | head -100
112+
113+
- name: Upload benchmark results
114+
uses: actions/upload-artifact@v3
115+
with:
116+
name: benchmark-results
117+
path: |
118+
results.md
119+
results.json
120+
retention-days: 30
121+
122+
- name: Comment PR with results
123+
if: github.event_name == 'pull_request'
124+
uses: actions/github-script@v6
125+
with:
126+
script: |
127+
const fs = require('fs');
128+
129+
// Read results file
130+
let resultsText = '';
131+
try {
132+
resultsText = fs.readFileSync('results.md', 'utf8');
133+
} catch (e) {
134+
resultsText = 'Results file not found';
135+
}
136+
137+
// Extract key metrics (first 50 lines)
138+
const summary = resultsText.split('\n').slice(0, 50).join('\n');
139+
140+
// Create comment body
141+
const body = `## 📊 PTC-Bench Benchmark Results
142+
143+
${summary}
144+
145+
<details>
146+
<summary>View full results</summary>
147+
148+
\`\`\`
149+
${resultsText.slice(0, 3000)}${resultsText.length > 3000 ? '\n... (truncated)' : ''}
150+
\`\`\`
151+
152+
</details>
153+
154+
---
155+
156+
*Benchmark run on commit ${{ github.sha }}*
157+
`;
158+
159+
github.rest.issues.createComment({
160+
issue_number: context.issue.number,
161+
owner: context.repo.owner,
162+
repo: context.repo.repo,
163+
body: body
164+
});
165+
166+
- name: Check performance regression
167+
if: github.event_name == 'pull_request'
168+
run: |
169+
echo "⚠️ Performance regression check would run here"
170+
echo "Compare results against baseline from main branch"
171+
echo "If regression > 10%, fail the check"
172+
173+
# TODO: Implement actual regression check
174+
# 1. Download baseline results from main branch artifact
175+
# 2. Parse JSON results
176+
# 3. Compare key metrics (success_rate, avg_time)
177+
# 4. Fail if regression exceeds threshold

0 commit comments

Comments
 (0)