Skip to content

Commit 9376524

Browse files
MichaelYochpazMichaelYochpaz
committed
Add benchmarks framework
Add performance benchmarking infrastructure for fromager including GitHub Actions workflows for nightly and on-demand runs. Signed-off-by: Michael Yochpaz <[email protected]>
1 parent f6e78fb commit 9376524

File tree

15 files changed

+4177
-0
lines changed

15 files changed

+4177
-0
lines changed

‎.github/workflows/benchmark-backfill.yml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
name: Benchmarks (Backfill)
2+
3+
# CodSpeed uses Valgrind to count CPU instructions.
4+
# Valgrind wraps `hatch run` and traces child processes (hatch → uv → python → pytest).
5+
# Do NOT skip hatch/uv in Valgrind args. Skipping a parent process causes Valgrind to lose track of ALL children (no data captured).
6+
# See `benchmarks/README.md` for more information.
7+
8+
on:
9+
workflow_dispatch:
10+
inputs:
11+
from_commit:
12+
description: 'Start commit SHA (older)'
13+
required: true
14+
type: string
15+
to_commit:
16+
description: 'End commit SHA (newer, default: HEAD)'
17+
required: false
18+
default: 'HEAD'
19+
type: string
20+
benchmark_set:
21+
description: 'Benchmark set to run'
22+
required: false
23+
default: 'fast'
24+
type: choice
25+
options:
26+
- fast
27+
- full
28+
force:
29+
description: 'Force recalculation (reserved for future use)'
30+
required: false
31+
default: false
32+
type: boolean
33+
34+
permissions:
35+
contents: read
36+
id-token: write
37+
38+
jobs:
39+
prepare:
40+
runs-on: ubuntu-latest
41+
outputs:
42+
commits: ${{ steps.get-commits.outputs.commits }}
43+
steps:
44+
- uses: actions/checkout@v4
45+
with:
46+
fetch-depth: 0
47+
48+
- name: Get commits between range
49+
id: get-commits
50+
run: |
51+
FROM="${{ inputs.from_commit }}"
52+
TO="${{ inputs.to_commit }}"
53+
54+
# Get list of commits from oldest to newest
55+
COMMITS=$(git rev-list --reverse "$FROM^..$TO" | head -20)
56+
57+
# Convert to JSON array
58+
JSON_COMMITS=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
59+
echo "commits=$JSON_COMMITS" >> $GITHUB_OUTPUT
60+
echo "Will benchmark these commits:"
61+
echo "$COMMITS"
62+
63+
backfill:
64+
needs: prepare
65+
runs-on: ubuntu-latest
66+
timeout-minutes: 60
67+
strategy:
68+
fail-fast: false
69+
max-parallel: 1 # Run sequentially to avoid overwhelming CodSpeed
70+
matrix:
71+
commit: ${{ fromJson(needs.prepare.outputs.commits) }}
72+
73+
steps:
74+
- uses: actions/checkout@v4
75+
with:
76+
ref: ${{ matrix.commit }}
77+
fetch-depth: 0
78+
79+
# Copy benchmarks from the workflow's source branch (to run up-to-date benchmarks on older commits)
80+
- name: Ensure benchmarks directory exists
81+
run: |
82+
if [ ! -d "benchmarks" ]; then
83+
echo "Benchmarks directory not found in commit ${{ matrix.commit }}"
84+
echo "Fetching from workflow source..."
85+
git fetch origin ${{ github.ref }}
86+
git checkout FETCH_HEAD -- benchmarks/ || echo "No benchmarks to copy"
87+
fi
88+
89+
- name: Set up Python
90+
uses: actions/setup-python@v5
91+
with:
92+
python-version: "3.11"
93+
94+
- name: Install hatch
95+
run: pip install hatch
96+
97+
- name: Run benchmarks with CodSpeed
98+
uses: CodSpeedHQ/action@v4
99+
with:
100+
run: |
101+
if [ "${{ inputs.benchmark_set }}" = "fast" ]; then
102+
hatch run benchmark:run --codspeed -m "not slow and not integration"
103+
else
104+
hatch run benchmark:run --codspeed
105+
fi

‎.github/workflows/benchmarks-nightly.yml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
name: Benchmarks (Nightly)
2+
3+
# CodSpeed uses Valgrind to count CPU instructions.
4+
# Valgrind wraps `hatch run` and traces child processes (hatch → uv → python → pytest).
5+
# Do NOT skip hatch/uv in Valgrind args. Skipping a parent process causes Valgrind to lose track of ALL children (no data captured).
6+
# See `benchmarks/README.md` for more information.
7+
8+
on:
9+
schedule:
10+
- cron: "0 2 * * *" # 2 AM UTC daily
11+
workflow_dispatch: # Allow manual trigger
12+
pull_request:
13+
types: [labeled]
14+
15+
permissions:
16+
contents: read
17+
id-token: write
18+
19+
jobs:
20+
check-changes:
21+
runs-on: ubuntu-latest
22+
outputs:
23+
should_run: ${{ steps.check.outputs.should_run }}
24+
steps:
25+
- uses: actions/checkout@v4
26+
with:
27+
fetch-depth: 2
28+
29+
- name: Check if should run
30+
id: check
31+
run: |
32+
# Always run for label triggers and manual dispatch
33+
if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
34+
echo "should_run=true" >> $GITHUB_OUTPUT
35+
exit 0
36+
fi
37+
38+
# For scheduled runs, skip if HEAD hasn't changed in 24 hours
39+
LAST_COMMIT_TIME=$(git log -1 --format=%ct)
40+
NOW=$(date +%s)
41+
HOURS_AGO=$(( (NOW - LAST_COMMIT_TIME) / 3600 ))
42+
43+
if [ "$HOURS_AGO" -gt 24 ]; then
44+
echo "No commits in the last 24 hours, skipping nightly benchmark"
45+
echo "should_run=false" >> $GITHUB_OUTPUT
46+
else
47+
echo "should_run=true" >> $GITHUB_OUTPUT
48+
fi
49+
50+
integration-benchmark:
51+
needs: check-changes
52+
if: |
53+
needs.check-changes.outputs.should_run == 'true' &&
54+
(github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
55+
runs-on: ubuntu-latest
56+
timeout-minutes: 180
57+
58+
steps:
59+
- uses: actions/checkout@v4
60+
with:
61+
fetch-depth: 0 # Required for baseline comparison
62+
63+
- name: Set up Python
64+
uses: actions/setup-python@v5
65+
with:
66+
python-version: "3.11"
67+
68+
- name: Install hatch
69+
run: pip install hatch
70+
71+
- name: Run full benchmarks with CodSpeed
72+
uses: CodSpeedHQ/action@v4
73+
env:
74+
# Skip shell and git in various install locations to reduce noise
75+
CODSPEED_VALGRIND_ARGS: "--trace-children=yes --trace-children-skip=*/sh,*/bash,*/git,/bin/sh,/usr/bin/git"
76+
UV_NO_PROGRESS: "1"
77+
with:
78+
run: hatch run benchmark:run --codspeed
79+
80+
- name: Generate benchmark JSON (fallback)
81+
if: always()
82+
env:
83+
UV_NO_PROGRESS: "1"
84+
run: |
85+
hatch run benchmark:run \
86+
--benchmark-only \
87+
--benchmark-json=benchmark-results-full.json || true
88+
89+
- name: Upload benchmark results
90+
uses: actions/upload-artifact@v4
91+
if: always()
92+
with:
93+
name: benchmark-results-full
94+
path: benchmark-results-full.json
95+
retention-days: 90
96+
97+
- name: Run memory profiling
98+
if: always()
99+
env:
100+
UV_NO_PROGRESS: "1"
101+
run: |
102+
hatch run benchmark:memory \
103+
--memray-bin-path=memray-results || true
104+
105+
- name: Upload memory results
106+
uses: actions/upload-artifact@v4
107+
if: always()
108+
with:
109+
name: memory-results
110+
path: memray-results/
111+
retention-days: 90

‎.github/workflows/benchmarks.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Benchmarks
2+
3+
# CodSpeed uses Valgrind to count CPU instructions.
4+
# Valgrind wraps `hatch run` and traces child processes (hatch → uv → python → pytest).
5+
# Do NOT skip hatch/uv in Valgrind args. Skipping a parent process causes Valgrind to lose track of ALL children (no data captured).
6+
# See `benchmarks/README.md` for more information.
7+
8+
on:
9+
pull_request:
10+
push:
11+
branches: [main]
12+
13+
permissions:
14+
contents: read
15+
id-token: write
16+
17+
jobs:
18+
benchmark-fast:
19+
runs-on: ubuntu-latest
20+
timeout-minutes: 30
21+
22+
steps:
23+
- uses: actions/checkout@v4
24+
with:
25+
fetch-depth: 0 # Required for baseline comparison
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: "3.11"
31+
32+
- name: Install hatch
33+
run: pip install hatch
34+
35+
- name: Run fast benchmarks with CodSpeed
36+
uses: CodSpeedHQ/action@v4
37+
with:
38+
run: hatch run benchmark:run --codspeed -m "not slow and not integration"
39+
40+
- name: Generate benchmark JSON (fallback)
41+
if: always()
42+
run: |
43+
hatch run benchmark:run \
44+
--benchmark-only \
45+
--benchmark-json=benchmark-results.json \
46+
-m "not slow and not integration" || true
47+
48+
- name: Upload benchmark results
49+
uses: actions/upload-artifact@v4
50+
if: always()
51+
with:
52+
name: benchmark-results
53+
path: benchmark-results.json
54+
retention-days: 30

0 commit comments

Comments
 (0)