Skip to content

Commit bec801a

Browse files
hulxvhulxv
committed
feat: setup benchmarking env
1 parent 7bcffb5 commit bec801a

25 files changed

+1422
-548
lines changed

‎benchmarks/README.md‎

Lines changed: 75 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,79 @@
1-
# Benchmarks
1+
# MetaSSR Benchmarks
22

3-
TODO: Implement docker compose, meanwhile you can use this for building and running:
3+
This directory contains all benchmark-related scripts and configurations for MetaSSR performance testing.
44

5-
Client:
6-
```sh
7-
docker build -t metacall/metassr_benchmarks:client .
8-
```
5+
## Scripts Overview
6+
7+
- `run-benchmarks.sh` - Main automated benchmark runner
8+
- `benchmark.sh` - Core benchmark execution script
9+
- `analyze-benchmarks.py` - Results analysis and reporting
10+
- `generate-pr-summary.py` - Generate PR comment summaries
11+
- `benchmark-config.json` - Test scenarios configuration
12+
- `requirements.txt` - Python dependencies
13+
14+
## Quick Start
15+
16+
```bash
17+
# Run full benchmark suite
18+
./benchmarks/run-benchmarks.sh
19+
20+
# Run with custom options
21+
./benchmarks/run-benchmarks.sh --port 3000 --build debug --graphs
922

10-
Next.js:
11-
```sh
12-
docker build -t nextjs-docker .
13-
docker run -p 3000:3000 nextjs-docker
23+
# Analyze existing results
24+
python3 benchmarks/analyze-benchmarks.py benchmark-results/results.json --plots
1425
```
26+
27+
## Dependencies
28+
29+
### System Requirements
30+
- `wrk` - HTTP benchmarking tool
31+
- `jq` - JSON processor
32+
- `curl` - HTTP client
33+
- `lsof` - List open files (for process monitoring)
34+
35+
### Python Requirements
36+
Install with: `pip install -r benchmarks/requirements.txt`
37+
- pandas - Data analysis
38+
- matplotlib - Plotting
39+
- seaborn - Statistical visualization
40+
- numpy - Numerical computing
41+
42+
## Benchmark Scenarios
43+
44+
Configured in `benchmark-config.json`:
45+
46+
| Scenario | Purpose | Threads | Connections | Duration |
47+
|----------|---------|---------|-------------|----------|
48+
| Light Load | Basic functionality | 1 | 10 | 30s |
49+
| Medium Load | Typical usage | 4 | 50 | 30s |
50+
| Standard Load | Standard testing | 8 | 100 | 30s |
51+
| Heavy Load | Peak performance | 12 | 500 | 30s |
52+
| Extreme Load | Stress testing | 16 | 1000 | 30s |
53+
| Sustained Load | Stability testing | 8 | 200 | 2min |
54+
| Endurance Test | Long-term stability | 4 | 100 | 5min |
55+
56+
## Output Formats
57+
58+
- **JSON** - Structured results for analysis
59+
- **CSV** - Tabular data for spreadsheets
60+
- **Markdown** - Human-readable reports
61+
- **PNG** - Performance charts (with --plots)
62+
63+
## CI/CD Integration
64+
65+
The benchmarks are automatically run via GitHub Actions on:
66+
- Push to master
67+
- Pull requests
68+
- Weekly schedule
69+
- Manual workflow dispatch
70+
71+
Results are posted as PR comments and stored as workflow artifacts.
72+
73+
## Contributing
74+
75+
When modifying benchmarks:
76+
1. Test locally first
77+
2. Update configuration if adding scenarios
78+
3. Ensure scripts remain executable
79+
4. Update documentation accordingly

‎benchmarks/analyze-benchmarks.py‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
#!/usr/bin/env python3
2+
"""
3+
MetaSSR Benchmark Results Analyzer
4+
Analyzes benchmark results and generates comprehensive reports
5+
"""
6+
7+
import json
8+
import csv
9+
import argparse
10+
import os
11+
import sys
12+
from datetime import datetime
13+
from pathlib import Path
14+
import statistics
15+
import matplotlib.pyplot as plt
16+
import pandas as pd
17+
import seaborn as sns
18+
19+
class BenchmarkAnalyzer:
20+
def __init__(self, results_dir="benchmark-results"):
21+
self.results_dir = Path(results_dir)
22+
self.results_dir.mkdir(exist_ok=True)
23+
24+
def load_results(self, result_file):
25+
"""Load benchmark results from JSON file"""
26+
with open(result_file, 'r') as f:
27+
return json.load(f)
28+
29+
def analyze_performance(self, results):
30+
"""Analyze performance metrics"""
31+
analysis = {
32+
'summary': {},
33+
'trends': {},
34+
'recommendations': []
35+
}
36+
37+
tests = results['tests']
38+
39+
# Calculate overall statistics
40+
rps_values = [float(test['results']['requests_per_sec'].replace(',', ''))
41+
for test in tests if test['results']['requests_per_sec']]
42+
43+
analysis['summary'] = {
44+
'total_tests': len(tests),
45+
'max_rps': max(rps_values) if rps_values else 0,
46+
'avg_rps': statistics.mean(rps_values) if rps_values else 0,
47+
'min_rps': min(rps_values) if rps_values else 0
48+
}
49+
50+
# Analyze each test
51+
for test in tests:
52+
test_name = test['name']
53+
results_data = test['results']
54+
55+
# Convert latency to milliseconds
56+
avg_latency = self.parse_latency(results_data['avg_latency'])
57+
p99_latency = self.parse_latency(results_data['latency_percentiles']['p99'])
58+
59+
analysis['trends'][test_name] = {
60+
'rps': float(results_data['requests_per_sec'].replace(',', '') or 0),
61+
'avg_latency_ms': avg_latency,
62+
'p99_latency_ms': p99_latency,
63+
'errors': int(results_data['total_errors'] or 0),
64+
'total_requests': int(results_data['total_requests'].replace(',', '') or 0)
65+
}
66+
67+
# Generate recommendations
68+
analysis['recommendations'] = self.generate_recommendations(analysis)
69+
70+
return analysis
71+
72+
def parse_latency(self, latency_str):
73+
"""Parse latency string and convert to milliseconds"""
74+
if not latency_str:
75+
return 0
76+
77+
latency_str = latency_str.lower()
78+
if 'ms' in latency_str:
79+
return float(latency_str.replace('ms', ''))
80+
elif 'us' in latency_str:
81+
return float(latency_str.replace('us', '')) / 1000
82+
elif 's' in latency_str:
83+
return float(latency_str.replace('s', '')) * 1000
84+
else:
85+
return float(latency_str)
86+
87+
def generate_recommendations(self, analysis):
88+
"""Generate performance recommendations"""
89+
recommendations = []
90+
trends = analysis['trends']
91+
92+
# Check for high latency
93+
high_latency_tests = [name for name, data in trends.items()
94+
if data['avg_latency_ms'] > 100]
95+
if high_latency_tests:
96+
recommendations.append({
97+
'type': 'warning',
98+
'message': f"High average latency detected in: {', '.join(high_latency_tests)}",
99+
'suggestion': "Consider optimizing server response time or reducing load"
100+
})
101+
102+
# Check for errors
103+
error_tests = [name for name, data in trends.items() if data['errors'] > 0]
104+
if error_tests:
105+
recommendations.append({
106+
'type': 'critical',
107+
'message': f"Errors detected in: {', '.join(error_tests)}",
108+
'suggestion': "Investigate error causes and improve error handling"
109+
})
110+
111+
# Check performance scaling
112+
rps_values = [(name, data['rps']) for name, data in trends.items()]
113+
rps_values.sort(key=lambda x: x[1], reverse=True)
114+
115+
if len(rps_values) > 1:
116+
best_test = rps_values[0]
117+
recommendations.append({
118+
'type': 'info',
119+
'message': f"Best performance: {best_test[0]} with {best_test[1]:.0f} RPS",
120+
'suggestion': "Use this configuration as baseline for optimization"
121+
})
122+
123+
return recommendations
124+
125+
def generate_plots(self, analysis, output_dir):
126+
"""Generate performance visualization plots"""
127+
plt.style.use('seaborn-v0_8')
128+
output_dir = Path(output_dir)
129+
output_dir.mkdir(exist_ok=True)
130+
131+
trends = analysis['trends']
132+
test_names = list(trends.keys())
133+
134+
# RPS vs Test scenario
135+
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))
136+
137+
# Requests per second
138+
rps_values = [trends[name]['rps'] for name in test_names]
139+
ax1.bar(test_names, rps_values, color='skyblue')
140+
ax1.set_title('Requests per Second by Test Scenario')
141+
ax1.set_ylabel('Requests/sec')
142+
ax1.tick_params(axis='x', rotation=45)
143+
144+
# Average latency
145+
latency_values = [trends[name]['avg_latency_ms'] for name in test_names]
146+
ax2.bar(test_names, latency_values, color='lightcoral')
147+
ax2.set_title('Average Latency by Test Scenario')
148+
ax2.set_ylabel('Latency (ms)')
149+
ax2.tick_params(axis='x', rotation=45)
150+
151+
# P99 latency
152+
p99_values = [trends[name]['p99_latency_ms'] for name in test_names]
153+
ax3.bar(test_names, p99_values, color='lightgreen')
154+
ax3.set_title('P99 Latency by Test Scenario')
155+
ax3.set_ylabel('P99 Latency (ms)')
156+
ax3.tick_params(axis='x', rotation=45)
157+
158+
# Error count
159+
error_values = [trends[name]['errors'] for name in test_names]
160+
ax4.bar(test_names, error_values, color='orange')
161+
ax4.set_title('Errors by Test Scenario')
162+
ax4.set_ylabel('Error Count')
163+
ax4.tick_params(axis='x', rotation=45)
164+
165+
plt.tight_layout()
166+
plt.savefig(output_dir / 'performance_overview.png', dpi=300, bbox_inches='tight')
167+
plt.close()
168+
169+
# Performance trend line
170+
plt.figure(figsize=(12, 6))
171+
plt.plot(test_names, rps_values, marker='o', linewidth=2, markersize=8)
172+
plt.title('Performance Trend Across Test Scenarios')
173+
plt.xlabel('Test Scenario')
174+
plt.ylabel('Requests per Second')
175+
plt.xticks(rotation=45)
176+
plt.grid(True, alpha=0.3)
177+
plt.tight_layout()
178+
plt.savefig(output_dir / 'performance_trend.png', dpi=300, bbox_inches='tight')
179+
plt.close()
180+
181+
def generate_report(self, results, analysis, output_file):
182+
"""Generate comprehensive markdown report"""
183+
with open(output_file, 'w') as f:
184+
f.write("# MetaSSR Benchmark Report\n\n")
185+
186+
# Metadata
187+
metadata = results['metadata']
188+
f.write("## System Information\n\n")
189+
f.write(f"- **Timestamp:** {metadata['timestamp']}\n")
190+
f.write(f"- **Hostname:** {metadata['hostname']}\n")
191+
f.write(f"- **OS:** {metadata['os']}\n")
192+
f.write(f"- **Architecture:** {metadata['arch']}\n")
193+
f.write(f"- **CPU Cores:** {metadata['cpu_cores']}\n")
194+
f.write(f"- **Memory:** {metadata['memory_gb']} GB\n\n")
195+
196+
# Summary
197+
summary = analysis['summary']
198+
f.write("## Performance Summary\n\n")
199+
f.write(f"- **Total Tests:** {summary['total_tests']}\n")
200+
f.write(f"- **Maximum RPS:** {summary['max_rps']:,.0f}\n")
201+
f.write(f"- **Average RPS:** {summary['avg_rps']:,.0f}\n")
202+
f.write(f"- **Minimum RPS:** {summary['min_rps']:,.0f}\n\n")
203+
204+
# Detailed results
205+
f.write("## Detailed Results\n\n")
206+
f.write("| Test Scenario | RPS | Avg Latency | P99 Latency | Errors | Total Requests |\n")
207+
f.write("|---------------|-----|-------------|-------------|--------|-----------------|\n")
208+
209+
for name, data in analysis['trends'].items():
210+
f.write(f"| {name} | {data['rps']:,.0f} | {data['avg_latency_ms']:.2f}ms | "
211+
f"{data['p99_latency_ms']:.2f}ms | {data['errors']} | {data['total_requests']:,} |\n")
212+
213+
# Recommendations
214+
if analysis['recommendations']:
215+
f.write("\n## Recommendations\n\n")
216+
for rec in analysis['recommendations']:
217+
icon = "🔴" if rec['type'] == 'critical' else "⚠️" if rec['type'] == 'warning' else "ℹ️"
218+
f.write(f"{icon} **{rec['message']}**\n")
219+
f.write(f" {rec['suggestion']}\n\n")
220+
221+
def export_csv(self, analysis, output_file):
222+
"""Export results to CSV format"""
223+
with open(output_file, 'w', newline='') as f:
224+
writer = csv.writer(f)
225+
writer.writerow(['Test', 'RPS', 'Avg_Latency_ms', 'P99_Latency_ms', 'Errors', 'Total_Requests'])
226+
227+
for name, data in analysis['trends'].items():
228+
writer.writerow([
229+
name,
230+
data['rps'],
231+
data['avg_latency_ms'],
232+
data['p99_latency_ms'],
233+
data['errors'],
234+
data['total_requests']
235+
])
236+
237+
def main():
238+
parser = argparse.ArgumentParser(description='Analyze MetaSSR benchmark results')
239+
parser.add_argument('result_file', help='Path to benchmark results JSON file')
240+
parser.add_argument('-o', '--output', default='analysis_report',
241+
help='Output directory for analysis results')
242+
parser.add_argument('--plots', action='store_true',
243+
help='Generate performance plots')
244+
245+
args = parser.parse_args()
246+
247+
if not os.path.exists(args.result_file):
248+
print(f"Error: Result file {args.result_file} not found")
249+
sys.exit(1)
250+
251+
# Create analyzer
252+
analyzer = BenchmarkAnalyzer()
253+
254+
# Load and analyze results
255+
print("Loading benchmark results...")
256+
results = analyzer.load_results(args.result_file)
257+
258+
print("Analyzing performance...")
259+
analysis = analyzer.analyze_performance(results)
260+
261+
# Create output directory
262+
output_dir = Path(args.output)
263+
output_dir.mkdir(exist_ok=True)
264+
265+
# Generate reports
266+
print("Generating reports...")
267+
268+
# Markdown report
269+
report_file = output_dir / 'benchmark_report.md'
270+
analyzer.generate_report(results, analysis, report_file)
271+
print(f"Generated report: {report_file}")
272+
273+
# CSV export
274+
csv_file = output_dir / 'benchmark_results.csv'
275+
analyzer.export_csv(analysis, csv_file)
276+
print(f"Generated CSV: {csv_file}")
277+
278+
# Generate plots if requested
279+
if args.plots:
280+
try:
281+
print("Generating performance plots...")
282+
analyzer.generate_plots(analysis, output_dir)
283+
print(f"Generated plots in: {output_dir}")
284+
except ImportError:
285+
print("Warning: matplotlib/seaborn not available, skipping plots")
286+
287+
# Print summary
288+
print("\n=== Performance Summary ===")
289+
summary = analysis['summary']
290+
print(f"Maximum RPS: {summary['max_rps']:,.0f}")
291+
print(f"Average RPS: {summary['avg_rps']:,.0f}")
292+
print(f"Total Tests: {summary['total_tests']}")
293+
294+
if analysis['recommendations']:
295+
print("\n=== Recommendations ===")
296+
for rec in analysis['recommendations']:
297+
print(f"- {rec['message']}")
298+
299+
if __name__ == '__main__':
300+
main()

0 commit comments

Comments
 (0)