Files
ai-soulmate-local-deployment/benchmark/summarize-business-results.py
T

131 lines
5.2 KiB
Python

#!/usr/bin/env python3
import argparse
import collections
import json
import pathlib
import re
import statistics
FILE_PATTERN = re.compile(r"^(?P<scenario>.+)-c(?P<concurrency>\d+)-r(?P<repeat>\d+)\.json$")
def median(values):
return round(statistics.median(values), 3) if values else None
def merge_counts(reports, key):
merged = collections.Counter()
for report in reports:
merged.update(report.get(key) or {})
return dict(sorted(merged.items()))
def main():
parser = argparse.ArgumentParser(
description="Aggregate repeated authenticated business benchmark results."
)
parser.add_argument("directory")
parser.add_argument("--markdown-output", required=True)
parser.add_argument("--json-output", required=True)
parser.add_argument("--require-concurrency-one-success", action="store_true")
args = parser.parse_args()
directory = pathlib.Path(args.directory)
grouped = collections.defaultdict(list)
for path in sorted(directory.glob("*-c*-r*.json")):
match = FILE_PATTERN.match(path.name)
if not match:
continue
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload.get("concurrency"), int):
continue
grouped[(match.group("scenario"), payload["concurrency"])].append(payload)
if not grouped:
raise SystemExit(f"No business benchmark result files found in: {directory}")
rows = []
baseline_valid = True
for (scenario, concurrency), reports in sorted(grouped.items()):
requests = sum(int(report.get("requests") or 0) for report in reports)
successes = sum(int(report.get("successful_requests") or 0) for report in reports)
failures = sum(int(report.get("failed_requests") or 0) for report in reports)
success_percent = round(100 * successes / requests, 3) if requests else 0
rps_values = [float(report["requests_per_second"]) for report in reports]
p95_values = [float(report["latency_ms"]["p95"]) for report in reports]
p99_values = [float(report["latency_ms"]["p99"]) for report in reports]
duration_seconds = round(
sum(float(report.get("duration_seconds") or 0) for report in reports), 3
)
row = {
"scenario": scenario,
"concurrency": concurrency,
"repeats": len(reports),
"requests": requests,
"successful_requests": successes,
"failed_requests": failures,
"success_percent": success_percent,
"duration_seconds_total": duration_seconds,
"rps_median": median(rps_values),
"rps_min": round(min(rps_values), 3),
"rps_max": round(max(rps_values), 3),
"run_p95_median_ms": median(p95_values),
"run_p99_worst_ms": round(max(p99_values), 3),
"status_counts": merge_counts(reports, "status_counts"),
"transport_error_counts": merge_counts(
reports, "transport_error_counts"
),
}
rows.append(row)
if concurrency == 1 and failures:
baseline_valid = False
payload = {
"schema_version": 1,
"baseline_concurrency_one_valid": baseline_valid,
"groups": rows,
}
json_path = pathlib.Path(args.json_output)
json_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
lines = [
"| 场景 | 并发 | 重复 | 请求 | 成功率 | RPS 中位数(范围) | 每轮 P95 中位数 ms | 最差一轮 P99 ms | 状态码 | 传输错误 |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |",
]
for row in rows:
status_counts = json.dumps(row["status_counts"], ensure_ascii=False)
transport_error_counts = json.dumps(
row["transport_error_counts"], ensure_ascii=False
)
lines.append(
"| {scenario} | {concurrency} | {repeats} | {requests} | {success_percent:.3f}% | "
"{rps_median} ({rps_min}-{rps_max}) | {run_p95_median_ms} | "
"{run_p99_worst_ms} | `{status_counts}` | `{transport_error_counts}` |".format(
scenario=row["scenario"],
concurrency=row["concurrency"],
repeats=row["repeats"],
requests=row["requests"],
success_percent=row["success_percent"],
rps_median=row["rps_median"],
rps_min=row["rps_min"],
rps_max=row["rps_max"],
run_p95_median_ms=row["run_p95_median_ms"],
run_p99_worst_ms=row["run_p99_worst_ms"],
status_counts=status_counts,
transport_error_counts=transport_error_counts,
)
)
markdown_path = pathlib.Path(args.markdown_output)
markdown_path.parent.mkdir(parents=True, exist_ok=True)
markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
if args.require_concurrency_one_success and not baseline_valid:
raise SystemExit("Concurrency-one correctness baseline contained failures.")
if __name__ == "__main__":
main()