feat: add authenticated business benchmark suite
This commit is contained in:
1 parent
5293d93a30
commit
c25b180021
9 files changed
+652
-19
No files matched your search
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import statistics
|
||||
|
||||
|
||||
FILE_PATTERN = re.compile(r"^(?P<scenario>.+)-c(?P<concurrency>\d+)-r(?P<repeat>\d+)\.json$")
|
||||
|
||||
|
||||
def median(values):
|
||||
return round(statistics.median(values), 3) if values else None
|
||||
|
||||
|
||||
def merge_counts(reports, key):
|
||||
merged = collections.Counter()
|
||||
for report in reports:
|
||||
merged.update(report.get(key) or {})
|
||||
return dict(sorted(merged.items()))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Aggregate repeated authenticated business benchmark results."
|
||||
)
|
||||
parser.add_argument("directory")
|
||||
parser.add_argument("--markdown-output", required=True)
|
||||
parser.add_argument("--json-output", required=True)
|
||||
parser.add_argument("--require-concurrency-one-success", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
directory = pathlib.Path(args.directory)
|
||||
grouped = collections.defaultdict(list)
|
||||
for path in sorted(directory.glob("*-c*-r*.json")):
|
||||
match = FILE_PATTERN.match(path.name)
|
||||
if not match:
|
||||
continue
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload.get("concurrency"), int):
|
||||
continue
|
||||
grouped[(match.group("scenario"), payload["concurrency"])].append(payload)
|
||||
if not grouped:
|
||||
raise SystemExit(f"No business benchmark result files found in: {directory}")
|
||||
|
||||
rows = []
|
||||
baseline_valid = True
|
||||
for (scenario, concurrency), reports in sorted(grouped.items()):
|
||||
requests = sum(int(report.get("requests") or 0) for report in reports)
|
||||
successes = sum(int(report.get("successful_requests") or 0) for report in reports)
|
||||
failures = sum(int(report.get("failed_requests") or 0) for report in reports)
|
||||
success_percent = round(100 * successes / requests, 3) if requests else 0
|
||||
rps_values = [float(report["requests_per_second"]) for report in reports]
|
||||
p95_values = [float(report["latency_ms"]["p95"]) for report in reports]
|
||||
p99_values = [float(report["latency_ms"]["p99"]) for report in reports]
|
||||
duration_seconds = round(
|
||||
sum(float(report.get("duration_seconds") or 0) for report in reports), 3
|
||||
)
|
||||
row = {
|
||||
"scenario": scenario,
|
||||
"concurrency": concurrency,
|
||||
"repeats": len(reports),
|
||||
"requests": requests,
|
||||
"successful_requests": successes,
|
||||
"failed_requests": failures,
|
||||
"success_percent": success_percent,
|
||||
"duration_seconds_total": duration_seconds,
|
||||
"rps_median": median(rps_values),
|
||||
"rps_min": round(min(rps_values), 3),
|
||||
"rps_max": round(max(rps_values), 3),
|
||||
"run_p95_median_ms": median(p95_values),
|
||||
"run_p99_worst_ms": round(max(p99_values), 3),
|
||||
"status_counts": merge_counts(reports, "status_counts"),
|
||||
"transport_error_counts": merge_counts(
|
||||
reports, "transport_error_counts"
|
||||
),
|
||||
}
|
||||
rows.append(row)
|
||||
if concurrency == 1 and failures:
|
||||
baseline_valid = False
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"baseline_concurrency_one_valid": baseline_valid,
|
||||
"groups": rows,
|
||||
}
|
||||
json_path = pathlib.Path(args.json_output)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
lines = [
|
||||
"| 场景 | 并发 | 重复 | 请求 | 成功率 | RPS 中位数(范围) | 每轮 P95 中位数 ms | 最差一轮 P99 ms | 状态码 | 传输错误 |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |",
|
||||
]
|
||||
for row in rows:
|
||||
status_counts = json.dumps(row["status_counts"], ensure_ascii=False)
|
||||
transport_error_counts = json.dumps(
|
||||
row["transport_error_counts"], ensure_ascii=False
|
||||
)
|
||||
lines.append(
|
||||
"| {scenario} | {concurrency} | {repeats} | {requests} | {success_percent:.3f}% | "
|
||||
"{rps_median} ({rps_min}-{rps_max}) | {run_p95_median_ms} | "
|
||||
"{run_p99_worst_ms} | `{status_counts}` | `{transport_error_counts}` |".format(
|
||||
scenario=row["scenario"],
|
||||
concurrency=row["concurrency"],
|
||||
repeats=row["repeats"],
|
||||
requests=row["requests"],
|
||||
success_percent=row["success_percent"],
|
||||
rps_median=row["rps_median"],
|
||||
rps_min=row["rps_min"],
|
||||
rps_max=row["rps_max"],
|
||||
run_p95_median_ms=row["run_p95_median_ms"],
|
||||
run_p99_worst_ms=row["run_p99_worst_ms"],
|
||||
status_counts=status_counts,
|
||||
transport_error_counts=transport_error_counts,
|
||||
)
|
||||
)
|
||||
markdown_path = pathlib.Path(args.markdown_output)
|
||||
markdown_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
if args.require_concurrency_one_success and not baseline_valid:
|
||||
raise SystemExit("Concurrency-one correctness baseline contained failures.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user