feat: add authenticated business benchmark suite

This commit is contained in:
wlx committed 2026-08-13 14:22:34 +08:00
1 parent 5293d93a30
commit c25b180021
9 files changed
+652 -19

No files matched your search

@@ -0,0 +1,83 @@
#!/usr/bin/env python3
import argparse
import ipaddress
import json
import pathlib
import socket
import urllib.error
import urllib.parse
import urllib.request
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
def validate_base_url(raw_url):
parsed = urllib.parse.urlsplit(raw_url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("Base URL must use http or https and include a host.")
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("Credentials, query strings, and fragments are forbidden.")
if parsed.path not in {"", "/"}:
raise ValueError("Base URL must not include an API path.")
if parsed.hostname == "localhost":
addresses = [ipaddress.ip_address("127.0.0.1")]
else:
addresses = [
ipaddress.ip_address(item[4][0])
for item in socket.getaddrinfo(
parsed.hostname, parsed.port, type=socket.SOCK_STREAM
)
]
if not addresses or any(
not (address.is_loopback or address.is_private or address.is_link_local)
for address in addresses
):
raise ValueError("Refusing to send a bearer token to a non-local host.")
return raw_url.rstrip("/")
def main():
parser = argparse.ArgumentParser(description="Close a local benchmark session.")
parser.add_argument("--base-url", default="http://127.0.0.1:3001")
parser.add_argument("--headers-file", required=True)
parser.add_argument("--timeout", type=float, default=5.0)
args = parser.parse_args()
try:
base_url = validate_base_url(args.base_url)
headers = json.loads(
pathlib.Path(args.headers_file).read_text(encoding="utf-8")
)
if not isinstance(headers, dict) or not isinstance(
headers.get("Authorization"), str
):
raise ValueError("Temporary headers file is invalid.")
opener = urllib.request.build_opener(
urllib.request.ProxyHandler({}), NoRedirectHandler()
)
request = urllib.request.Request(
base_url + "/api/auth/logout",
data=b"{}",
headers=headers,
method="POST",
)
with opener.open(request, timeout=args.timeout) as response:
response.read(1024)
if not 200 <= response.status < 300:
raise RuntimeError(f"Logout returned HTTP {response.status}.")
except (
OSError,
ValueError,
RuntimeError,
json.JSONDecodeError,
socket.gaierror,
urllib.error.URLError,
) as exc:
raise SystemExit(str(exc)) from exc
if __name__ == "__main__":
main()
@@ -0,0 +1,192 @@
#!/usr/bin/env python3
import argparse
import datetime
import ipaddress
import json
import os
import pathlib
import secrets
import socket
import time
import urllib.error
import urllib.parse
import urllib.request
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
def parse_args():
parser = argparse.ArgumentParser(
description="Create one local benchmark user and a temporary bearer header file."
)
parser.add_argument("--base-url", default="http://127.0.0.1:3001")
parser.add_argument("--headers-output", required=True)
parser.add_argument("--metadata-output", required=True)
parser.add_argument("--timeout", type=float, default=10.0)
parser.add_argument("--deployment-commit", default="unknown")
parser.add_argument("--deployment-dirty", default="unknown")
parser.add_argument("--source-commit", default="unknown")
parser.add_argument("--backend-image", default="unknown")
parser.add_argument("--backend-image-id", default="unknown")
parser.add_argument("--backend-container-started-at", default="unknown")
parser.add_argument("--requests-per-level", type=int, default=0)
parser.add_argument("--repeat-count", type=int, default=0)
parser.add_argument("--concurrencies", default="")
return parser.parse_args()
def validate_base_url(raw_url):
parsed = urllib.parse.urlsplit(raw_url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("Base URL must use http or https and include a host.")
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("Credentials, query strings, and fragments are forbidden.")
if parsed.path not in {"", "/"}:
raise ValueError("Base URL must not include an API path.")
if parsed.hostname == "localhost":
addresses = [ipaddress.ip_address("127.0.0.1")]
else:
addresses = [
ipaddress.ip_address(item[4][0])
for item in socket.getaddrinfo(
parsed.hostname, parsed.port, type=socket.SOCK_STREAM
)
]
if not addresses or any(
not (address.is_loopback or address.is_private or address.is_link_local)
for address in addresses
):
raise ValueError("Refusing to create a benchmark session on a non-local host.")
return raw_url.rstrip("/")
def request_json(opener, timeout, method, url, body=None):
headers = {
"Accept": "application/json",
"User-Agent": "ai-soulmate-local-business-benchmark/1",
}
data = None
if body is not None:
data = json.dumps(body, separators=(",", ":")).encode("utf-8")
headers["Content-Type"] = "application/json"
request = urllib.request.Request(url, data=data, headers=headers, method=method)
started = time.perf_counter()
try:
with opener.open(request, timeout=timeout) as response:
raw_body = response.read(MAX_RESPONSE_BYTES + 1)
if len(raw_body) > MAX_RESPONSE_BYTES:
raise RuntimeError("Benchmark setup response exceeded 2 MiB.")
payload = json.loads(raw_body.decode("utf-8"))
if not isinstance(payload, dict):
raise RuntimeError("Benchmark setup returned a non-object JSON response.")
return response.status, payload, round(
(time.perf_counter() - started) * 1000, 3
)
except urllib.error.HTTPError as exc:
exc.close()
raise RuntimeError(f"{method} {url} returned HTTP {exc.code}.") from exc
def write_private_json(path, payload):
output_path = pathlib.Path(path)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
os.chmod(output_path, 0o600)
def main():
args = parse_args()
if args.timeout <= 0 or args.timeout > 60:
raise SystemExit("--timeout must be between 0 and 60 seconds.")
try:
base_url = validate_base_url(args.base_url)
opener = urllib.request.build_opener(
urllib.request.ProxyHandler({}), NoRedirectHandler()
)
health_status, health, health_latency = request_json(
opener, args.timeout, "GET", base_url + "/api/health"
)
mode = str(health.get("tripo_mode") or "").strip().lower()
profile = str(health.get("tripo_mock_profile") or "").strip()
if (
health_status != 200
or mode != "mock"
or not profile
or profile == "not-selected"
):
raise RuntimeError("Backend did not confirm an explicit Tripo mock profile.")
username = f"bench_business_{int(time.time())}_{secrets.token_hex(4)}"
password = "Bm!9" + secrets.token_urlsafe(18)
register_status, registration, register_latency = request_json(
opener,
args.timeout,
"POST",
base_url + "/api/auth/register",
{
"username": username,
"password": password,
"password_confirm": password,
"register_via": "account",
},
)
token = registration.get("access_token")
user = registration.get("user")
if register_status != 200 or not isinstance(token, str) or not token:
raise RuntimeError("Registration did not return a bearer token.")
if not isinstance(user, dict) or not user.get("id"):
raise RuntimeError("Registration did not return a user ID.")
write_private_json(
args.headers_output,
{
"Accept": "application/json",
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
"User-Agent": "ai-soulmate-local-business-benchmark/1",
},
)
write_private_json(
args.metadata_output,
{
"schema_version": 1,
"created_at_utc": datetime.datetime.now(
datetime.timezone.utc
).isoformat(),
"base_url": base_url,
"tripo_mode": mode,
"tripo_mock_profile": profile,
"health_latency_ms": health_latency,
"registration_latency_ms": register_latency,
"user_id": str(user["id"]),
"deployment_commit": args.deployment_commit,
"deployment_dirty": args.deployment_dirty.lower() == "true",
"source_commit": args.source_commit,
"backend_image": args.backend_image,
"backend_image_id": args.backend_image_id,
"backend_container_started_at": args.backend_container_started_at,
"requests_per_level": args.requests_per_level,
"repeat_count": args.repeat_count,
"concurrencies": [
int(value)
for value in args.concurrencies.split(",")
if value.strip()
],
"test_account_persisted": True,
"plaintext_credentials_persisted": False,
},
)
except (OSError, ValueError, RuntimeError, json.JSONDecodeError) as exc:
raise SystemExit(str(exc)) from exc
if __name__ == "__main__":
main()
+33 -16
View File
@@ -31,6 +31,8 @@ def parse_args():
parser.add_argument("--body-file")
parser.add_argument("--headers-file")
parser.add_argument("--output")
parser.add_argument("--allow-failures", action="store_true")
parser.add_argument("--max-response-bytes", type=int, default=64 * 1024)
return parser.parse_args()
@@ -68,6 +70,8 @@ def main():
raise SystemExit("--concurrency must be between 1 and 256")
if args.warmup < 0 or args.timeout <= 0:
raise SystemExit("--warmup must be non-negative and --timeout must be positive")
if not 1 <= args.max_response_bytes <= 64 * 1024 * 1024:
raise SystemExit("--max-response-bytes must be between 1 byte and 64 MiB")
validate_target(args.url)
body = pathlib.Path(args.body_file).read_bytes() if args.body_file else None
@@ -83,20 +87,38 @@ def main():
NoRedirectHandler(),
)
def application_code(response_body):
if not response_body:
return None
try:
parsed_body = json.loads(response_body)
except (UnicodeDecodeError, json.JSONDecodeError):
return None
if isinstance(parsed_body, dict) and parsed_body.get("code") is not None:
return str(parsed_body["code"])
return None
def request_once():
request = urllib.request.Request(args.url, data=body, headers=headers, method=args.method)
started = time.perf_counter()
try:
with opener.open(request, timeout=args.timeout) as response:
response_body = response.read()
response_body = response.read(args.max_response_bytes + 1)
if len(response_body) > args.max_response_bytes:
return response.status, (time.perf_counter() - started) * 1000, "ResponseTooLarge", None, response.headers.get("Retry-After")
retry_after = response.headers.get("Retry-After")
return response.status, (time.perf_counter() - started) * 1000, None, response_body, retry_after
return response.status, (time.perf_counter() - started) * 1000, None, application_code(response_body), retry_after
except urllib.error.HTTPError as exc:
response_body = exc.read()
response_body = exc.read(args.max_response_bytes + 1)
if len(response_body) > args.max_response_bytes:
response_body = b""
error_name = "ResponseTooLarge"
else:
error_name = None
retry_after = exc.headers.get("Retry-After")
return exc.code, (time.perf_counter() - started) * 1000, None, response_body, retry_after
return exc.code, (time.perf_counter() - started) * 1000, error_name, application_code(response_body), retry_after
except Exception as exc:
return None, (time.perf_counter() - started) * 1000, type(exc).__name__, b"", None
return None, (time.perf_counter() - started) * 1000, type(exc).__name__, None, None
for _ in range(args.warmup):
request_once()
@@ -113,23 +135,17 @@ def main():
response_codes = {}
retry_after_values = {}
successful = 0
for status, _, error, response_body, retry_after in results:
for status, _, error, response_code, retry_after in results:
if status is not None:
statuses[str(status)] = statuses.get(str(status), 0) + 1
if status in expected_statuses:
if status in expected_statuses and error is None:
successful += 1
if error:
errors[error] = errors.get(error, 0) + 1
if retry_after:
retry_after_values[retry_after] = retry_after_values.get(retry_after, 0) + 1
if response_body:
try:
parsed_body = json.loads(response_body)
except (UnicodeDecodeError, json.JSONDecodeError):
parsed_body = None
if isinstance(parsed_body, dict) and parsed_body.get("code") is not None:
code = str(parsed_body["code"])
response_codes[code] = response_codes.get(code, 0) + 1
if response_code is not None:
response_codes[response_code] = response_codes.get(response_code, 0) + 1
report = {
"schema_version": 1,
@@ -142,6 +158,7 @@ def main():
"warmup_requests": args.warmup,
"body_bytes": len(body or b""),
"custom_headers_supplied": bool(headers),
"max_response_bytes": args.max_response_bytes,
"duration_seconds": round(duration_seconds, 6),
"requests_per_second": round(args.requests / duration_seconds, 3),
"successful_requests": successful,
@@ -167,7 +184,7 @@ def main():
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(rendered + "\n", encoding="utf-8")
if successful != args.requests:
if not args.allow_failures and successful != args.requests:
raise SystemExit(2)
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
import argparse
import collections
import json
import pathlib
import re
import statistics
FILE_PATTERN = re.compile(r"^(?P<scenario>.+)-c(?P<concurrency>\d+)-r(?P<repeat>\d+)\.json$")
def median(values):
return round(statistics.median(values), 3) if values else None
def merge_counts(reports, key):
merged = collections.Counter()
for report in reports:
merged.update(report.get(key) or {})
return dict(sorted(merged.items()))
def main():
parser = argparse.ArgumentParser(
description="Aggregate repeated authenticated business benchmark results."
)
parser.add_argument("directory")
parser.add_argument("--markdown-output", required=True)
parser.add_argument("--json-output", required=True)
parser.add_argument("--require-concurrency-one-success", action="store_true")
args = parser.parse_args()
directory = pathlib.Path(args.directory)
grouped = collections.defaultdict(list)
for path in sorted(directory.glob("*-c*-r*.json")):
match = FILE_PATTERN.match(path.name)
if not match:
continue
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload.get("concurrency"), int):
continue
grouped[(match.group("scenario"), payload["concurrency"])].append(payload)
if not grouped:
raise SystemExit(f"No business benchmark result files found in: {directory}")
rows = []
baseline_valid = True
for (scenario, concurrency), reports in sorted(grouped.items()):
requests = sum(int(report.get("requests") or 0) for report in reports)
successes = sum(int(report.get("successful_requests") or 0) for report in reports)
failures = sum(int(report.get("failed_requests") or 0) for report in reports)
success_percent = round(100 * successes / requests, 3) if requests else 0
rps_values = [float(report["requests_per_second"]) for report in reports]
p95_values = [float(report["latency_ms"]["p95"]) for report in reports]
p99_values = [float(report["latency_ms"]["p99"]) for report in reports]
duration_seconds = round(
sum(float(report.get("duration_seconds") or 0) for report in reports), 3
)
row = {
"scenario": scenario,
"concurrency": concurrency,
"repeats": len(reports),
"requests": requests,
"successful_requests": successes,
"failed_requests": failures,
"success_percent": success_percent,
"duration_seconds_total": duration_seconds,
"rps_median": median(rps_values),
"rps_min": round(min(rps_values), 3),
"rps_max": round(max(rps_values), 3),
"run_p95_median_ms": median(p95_values),
"run_p99_worst_ms": round(max(p99_values), 3),
"status_counts": merge_counts(reports, "status_counts"),
"transport_error_counts": merge_counts(
reports, "transport_error_counts"
),
}
rows.append(row)
if concurrency == 1 and failures:
baseline_valid = False
payload = {
"schema_version": 1,
"baseline_concurrency_one_valid": baseline_valid,
"groups": rows,
}
json_path = pathlib.Path(args.json_output)
json_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
lines = [
"| 场景 | 并发 | 重复 | 请求 | 成功率 | RPS 中位数(范围) | 每轮 P95 中位数 ms | 最差一轮 P99 ms | 状态码 | 传输错误 |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |",
]
for row in rows:
status_counts = json.dumps(row["status_counts"], ensure_ascii=False)
transport_error_counts = json.dumps(
row["transport_error_counts"], ensure_ascii=False
)
lines.append(
"| {scenario} | {concurrency} | {repeats} | {requests} | {success_percent:.3f}% | "
"{rps_median} ({rps_min}-{rps_max}) | {run_p95_median_ms} | "
"{run_p99_worst_ms} | `{status_counts}` | `{transport_error_counts}` |".format(
scenario=row["scenario"],
concurrency=row["concurrency"],
repeats=row["repeats"],
requests=row["requests"],
success_percent=row["success_percent"],
rps_median=row["rps_median"],
rps_min=row["rps_min"],
rps_max=row["rps_max"],
run_p95_median_ms=row["run_p95_median_ms"],
run_p99_worst_ms=row["run_p99_worst_ms"],
status_counts=status_counts,
transport_error_counts=transport_error_counts,
)
)
markdown_path = pathlib.Path(args.markdown_output)
markdown_path.parent.mkdir(parents=True, exist_ok=True)
markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
if args.require_concurrency_one_success and not baseline_valid:
raise SystemExit("Concurrency-one correctness baseline contained failures.")
if __name__ == "__main__":
main()
+6
View File
@@ -14,6 +14,12 @@ def main():
rows = []
for path in sorted(directory.glob("*.json")):
payload = json.loads(path.read_text(encoding="utf-8"))
if (
payload.get("schema_version") != 1
or not isinstance(payload.get("concurrency"), int)
or not isinstance(payload.get("latency_ms"), dict)
):
continue
latency = payload.get("latency_ms") or {}
rows.append(
(
+18
View File
@@ -0,0 +1,18 @@
{
"nodes": [
{
"id": "start",
"type": "start"
},
{
"id": "end",
"type": "end"
}
],
"edges": [
{
"source": "start",
"target": "end"
}
]
}