From c25b180021c6a0b44c7b9b5afc1bafb00a082855 Mon Sep 17 00:00:00 2001 From: WangLX <519407855@qq.com> Date: Thu, 13 Aug 2026 14:22:34 +0800 Subject: [PATCH] feat: add authenticated business benchmark suite --- benchmark/close-local-benchmark-session.py | 83 ++++++++ benchmark/prepare-local-benchmark-session.py | 192 +++++++++++++++++++ benchmark/run-http-benchmark.py | 49 +++-- benchmark/summarize-business-results.py | 130 +++++++++++++ benchmark/summarize-http-results.py | 6 + benchmark/workflow-validate-body.json | 18 ++ docs/TRIPO-ZERO-COST-BENCHMARK.md | 14 ++ scripts/run-business-benchmark-suite.sh | 168 ++++++++++++++++ scripts/sample-benchmark-resources.sh | 11 +- 9 files changed, 652 insertions(+), 19 deletions(-) create mode 100644 benchmark/close-local-benchmark-session.py create mode 100644 benchmark/prepare-local-benchmark-session.py create mode 100644 benchmark/summarize-business-results.py create mode 100644 benchmark/workflow-validate-body.json create mode 100755 scripts/run-business-benchmark-suite.sh diff --git a/benchmark/close-local-benchmark-session.py b/benchmark/close-local-benchmark-session.py new file mode 100644 index 0000000..4f3b130 --- /dev/null +++ b/benchmark/close-local-benchmark-session.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +import argparse +import ipaddress +import json +import pathlib +import socket +import urllib.error +import urllib.parse +import urllib.request + + +class NoRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def validate_base_url(raw_url): + parsed = urllib.parse.urlsplit(raw_url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("Base URL must use http or https and include a host.") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("Credentials, query strings, and fragments are forbidden.") + if parsed.path not in {"", "/"}: + raise ValueError("Base URL must not include an API path.") + if parsed.hostname == "localhost": + addresses = [ipaddress.ip_address("127.0.0.1")] + else: + addresses = [ + ipaddress.ip_address(item[4][0]) + for item in socket.getaddrinfo( + parsed.hostname, parsed.port, type=socket.SOCK_STREAM + ) + ] + if not addresses or any( + not (address.is_loopback or address.is_private or address.is_link_local) + for address in addresses + ): + raise ValueError("Refusing to send a bearer token to a non-local host.") + return raw_url.rstrip("/") + + +def main(): + parser = argparse.ArgumentParser(description="Close a local benchmark session.") + parser.add_argument("--base-url", default="http://127.0.0.1:3001") + parser.add_argument("--headers-file", required=True) + parser.add_argument("--timeout", type=float, default=5.0) + args = parser.parse_args() + + try: + base_url = validate_base_url(args.base_url) + headers = json.loads( + pathlib.Path(args.headers_file).read_text(encoding="utf-8") + ) + if not isinstance(headers, dict) or not isinstance( + headers.get("Authorization"), str + ): + raise ValueError("Temporary headers file is invalid.") + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), NoRedirectHandler() + ) + request = urllib.request.Request( + base_url + "/api/auth/logout", + data=b"{}", + headers=headers, + method="POST", + ) + with opener.open(request, timeout=args.timeout) as response: + response.read(1024) + if not 200 <= response.status < 300: + raise RuntimeError(f"Logout returned HTTP {response.status}.") + except ( + OSError, + ValueError, + RuntimeError, + json.JSONDecodeError, + socket.gaierror, + urllib.error.URLError, + ) as exc: + raise SystemExit(str(exc)) from exc + + +if __name__ == "__main__": + main() diff --git a/benchmark/prepare-local-benchmark-session.py b/benchmark/prepare-local-benchmark-session.py new file mode 100644 index 0000000..d55f208 --- /dev/null +++ b/benchmark/prepare-local-benchmark-session.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +import argparse +import datetime +import ipaddress +import json +import os +import pathlib +import secrets +import socket +import time +import urllib.error +import urllib.parse +import urllib.request + + +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + + +class NoRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Create one local benchmark user and a temporary bearer header file." + ) + parser.add_argument("--base-url", default="http://127.0.0.1:3001") + parser.add_argument("--headers-output", required=True) + parser.add_argument("--metadata-output", required=True) + parser.add_argument("--timeout", type=float, default=10.0) + parser.add_argument("--deployment-commit", default="unknown") + parser.add_argument("--deployment-dirty", default="unknown") + parser.add_argument("--source-commit", default="unknown") + parser.add_argument("--backend-image", default="unknown") + parser.add_argument("--backend-image-id", default="unknown") + parser.add_argument("--backend-container-started-at", default="unknown") + parser.add_argument("--requests-per-level", type=int, default=0) + parser.add_argument("--repeat-count", type=int, default=0) + parser.add_argument("--concurrencies", default="") + return parser.parse_args() + + +def validate_base_url(raw_url): + parsed = urllib.parse.urlsplit(raw_url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("Base URL must use http or https and include a host.") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("Credentials, query strings, and fragments are forbidden.") + if parsed.path not in {"", "/"}: + raise ValueError("Base URL must not include an API path.") + if parsed.hostname == "localhost": + addresses = [ipaddress.ip_address("127.0.0.1")] + else: + addresses = [ + ipaddress.ip_address(item[4][0]) + for item in socket.getaddrinfo( + parsed.hostname, parsed.port, type=socket.SOCK_STREAM + ) + ] + if not addresses or any( + not (address.is_loopback or address.is_private or address.is_link_local) + for address in addresses + ): + raise ValueError("Refusing to create a benchmark session on a non-local host.") + return raw_url.rstrip("/") + + +def request_json(opener, timeout, method, url, body=None): + headers = { + "Accept": "application/json", + "User-Agent": "ai-soulmate-local-business-benchmark/1", + } + data = None + if body is not None: + data = json.dumps(body, separators=(",", ":")).encode("utf-8") + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + started = time.perf_counter() + try: + with opener.open(request, timeout=timeout) as response: + raw_body = response.read(MAX_RESPONSE_BYTES + 1) + if len(raw_body) > MAX_RESPONSE_BYTES: + raise RuntimeError("Benchmark setup response exceeded 2 MiB.") + payload = json.loads(raw_body.decode("utf-8")) + if not isinstance(payload, dict): + raise RuntimeError("Benchmark setup returned a non-object JSON response.") + return response.status, payload, round( + (time.perf_counter() - started) * 1000, 3 + ) + except urllib.error.HTTPError as exc: + exc.close() + raise RuntimeError(f"{method} {url} returned HTTP {exc.code}.") from exc + + +def write_private_json(path, payload): + output_path = pathlib.Path(path) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + os.chmod(output_path, 0o600) + + +def main(): + args = parse_args() + if args.timeout <= 0 or args.timeout > 60: + raise SystemExit("--timeout must be between 0 and 60 seconds.") + try: + base_url = validate_base_url(args.base_url) + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), NoRedirectHandler() + ) + health_status, health, health_latency = request_json( + opener, args.timeout, "GET", base_url + "/api/health" + ) + mode = str(health.get("tripo_mode") or "").strip().lower() + profile = str(health.get("tripo_mock_profile") or "").strip() + if ( + health_status != 200 + or mode != "mock" + or not profile + or profile == "not-selected" + ): + raise RuntimeError("Backend did not confirm an explicit Tripo mock profile.") + + username = f"bench_business_{int(time.time())}_{secrets.token_hex(4)}" + password = "Bm!9" + secrets.token_urlsafe(18) + register_status, registration, register_latency = request_json( + opener, + args.timeout, + "POST", + base_url + "/api/auth/register", + { + "username": username, + "password": password, + "password_confirm": password, + "register_via": "account", + }, + ) + token = registration.get("access_token") + user = registration.get("user") + if register_status != 200 or not isinstance(token, str) or not token: + raise RuntimeError("Registration did not return a bearer token.") + if not isinstance(user, dict) or not user.get("id"): + raise RuntimeError("Registration did not return a user ID.") + + write_private_json( + args.headers_output, + { + "Accept": "application/json", + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "User-Agent": "ai-soulmate-local-business-benchmark/1", + }, + ) + write_private_json( + args.metadata_output, + { + "schema_version": 1, + "created_at_utc": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(), + "base_url": base_url, + "tripo_mode": mode, + "tripo_mock_profile": profile, + "health_latency_ms": health_latency, + "registration_latency_ms": register_latency, + "user_id": str(user["id"]), + "deployment_commit": args.deployment_commit, + "deployment_dirty": args.deployment_dirty.lower() == "true", + "source_commit": args.source_commit, + "backend_image": args.backend_image, + "backend_image_id": args.backend_image_id, + "backend_container_started_at": args.backend_container_started_at, + "requests_per_level": args.requests_per_level, + "repeat_count": args.repeat_count, + "concurrencies": [ + int(value) + for value in args.concurrencies.split(",") + if value.strip() + ], + "test_account_persisted": True, + "plaintext_credentials_persisted": False, + }, + ) + except (OSError, ValueError, RuntimeError, json.JSONDecodeError) as exc: + raise SystemExit(str(exc)) from exc + + +if __name__ == "__main__": + main() diff --git a/benchmark/run-http-benchmark.py b/benchmark/run-http-benchmark.py index 7b5117f..8653872 100644 --- a/benchmark/run-http-benchmark.py +++ b/benchmark/run-http-benchmark.py @@ -31,6 +31,8 @@ def parse_args(): parser.add_argument("--body-file") parser.add_argument("--headers-file") parser.add_argument("--output") + parser.add_argument("--allow-failures", action="store_true") + parser.add_argument("--max-response-bytes", type=int, default=64 * 1024) return parser.parse_args() @@ -68,6 +70,8 @@ def main(): raise SystemExit("--concurrency must be between 1 and 256") if args.warmup < 0 or args.timeout <= 0: raise SystemExit("--warmup must be non-negative and --timeout must be positive") + if not 1 <= args.max_response_bytes <= 64 * 1024 * 1024: + raise SystemExit("--max-response-bytes must be between 1 byte and 64 MiB") validate_target(args.url) body = pathlib.Path(args.body_file).read_bytes() if args.body_file else None @@ -83,20 +87,38 @@ def main(): NoRedirectHandler(), ) + def application_code(response_body): + if not response_body: + return None + try: + parsed_body = json.loads(response_body) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + if isinstance(parsed_body, dict) and parsed_body.get("code") is not None: + return str(parsed_body["code"]) + return None + def request_once(): request = urllib.request.Request(args.url, data=body, headers=headers, method=args.method) started = time.perf_counter() try: with opener.open(request, timeout=args.timeout) as response: - response_body = response.read() + response_body = response.read(args.max_response_bytes + 1) + if len(response_body) > args.max_response_bytes: + return response.status, (time.perf_counter() - started) * 1000, "ResponseTooLarge", None, response.headers.get("Retry-After") retry_after = response.headers.get("Retry-After") - return response.status, (time.perf_counter() - started) * 1000, None, response_body, retry_after + return response.status, (time.perf_counter() - started) * 1000, None, application_code(response_body), retry_after except urllib.error.HTTPError as exc: - response_body = exc.read() + response_body = exc.read(args.max_response_bytes + 1) + if len(response_body) > args.max_response_bytes: + response_body = b"" + error_name = "ResponseTooLarge" + else: + error_name = None retry_after = exc.headers.get("Retry-After") - return exc.code, (time.perf_counter() - started) * 1000, None, response_body, retry_after + return exc.code, (time.perf_counter() - started) * 1000, error_name, application_code(response_body), retry_after except Exception as exc: - return None, (time.perf_counter() - started) * 1000, type(exc).__name__, b"", None + return None, (time.perf_counter() - started) * 1000, type(exc).__name__, None, None for _ in range(args.warmup): request_once() @@ -113,23 +135,17 @@ def main(): response_codes = {} retry_after_values = {} successful = 0 - for status, _, error, response_body, retry_after in results: + for status, _, error, response_code, retry_after in results: if status is not None: statuses[str(status)] = statuses.get(str(status), 0) + 1 - if status in expected_statuses: + if status in expected_statuses and error is None: successful += 1 if error: errors[error] = errors.get(error, 0) + 1 if retry_after: retry_after_values[retry_after] = retry_after_values.get(retry_after, 0) + 1 - if response_body: - try: - parsed_body = json.loads(response_body) - except (UnicodeDecodeError, json.JSONDecodeError): - parsed_body = None - if isinstance(parsed_body, dict) and parsed_body.get("code") is not None: - code = str(parsed_body["code"]) - response_codes[code] = response_codes.get(code, 0) + 1 + if response_code is not None: + response_codes[response_code] = response_codes.get(response_code, 0) + 1 report = { "schema_version": 1, @@ -142,6 +158,7 @@ def main(): "warmup_requests": args.warmup, "body_bytes": len(body or b""), "custom_headers_supplied": bool(headers), + "max_response_bytes": args.max_response_bytes, "duration_seconds": round(duration_seconds, 6), "requests_per_second": round(args.requests / duration_seconds, 3), "successful_requests": successful, @@ -167,7 +184,7 @@ def main(): output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(rendered + "\n", encoding="utf-8") - if successful != args.requests: + if not args.allow_failures and successful != args.requests: raise SystemExit(2) diff --git a/benchmark/summarize-business-results.py b/benchmark/summarize-business-results.py new file mode 100644 index 0000000..cf2bc1e --- /dev/null +++ b/benchmark/summarize-business-results.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +import argparse +import collections +import json +import pathlib +import re +import statistics + + +FILE_PATTERN = re.compile(r"^(?P.+)-c(?P\d+)-r(?P\d+)\.json$") + + +def median(values): + return round(statistics.median(values), 3) if values else None + + +def merge_counts(reports, key): + merged = collections.Counter() + for report in reports: + merged.update(report.get(key) or {}) + return dict(sorted(merged.items())) + + +def main(): + parser = argparse.ArgumentParser( + description="Aggregate repeated authenticated business benchmark results." + ) + parser.add_argument("directory") + parser.add_argument("--markdown-output", required=True) + parser.add_argument("--json-output", required=True) + parser.add_argument("--require-concurrency-one-success", action="store_true") + args = parser.parse_args() + + directory = pathlib.Path(args.directory) + grouped = collections.defaultdict(list) + for path in sorted(directory.glob("*-c*-r*.json")): + match = FILE_PATTERN.match(path.name) + if not match: + continue + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload.get("concurrency"), int): + continue + grouped[(match.group("scenario"), payload["concurrency"])].append(payload) + if not grouped: + raise SystemExit(f"No business benchmark result files found in: {directory}") + + rows = [] + baseline_valid = True + for (scenario, concurrency), reports in sorted(grouped.items()): + requests = sum(int(report.get("requests") or 0) for report in reports) + successes = sum(int(report.get("successful_requests") or 0) for report in reports) + failures = sum(int(report.get("failed_requests") or 0) for report in reports) + success_percent = round(100 * successes / requests, 3) if requests else 0 + rps_values = [float(report["requests_per_second"]) for report in reports] + p95_values = [float(report["latency_ms"]["p95"]) for report in reports] + p99_values = [float(report["latency_ms"]["p99"]) for report in reports] + duration_seconds = round( + sum(float(report.get("duration_seconds") or 0) for report in reports), 3 + ) + row = { + "scenario": scenario, + "concurrency": concurrency, + "repeats": len(reports), + "requests": requests, + "successful_requests": successes, + "failed_requests": failures, + "success_percent": success_percent, + "duration_seconds_total": duration_seconds, + "rps_median": median(rps_values), + "rps_min": round(min(rps_values), 3), + "rps_max": round(max(rps_values), 3), + "run_p95_median_ms": median(p95_values), + "run_p99_worst_ms": round(max(p99_values), 3), + "status_counts": merge_counts(reports, "status_counts"), + "transport_error_counts": merge_counts( + reports, "transport_error_counts" + ), + } + rows.append(row) + if concurrency == 1 and failures: + baseline_valid = False + + payload = { + "schema_version": 1, + "baseline_concurrency_one_valid": baseline_valid, + "groups": rows, + } + json_path = pathlib.Path(args.json_output) + json_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + + lines = [ + "| 场景 | 并发 | 重复 | 请求 | 成功率 | RPS 中位数(范围) | 每轮 P95 中位数 ms | 最差一轮 P99 ms | 状态码 | 传输错误 |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", + ] + for row in rows: + status_counts = json.dumps(row["status_counts"], ensure_ascii=False) + transport_error_counts = json.dumps( + row["transport_error_counts"], ensure_ascii=False + ) + lines.append( + "| {scenario} | {concurrency} | {repeats} | {requests} | {success_percent:.3f}% | " + "{rps_median} ({rps_min}-{rps_max}) | {run_p95_median_ms} | " + "{run_p99_worst_ms} | `{status_counts}` | `{transport_error_counts}` |".format( + scenario=row["scenario"], + concurrency=row["concurrency"], + repeats=row["repeats"], + requests=row["requests"], + success_percent=row["success_percent"], + rps_median=row["rps_median"], + rps_min=row["rps_min"], + rps_max=row["rps_max"], + run_p95_median_ms=row["run_p95_median_ms"], + run_p99_worst_ms=row["run_p99_worst_ms"], + status_counts=status_counts, + transport_error_counts=transport_error_counts, + ) + ) + markdown_path = pathlib.Path(args.markdown_output) + markdown_path.parent.mkdir(parents=True, exist_ok=True) + markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + if args.require_concurrency_one_success and not baseline_valid: + raise SystemExit("Concurrency-one correctness baseline contained failures.") + + +if __name__ == "__main__": + main() diff --git a/benchmark/summarize-http-results.py b/benchmark/summarize-http-results.py index 3af6aaf..1e22dcb 100644 --- a/benchmark/summarize-http-results.py +++ b/benchmark/summarize-http-results.py @@ -14,6 +14,12 @@ def main(): rows = [] for path in sorted(directory.glob("*.json")): payload = json.loads(path.read_text(encoding="utf-8")) + if ( + payload.get("schema_version") != 1 + or not isinstance(payload.get("concurrency"), int) + or not isinstance(payload.get("latency_ms"), dict) + ): + continue latency = payload.get("latency_ms") or {} rows.append( ( diff --git a/benchmark/workflow-validate-body.json b/benchmark/workflow-validate-body.json new file mode 100644 index 0000000..fe25e12 --- /dev/null +++ b/benchmark/workflow-validate-body.json @@ -0,0 +1,18 @@ +{ + "nodes": [ + { + "id": "start", + "type": "start" + }, + { + "id": "end", + "type": "end" + } + ], + "edges": [ + { + "source": "start", + "target": "end" + } + ] +} diff --git a/docs/TRIPO-ZERO-COST-BENCHMARK.md b/docs/TRIPO-ZERO-COST-BENCHMARK.md index e7a9519..78cd128 100644 --- a/docs/TRIPO-ZERO-COST-BENCHMARK.md +++ b/docs/TRIPO-ZERO-COST-BENCHMARK.md @@ -152,6 +152,20 @@ Tripo 的“请求速率限制”和“运行任务并发限制”也不是同 ## 7. 必须采集的指标 +在将健康检查结果解释为业务容量前,先运行已认证本地业务接口基线: + +```bash +BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL=500 \ +BUSINESS_BENCHMARK_REPEATS=3 \ +./scripts/run-business-benchmark-suite.sh \ + compose/.env \ + business-baseline-$(date +%Y%m%d-%H%M%S) +``` + +该套件只注册一个随机 `bench_business_` 测试账号,复用 Bearer Token 测试工作流校验、用户资料、项目列表、地址列表和设备列表。请求会经过 JWT 和 PostgreSQL;套件先验证 Redis 健康,Redis 可用时认证链会更新同一用户的滑动会话键。它不会调用 Tripo、对象存储、阿里云或支付。测试账号会留在数据库中供管理端核验;临时明文密码与 Token 不写入结果,Header 文件只放在 `/dev/shm` 并在脚本退出时删除,退出时还会尝试注销 Redis 会话,但已签发 JWT 不具备服务端强制撤销机制。 + +所有并发请求复用一个账号,直连 `127.0.0.1` 后端,列表数据量很小。因此结果只代表“单账号热点型、已认证后端 API 短时基线”,不代表多用户分布、前端/Nginx/Tailscale 链路或真实大数据量。资源采样的名义间隔为 1 秒,实际间隔以 CSV 时间戳为准;phase 是采样启动时的近似标签,短场景可能没有独立样本,不能据此精确归因单档峰值。短时阶梯测试只能称为业务 API 基线;稳定容量仍需长时间稳态和独立重复验证。 + | 层级 | 指标 | | --- | --- | | 请求 | 请求总数、成功数、吞吐量、错误率、超时率、HTTP 状态码分布、P50/P90/P95/P99/最大延迟 | diff --git a/scripts/run-business-benchmark-suite.sh b/scripts/run-business-benchmark-suite.sh new file mode 100755 index 0000000..6a8590d --- /dev/null +++ b/scripts/run-business-benchmark-suite.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +env_file="${1:-${repo_root}/compose/.env}" +run_id="${2:-business-baseline-$(date +%Y%m%d-%H%M%S)}" +request_count="${BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL:-500}" +repeat_count="${BUSINESS_BENCHMARK_REPEATS:-3}" +output_root="${repo_root}/results/raw/${run_id}" +temporary_root="" + +if [[ ! -f "$env_file" ]]; then + echo "Environment file not found: $env_file" >&2 + exit 1 +fi +if [[ ! "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}$ || "$run_id" == *..* ]]; then + echo "Run ID must be 1-120 safe filename characters and must not contain '..'." >&2 + exit 1 +fi +if ! [[ "$request_count" =~ ^[1-9][0-9]*$ ]] || (( request_count > 100000 )); then + echo "BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL must be between 1 and 100000." >&2 + exit 1 +fi +if ! [[ "$repeat_count" =~ ^[1-9][0-9]*$ ]] || (( repeat_count > 10 )); then + echo "BUSINESS_BENCHMARK_REPEATS must be between 1 and 10." >&2 + exit 1 +fi + +mkdir -p "$output_root" +runtime_root="/dev/shm" +if [[ ! -d "$runtime_root" || ! -w "$runtime_root" || "$(stat -f -c %T "$runtime_root")" != "tmpfs" ]]; then + echo "Writable tmpfs runtime directory is unavailable: $runtime_root" >&2 + exit 1 +fi +temporary_root="$(mktemp -d "${runtime_root}/ai-soulmate-business-benchmark.XXXXXX")" +headers_file="${temporary_root}/headers.json" +phase_file="${temporary_root}/phase.txt" +sampler_stop="${temporary_root}/sampler-stop" + +cleanup() { + if [[ -n "${sampler_pid:-}" ]]; then + touch "$sampler_stop" + wait "$sampler_pid" 2>/dev/null || true + fi + if [[ -r "$headers_file" ]]; then + python3 "${repo_root}/benchmark/close-local-benchmark-session.py" \ + --base-url http://127.0.0.1:3001 \ + --headers-file "$headers_file" >/dev/null 2>&1 || true + fi + rm -f "$headers_file" "$phase_file" "$sampler_stop" + rmdir "$temporary_root" 2>/dev/null || true +} +trap cleanup EXIT + +declare -a concurrencies=(1 5 10 20 40 80) +deployment_commit="$(git -C "$repo_root" rev-parse HEAD)" +if [[ -n "$(git -C "$repo_root" status --short --untracked-files=no)" ]]; then + deployment_dirty=true +else + deployment_dirty=false +fi +project_name="$(sed -n 's/^BACKEND_COMPOSE_PROJECT_NAME=//p' "$env_file" | head -n 1)" +project_name="${project_name:-ai-soulmate-backend}" +backend_container_id="$(docker compose --project-name "$project_name" --env-file "$env_file" -f "${repo_root}/compose/docker-compose.backend.yml" ps -q backend)" +if [[ -z "$backend_container_id" ]]; then + echo "Running backend container was not found." >&2 + exit 1 +fi +source_commit="$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$backend_container_id")" +backend_image="$(sed -n 's/^BACKEND_IMAGE=//p' "$env_file" | head -n 1)" +backend_image_id="$(docker inspect --format '{{.Image}}' "$backend_container_id")" +backend_container_started_at="$(docker inspect --format '{{.State.StartedAt}}' "$backend_container_id")" + +bash "${repo_root}/scripts/verify-tripo-mock.sh" "$env_file" +bash "${repo_root}/scripts/verify-infra.sh" "$env_file" >/dev/null +python3 "${repo_root}/benchmark/prepare-local-benchmark-session.py" \ + --base-url http://127.0.0.1:3001 \ + --headers-output "$headers_file" \ + --metadata-output "${output_root}/session-metadata.json" \ + --deployment-commit "$deployment_commit" \ + --deployment-dirty "$deployment_dirty" \ + --source-commit "$source_commit" \ + --backend-image "$backend_image" \ + --backend-image-id "$backend_image_id" \ + --backend-container-started-at "$backend_container_started_at" \ + --requests-per-level "$request_count" \ + --repeat-count "$repeat_count" \ + --concurrencies "$(IFS=,; echo "${concurrencies[*]}")" +chmod 600 "$headers_file" + +printf '%s\n' setup > "$phase_file" +bash "${repo_root}/scripts/sample-benchmark-resources.sh" \ + "$sampler_stop" 1 "$phase_file" \ + > "${output_root}/resources-during-load.csv" & +sampler_pid=$! + +bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \ + > "${output_root}/snapshot-before.txt" + +declare -a names=(workflow-validate user-profile projects-list address-list device-list) +declare -a methods=(POST GET GET GET GET) +declare -a paths=( + /api/workflow/validate + /api/user/profile + /api/projects + /api/address + /api/device +) +for repeat in $(seq 1 "$repeat_count"); do + for index in "${!names[@]}"; do + name="${names[$index]}" + method="${methods[$index]}" + url="http://127.0.0.1:3001${paths[$index]}" + for concurrency in "${concurrencies[@]}"; do + phase="${name}-c${concurrency}-r${repeat}" + printf '%s\n' "$phase" > "$phase_file" + command=( + python3 "${repo_root}/benchmark/run-http-benchmark.py" + --url "$url" + --method "$method" + --requests "$request_count" + --concurrency "$concurrency" + --warmup 5 + --timeout 10 + --expected-status 200 + --headers-file "$headers_file" + --allow-failures + --output "${output_root}/${phase}.json" + ) + if [[ "$method" == POST ]]; then + command+=(--body-file "${repo_root}/benchmark/workflow-validate-body.json") + fi + "${command[@]}" > /dev/null + echo "Completed ${phase}." + sleep 1 + done + done +done + +printf '%s\n' snapshot-after > "$phase_file" +bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \ + > "${output_root}/snapshot-after.txt" + +touch "$sampler_stop" +if ! wait "$sampler_pid"; then + echo "Resource sampler failed during the benchmark." >&2 + exit 1 +fi +unset sampler_pid + +expected_results="$(( ${#names[@]} * ${#concurrencies[@]} * repeat_count ))" +actual_results="$(find "$output_root" -maxdepth 1 -type f -name '*-c*-r*.json' | wc -l)" +if [[ "$actual_results" -ne "$expected_results" ]]; then + echo "Expected ${expected_results} benchmark result files, found ${actual_results}." >&2 + exit 1 +fi +if ! awk -F, 'NR > 1 && $2 != "" { found = 1 } END { exit(found ? 0 : 1) }' \ + "${output_root}/resources-during-load.csv"; then + echo "Resource sampler did not capture any phase-tagged rows." >&2 + exit 1 +fi + +python3 "${repo_root}/benchmark/summarize-business-results.py" "$output_root" \ + --markdown-output "${output_root}/summary.md" \ + --json-output "${output_root}/summary.json" \ + --require-concurrency-one-success + +echo "Authenticated business benchmark completed: $output_root" diff --git a/scripts/sample-benchmark-resources.sh b/scripts/sample-benchmark-resources.sh index 2a5c28d..c1bc94f 100755 --- a/scripts/sample-benchmark-resources.sh +++ b/scripts/sample-benchmark-resources.sh @@ -3,19 +3,24 @@ set -Eeuo pipefail stop_file="${1:?stop file is required}" interval_seconds="${2:-1}" +phase_file="${3:-}" if ! [[ "$interval_seconds" =~ ^[1-9][0-9]*$ ]]; then echo "Sampling interval must be a positive integer." >&2 exit 1 fi -printf '%s\n' 'captured_at,load_1m,mem_available_kib,container,cpu_percent,memory_usage,network_io,block_io,pids' +printf '%s\n' 'captured_at,phase,load_1m,mem_available_kib,container,cpu_percent,memory_usage,network_io,block_io,pids' while [[ ! -e "$stop_file" ]]; do captured_at="$(date --iso-8601=seconds)" + phase="" + if [[ -n "$phase_file" && -r "$phase_file" ]]; then + phase="$(tr -cd 'A-Za-z0-9_.-' < "$phase_file")" + phase="${phase:0:120}" + fi load_1m="$(awk '{print $1}' /proc/loadavg)" mem_available_kib="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)" docker stats --no-stream \ - --format "${captured_at},${load_1m},${mem_available_kib},{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" \ - || true + --format "${captured_at},${phase},${load_1m},${mem_available_kib},{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" sleep "$interval_seconds" done