feat: add authenticated business benchmark suite
This commit is contained in:
1 parent
5293d93a30
commit
c25b180021
9 files changed
+652
-19
No files matched your search
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import ipaddress
|
||||
import json
|
||||
import pathlib
|
||||
import socket
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
def validate_base_url(raw_url):
|
||||
parsed = urllib.parse.urlsplit(raw_url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("Base URL must use http or https and include a host.")
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("Credentials, query strings, and fragments are forbidden.")
|
||||
if parsed.path not in {"", "/"}:
|
||||
raise ValueError("Base URL must not include an API path.")
|
||||
if parsed.hostname == "localhost":
|
||||
addresses = [ipaddress.ip_address("127.0.0.1")]
|
||||
else:
|
||||
addresses = [
|
||||
ipaddress.ip_address(item[4][0])
|
||||
for item in socket.getaddrinfo(
|
||||
parsed.hostname, parsed.port, type=socket.SOCK_STREAM
|
||||
)
|
||||
]
|
||||
if not addresses or any(
|
||||
not (address.is_loopback or address.is_private or address.is_link_local)
|
||||
for address in addresses
|
||||
):
|
||||
raise ValueError("Refusing to send a bearer token to a non-local host.")
|
||||
return raw_url.rstrip("/")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Close a local benchmark session.")
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:3001")
|
||||
parser.add_argument("--headers-file", required=True)
|
||||
parser.add_argument("--timeout", type=float, default=5.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
base_url = validate_base_url(args.base_url)
|
||||
headers = json.loads(
|
||||
pathlib.Path(args.headers_file).read_text(encoding="utf-8")
|
||||
)
|
||||
if not isinstance(headers, dict) or not isinstance(
|
||||
headers.get("Authorization"), str
|
||||
):
|
||||
raise ValueError("Temporary headers file is invalid.")
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({}), NoRedirectHandler()
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
base_url + "/api/auth/logout",
|
||||
data=b"{}",
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
with opener.open(request, timeout=args.timeout) as response:
|
||||
response.read(1024)
|
||||
if not 200 <= response.status < 300:
|
||||
raise RuntimeError(f"Logout returned HTTP {response.status}.")
|
||||
except (
|
||||
OSError,
|
||||
ValueError,
|
||||
RuntimeError,
|
||||
json.JSONDecodeError,
|
||||
socket.gaierror,
|
||||
urllib.error.URLError,
|
||||
) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import secrets
|
||||
import socket
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
||||
|
||||
|
||||
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create one local benchmark user and a temporary bearer header file."
|
||||
)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:3001")
|
||||
parser.add_argument("--headers-output", required=True)
|
||||
parser.add_argument("--metadata-output", required=True)
|
||||
parser.add_argument("--timeout", type=float, default=10.0)
|
||||
parser.add_argument("--deployment-commit", default="unknown")
|
||||
parser.add_argument("--deployment-dirty", default="unknown")
|
||||
parser.add_argument("--source-commit", default="unknown")
|
||||
parser.add_argument("--backend-image", default="unknown")
|
||||
parser.add_argument("--backend-image-id", default="unknown")
|
||||
parser.add_argument("--backend-container-started-at", default="unknown")
|
||||
parser.add_argument("--requests-per-level", type=int, default=0)
|
||||
parser.add_argument("--repeat-count", type=int, default=0)
|
||||
parser.add_argument("--concurrencies", default="")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_base_url(raw_url):
|
||||
parsed = urllib.parse.urlsplit(raw_url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("Base URL must use http or https and include a host.")
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("Credentials, query strings, and fragments are forbidden.")
|
||||
if parsed.path not in {"", "/"}:
|
||||
raise ValueError("Base URL must not include an API path.")
|
||||
if parsed.hostname == "localhost":
|
||||
addresses = [ipaddress.ip_address("127.0.0.1")]
|
||||
else:
|
||||
addresses = [
|
||||
ipaddress.ip_address(item[4][0])
|
||||
for item in socket.getaddrinfo(
|
||||
parsed.hostname, parsed.port, type=socket.SOCK_STREAM
|
||||
)
|
||||
]
|
||||
if not addresses or any(
|
||||
not (address.is_loopback or address.is_private or address.is_link_local)
|
||||
for address in addresses
|
||||
):
|
||||
raise ValueError("Refusing to create a benchmark session on a non-local host.")
|
||||
return raw_url.rstrip("/")
|
||||
|
||||
|
||||
def request_json(opener, timeout, method, url, body=None):
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "ai-soulmate-local-business-benchmark/1",
|
||||
}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body, separators=(",", ":")).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with opener.open(request, timeout=timeout) as response:
|
||||
raw_body = response.read(MAX_RESPONSE_BYTES + 1)
|
||||
if len(raw_body) > MAX_RESPONSE_BYTES:
|
||||
raise RuntimeError("Benchmark setup response exceeded 2 MiB.")
|
||||
payload = json.loads(raw_body.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("Benchmark setup returned a non-object JSON response.")
|
||||
return response.status, payload, round(
|
||||
(time.perf_counter() - started) * 1000, 3
|
||||
)
|
||||
except urllib.error.HTTPError as exc:
|
||||
exc.close()
|
||||
raise RuntimeError(f"{method} {url} returned HTTP {exc.code}.") from exc
|
||||
|
||||
|
||||
def write_private_json(path, payload):
|
||||
output_path = pathlib.Path(path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
os.chmod(output_path, 0o600)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if args.timeout <= 0 or args.timeout > 60:
|
||||
raise SystemExit("--timeout must be between 0 and 60 seconds.")
|
||||
try:
|
||||
base_url = validate_base_url(args.base_url)
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({}), NoRedirectHandler()
|
||||
)
|
||||
health_status, health, health_latency = request_json(
|
||||
opener, args.timeout, "GET", base_url + "/api/health"
|
||||
)
|
||||
mode = str(health.get("tripo_mode") or "").strip().lower()
|
||||
profile = str(health.get("tripo_mock_profile") or "").strip()
|
||||
if (
|
||||
health_status != 200
|
||||
or mode != "mock"
|
||||
or not profile
|
||||
or profile == "not-selected"
|
||||
):
|
||||
raise RuntimeError("Backend did not confirm an explicit Tripo mock profile.")
|
||||
|
||||
username = f"bench_business_{int(time.time())}_{secrets.token_hex(4)}"
|
||||
password = "Bm!9" + secrets.token_urlsafe(18)
|
||||
register_status, registration, register_latency = request_json(
|
||||
opener,
|
||||
args.timeout,
|
||||
"POST",
|
||||
base_url + "/api/auth/register",
|
||||
{
|
||||
"username": username,
|
||||
"password": password,
|
||||
"password_confirm": password,
|
||||
"register_via": "account",
|
||||
},
|
||||
)
|
||||
token = registration.get("access_token")
|
||||
user = registration.get("user")
|
||||
if register_status != 200 or not isinstance(token, str) or not token:
|
||||
raise RuntimeError("Registration did not return a bearer token.")
|
||||
if not isinstance(user, dict) or not user.get("id"):
|
||||
raise RuntimeError("Registration did not return a user ID.")
|
||||
|
||||
write_private_json(
|
||||
args.headers_output,
|
||||
{
|
||||
"Accept": "application/json",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "ai-soulmate-local-business-benchmark/1",
|
||||
},
|
||||
)
|
||||
write_private_json(
|
||||
args.metadata_output,
|
||||
{
|
||||
"schema_version": 1,
|
||||
"created_at_utc": datetime.datetime.now(
|
||||
datetime.timezone.utc
|
||||
).isoformat(),
|
||||
"base_url": base_url,
|
||||
"tripo_mode": mode,
|
||||
"tripo_mock_profile": profile,
|
||||
"health_latency_ms": health_latency,
|
||||
"registration_latency_ms": register_latency,
|
||||
"user_id": str(user["id"]),
|
||||
"deployment_commit": args.deployment_commit,
|
||||
"deployment_dirty": args.deployment_dirty.lower() == "true",
|
||||
"source_commit": args.source_commit,
|
||||
"backend_image": args.backend_image,
|
||||
"backend_image_id": args.backend_image_id,
|
||||
"backend_container_started_at": args.backend_container_started_at,
|
||||
"requests_per_level": args.requests_per_level,
|
||||
"repeat_count": args.repeat_count,
|
||||
"concurrencies": [
|
||||
int(value)
|
||||
for value in args.concurrencies.split(",")
|
||||
if value.strip()
|
||||
],
|
||||
"test_account_persisted": True,
|
||||
"plaintext_credentials_persisted": False,
|
||||
},
|
||||
)
|
||||
except (OSError, ValueError, RuntimeError, json.JSONDecodeError) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -31,6 +31,8 @@ def parse_args():
|
||||
parser.add_argument("--body-file")
|
||||
parser.add_argument("--headers-file")
|
||||
parser.add_argument("--output")
|
||||
parser.add_argument("--allow-failures", action="store_true")
|
||||
parser.add_argument("--max-response-bytes", type=int, default=64 * 1024)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -68,6 +70,8 @@ def main():
|
||||
raise SystemExit("--concurrency must be between 1 and 256")
|
||||
if args.warmup < 0 or args.timeout <= 0:
|
||||
raise SystemExit("--warmup must be non-negative and --timeout must be positive")
|
||||
if not 1 <= args.max_response_bytes <= 64 * 1024 * 1024:
|
||||
raise SystemExit("--max-response-bytes must be between 1 byte and 64 MiB")
|
||||
|
||||
validate_target(args.url)
|
||||
body = pathlib.Path(args.body_file).read_bytes() if args.body_file else None
|
||||
@@ -83,20 +87,38 @@ def main():
|
||||
NoRedirectHandler(),
|
||||
)
|
||||
|
||||
def application_code(response_body):
|
||||
if not response_body:
|
||||
return None
|
||||
try:
|
||||
parsed_body = json.loads(response_body)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return None
|
||||
if isinstance(parsed_body, dict) and parsed_body.get("code") is not None:
|
||||
return str(parsed_body["code"])
|
||||
return None
|
||||
|
||||
def request_once():
|
||||
request = urllib.request.Request(args.url, data=body, headers=headers, method=args.method)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with opener.open(request, timeout=args.timeout) as response:
|
||||
response_body = response.read()
|
||||
response_body = response.read(args.max_response_bytes + 1)
|
||||
if len(response_body) > args.max_response_bytes:
|
||||
return response.status, (time.perf_counter() - started) * 1000, "ResponseTooLarge", None, response.headers.get("Retry-After")
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
return response.status, (time.perf_counter() - started) * 1000, None, response_body, retry_after
|
||||
return response.status, (time.perf_counter() - started) * 1000, None, application_code(response_body), retry_after
|
||||
except urllib.error.HTTPError as exc:
|
||||
response_body = exc.read()
|
||||
response_body = exc.read(args.max_response_bytes + 1)
|
||||
if len(response_body) > args.max_response_bytes:
|
||||
response_body = b""
|
||||
error_name = "ResponseTooLarge"
|
||||
else:
|
||||
error_name = None
|
||||
retry_after = exc.headers.get("Retry-After")
|
||||
return exc.code, (time.perf_counter() - started) * 1000, None, response_body, retry_after
|
||||
return exc.code, (time.perf_counter() - started) * 1000, error_name, application_code(response_body), retry_after
|
||||
except Exception as exc:
|
||||
return None, (time.perf_counter() - started) * 1000, type(exc).__name__, b"", None
|
||||
return None, (time.perf_counter() - started) * 1000, type(exc).__name__, None, None
|
||||
|
||||
for _ in range(args.warmup):
|
||||
request_once()
|
||||
@@ -113,23 +135,17 @@ def main():
|
||||
response_codes = {}
|
||||
retry_after_values = {}
|
||||
successful = 0
|
||||
for status, _, error, response_body, retry_after in results:
|
||||
for status, _, error, response_code, retry_after in results:
|
||||
if status is not None:
|
||||
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
||||
if status in expected_statuses:
|
||||
if status in expected_statuses and error is None:
|
||||
successful += 1
|
||||
if error:
|
||||
errors[error] = errors.get(error, 0) + 1
|
||||
if retry_after:
|
||||
retry_after_values[retry_after] = retry_after_values.get(retry_after, 0) + 1
|
||||
if response_body:
|
||||
try:
|
||||
parsed_body = json.loads(response_body)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
parsed_body = None
|
||||
if isinstance(parsed_body, dict) and parsed_body.get("code") is not None:
|
||||
code = str(parsed_body["code"])
|
||||
response_codes[code] = response_codes.get(code, 0) + 1
|
||||
if response_code is not None:
|
||||
response_codes[response_code] = response_codes.get(response_code, 0) + 1
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
@@ -142,6 +158,7 @@ def main():
|
||||
"warmup_requests": args.warmup,
|
||||
"body_bytes": len(body or b""),
|
||||
"custom_headers_supplied": bool(headers),
|
||||
"max_response_bytes": args.max_response_bytes,
|
||||
"duration_seconds": round(duration_seconds, 6),
|
||||
"requests_per_second": round(args.requests / duration_seconds, 3),
|
||||
"successful_requests": successful,
|
||||
@@ -167,7 +184,7 @@ def main():
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered + "\n", encoding="utf-8")
|
||||
|
||||
if successful != args.requests:
|
||||
if not args.allow_failures and successful != args.requests:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import statistics
|
||||
|
||||
|
||||
FILE_PATTERN = re.compile(r"^(?P<scenario>.+)-c(?P<concurrency>\d+)-r(?P<repeat>\d+)\.json$")
|
||||
|
||||
|
||||
def median(values):
|
||||
return round(statistics.median(values), 3) if values else None
|
||||
|
||||
|
||||
def merge_counts(reports, key):
|
||||
merged = collections.Counter()
|
||||
for report in reports:
|
||||
merged.update(report.get(key) or {})
|
||||
return dict(sorted(merged.items()))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Aggregate repeated authenticated business benchmark results."
|
||||
)
|
||||
parser.add_argument("directory")
|
||||
parser.add_argument("--markdown-output", required=True)
|
||||
parser.add_argument("--json-output", required=True)
|
||||
parser.add_argument("--require-concurrency-one-success", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
directory = pathlib.Path(args.directory)
|
||||
grouped = collections.defaultdict(list)
|
||||
for path in sorted(directory.glob("*-c*-r*.json")):
|
||||
match = FILE_PATTERN.match(path.name)
|
||||
if not match:
|
||||
continue
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload.get("concurrency"), int):
|
||||
continue
|
||||
grouped[(match.group("scenario"), payload["concurrency"])].append(payload)
|
||||
if not grouped:
|
||||
raise SystemExit(f"No business benchmark result files found in: {directory}")
|
||||
|
||||
rows = []
|
||||
baseline_valid = True
|
||||
for (scenario, concurrency), reports in sorted(grouped.items()):
|
||||
requests = sum(int(report.get("requests") or 0) for report in reports)
|
||||
successes = sum(int(report.get("successful_requests") or 0) for report in reports)
|
||||
failures = sum(int(report.get("failed_requests") or 0) for report in reports)
|
||||
success_percent = round(100 * successes / requests, 3) if requests else 0
|
||||
rps_values = [float(report["requests_per_second"]) for report in reports]
|
||||
p95_values = [float(report["latency_ms"]["p95"]) for report in reports]
|
||||
p99_values = [float(report["latency_ms"]["p99"]) for report in reports]
|
||||
duration_seconds = round(
|
||||
sum(float(report.get("duration_seconds") or 0) for report in reports), 3
|
||||
)
|
||||
row = {
|
||||
"scenario": scenario,
|
||||
"concurrency": concurrency,
|
||||
"repeats": len(reports),
|
||||
"requests": requests,
|
||||
"successful_requests": successes,
|
||||
"failed_requests": failures,
|
||||
"success_percent": success_percent,
|
||||
"duration_seconds_total": duration_seconds,
|
||||
"rps_median": median(rps_values),
|
||||
"rps_min": round(min(rps_values), 3),
|
||||
"rps_max": round(max(rps_values), 3),
|
||||
"run_p95_median_ms": median(p95_values),
|
||||
"run_p99_worst_ms": round(max(p99_values), 3),
|
||||
"status_counts": merge_counts(reports, "status_counts"),
|
||||
"transport_error_counts": merge_counts(
|
||||
reports, "transport_error_counts"
|
||||
),
|
||||
}
|
||||
rows.append(row)
|
||||
if concurrency == 1 and failures:
|
||||
baseline_valid = False
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"baseline_concurrency_one_valid": baseline_valid,
|
||||
"groups": rows,
|
||||
}
|
||||
json_path = pathlib.Path(args.json_output)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
lines = [
|
||||
"| 场景 | 并发 | 重复 | 请求 | 成功率 | RPS 中位数(范围) | 每轮 P95 中位数 ms | 最差一轮 P99 ms | 状态码 | 传输错误 |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |",
|
||||
]
|
||||
for row in rows:
|
||||
status_counts = json.dumps(row["status_counts"], ensure_ascii=False)
|
||||
transport_error_counts = json.dumps(
|
||||
row["transport_error_counts"], ensure_ascii=False
|
||||
)
|
||||
lines.append(
|
||||
"| {scenario} | {concurrency} | {repeats} | {requests} | {success_percent:.3f}% | "
|
||||
"{rps_median} ({rps_min}-{rps_max}) | {run_p95_median_ms} | "
|
||||
"{run_p99_worst_ms} | `{status_counts}` | `{transport_error_counts}` |".format(
|
||||
scenario=row["scenario"],
|
||||
concurrency=row["concurrency"],
|
||||
repeats=row["repeats"],
|
||||
requests=row["requests"],
|
||||
success_percent=row["success_percent"],
|
||||
rps_median=row["rps_median"],
|
||||
rps_min=row["rps_min"],
|
||||
rps_max=row["rps_max"],
|
||||
run_p95_median_ms=row["run_p95_median_ms"],
|
||||
run_p99_worst_ms=row["run_p99_worst_ms"],
|
||||
status_counts=status_counts,
|
||||
transport_error_counts=transport_error_counts,
|
||||
)
|
||||
)
|
||||
markdown_path = pathlib.Path(args.markdown_output)
|
||||
markdown_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
if args.require_concurrency_one_success and not baseline_valid:
|
||||
raise SystemExit("Concurrency-one correctness baseline contained failures.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,6 +14,12 @@ def main():
|
||||
rows = []
|
||||
for path in sorted(directory.glob("*.json")):
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if (
|
||||
payload.get("schema_version") != 1
|
||||
or not isinstance(payload.get("concurrency"), int)
|
||||
or not isinstance(payload.get("latency_ms"), dict)
|
||||
):
|
||||
continue
|
||||
latency = payload.get("latency_ms") or {}
|
||||
rows.append(
|
||||
(
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"nodes": [
|
||||
{
|
||||
"id": "start",
|
||||
"type": "start"
|
||||
},
|
||||
{
|
||||
"id": "end",
|
||||
"type": "end"
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source": "start",
|
||||
"target": "end"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -152,6 +152,20 @@ Tripo 的“请求速率限制”和“运行任务并发限制”也不是同
|
||||
|
||||
## 7. 必须采集的指标
|
||||
|
||||
在将健康检查结果解释为业务容量前,先运行已认证本地业务接口基线:
|
||||
|
||||
```bash
|
||||
BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL=500 \
|
||||
BUSINESS_BENCHMARK_REPEATS=3 \
|
||||
./scripts/run-business-benchmark-suite.sh \
|
||||
compose/.env \
|
||||
business-baseline-$(date +%Y%m%d-%H%M%S)
|
||||
```
|
||||
|
||||
该套件只注册一个随机 `bench_business_` 测试账号,复用 Bearer Token 测试工作流校验、用户资料、项目列表、地址列表和设备列表。请求会经过 JWT 和 PostgreSQL;套件先验证 Redis 健康,Redis 可用时认证链会更新同一用户的滑动会话键。它不会调用 Tripo、对象存储、阿里云或支付。测试账号会留在数据库中供管理端核验;临时明文密码与 Token 不写入结果,Header 文件只放在 `/dev/shm` 并在脚本退出时删除,退出时还会尝试注销 Redis 会话,但已签发 JWT 不具备服务端强制撤销机制。
|
||||
|
||||
所有并发请求复用一个账号,直连 `127.0.0.1` 后端,列表数据量很小。因此结果只代表“单账号热点型、已认证后端 API 短时基线”,不代表多用户分布、前端/Nginx/Tailscale 链路或真实大数据量。资源采样的名义间隔为 1 秒,实际间隔以 CSV 时间戳为准;phase 是采样启动时的近似标签,短场景可能没有独立样本,不能据此精确归因单档峰值。短时阶梯测试只能称为业务 API 基线;稳定容量仍需长时间稳态和独立重复验证。
|
||||
|
||||
| 层级 | 指标 |
|
||||
| --- | --- |
|
||||
| 请求 | 请求总数、成功数、吞吐量、错误率、超时率、HTTP 状态码分布、P50/P90/P95/P99/最大延迟 |
|
||||
|
||||
Executable
+168
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
env_file="${1:-${repo_root}/compose/.env}"
|
||||
run_id="${2:-business-baseline-$(date +%Y%m%d-%H%M%S)}"
|
||||
request_count="${BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL:-500}"
|
||||
repeat_count="${BUSINESS_BENCHMARK_REPEATS:-3}"
|
||||
output_root="${repo_root}/results/raw/${run_id}"
|
||||
temporary_root=""
|
||||
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "Environment file not found: $env_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}$ || "$run_id" == *..* ]]; then
|
||||
echo "Run ID must be 1-120 safe filename characters and must not contain '..'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$request_count" =~ ^[1-9][0-9]*$ ]] || (( request_count > 100000 )); then
|
||||
echo "BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL must be between 1 and 100000." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$repeat_count" =~ ^[1-9][0-9]*$ ]] || (( repeat_count > 10 )); then
|
||||
echo "BUSINESS_BENCHMARK_REPEATS must be between 1 and 10." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$output_root"
|
||||
runtime_root="/dev/shm"
|
||||
if [[ ! -d "$runtime_root" || ! -w "$runtime_root" || "$(stat -f -c %T "$runtime_root")" != "tmpfs" ]]; then
|
||||
echo "Writable tmpfs runtime directory is unavailable: $runtime_root" >&2
|
||||
exit 1
|
||||
fi
|
||||
temporary_root="$(mktemp -d "${runtime_root}/ai-soulmate-business-benchmark.XXXXXX")"
|
||||
headers_file="${temporary_root}/headers.json"
|
||||
phase_file="${temporary_root}/phase.txt"
|
||||
sampler_stop="${temporary_root}/sampler-stop"
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${sampler_pid:-}" ]]; then
|
||||
touch "$sampler_stop"
|
||||
wait "$sampler_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [[ -r "$headers_file" ]]; then
|
||||
python3 "${repo_root}/benchmark/close-local-benchmark-session.py" \
|
||||
--base-url http://127.0.0.1:3001 \
|
||||
--headers-file "$headers_file" >/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -f "$headers_file" "$phase_file" "$sampler_stop"
|
||||
rmdir "$temporary_root" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
declare -a concurrencies=(1 5 10 20 40 80)
|
||||
deployment_commit="$(git -C "$repo_root" rev-parse HEAD)"
|
||||
if [[ -n "$(git -C "$repo_root" status --short --untracked-files=no)" ]]; then
|
||||
deployment_dirty=true
|
||||
else
|
||||
deployment_dirty=false
|
||||
fi
|
||||
project_name="$(sed -n 's/^BACKEND_COMPOSE_PROJECT_NAME=//p' "$env_file" | head -n 1)"
|
||||
project_name="${project_name:-ai-soulmate-backend}"
|
||||
backend_container_id="$(docker compose --project-name "$project_name" --env-file "$env_file" -f "${repo_root}/compose/docker-compose.backend.yml" ps -q backend)"
|
||||
if [[ -z "$backend_container_id" ]]; then
|
||||
echo "Running backend container was not found." >&2
|
||||
exit 1
|
||||
fi
|
||||
source_commit="$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$backend_container_id")"
|
||||
backend_image="$(sed -n 's/^BACKEND_IMAGE=//p' "$env_file" | head -n 1)"
|
||||
backend_image_id="$(docker inspect --format '{{.Image}}' "$backend_container_id")"
|
||||
backend_container_started_at="$(docker inspect --format '{{.State.StartedAt}}' "$backend_container_id")"
|
||||
|
||||
bash "${repo_root}/scripts/verify-tripo-mock.sh" "$env_file"
|
||||
bash "${repo_root}/scripts/verify-infra.sh" "$env_file" >/dev/null
|
||||
python3 "${repo_root}/benchmark/prepare-local-benchmark-session.py" \
|
||||
--base-url http://127.0.0.1:3001 \
|
||||
--headers-output "$headers_file" \
|
||||
--metadata-output "${output_root}/session-metadata.json" \
|
||||
--deployment-commit "$deployment_commit" \
|
||||
--deployment-dirty "$deployment_dirty" \
|
||||
--source-commit "$source_commit" \
|
||||
--backend-image "$backend_image" \
|
||||
--backend-image-id "$backend_image_id" \
|
||||
--backend-container-started-at "$backend_container_started_at" \
|
||||
--requests-per-level "$request_count" \
|
||||
--repeat-count "$repeat_count" \
|
||||
--concurrencies "$(IFS=,; echo "${concurrencies[*]}")"
|
||||
chmod 600 "$headers_file"
|
||||
|
||||
printf '%s\n' setup > "$phase_file"
|
||||
bash "${repo_root}/scripts/sample-benchmark-resources.sh" \
|
||||
"$sampler_stop" 1 "$phase_file" \
|
||||
> "${output_root}/resources-during-load.csv" &
|
||||
sampler_pid=$!
|
||||
|
||||
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
|
||||
> "${output_root}/snapshot-before.txt"
|
||||
|
||||
declare -a names=(workflow-validate user-profile projects-list address-list device-list)
|
||||
declare -a methods=(POST GET GET GET GET)
|
||||
declare -a paths=(
|
||||
/api/workflow/validate
|
||||
/api/user/profile
|
||||
/api/projects
|
||||
/api/address
|
||||
/api/device
|
||||
)
|
||||
for repeat in $(seq 1 "$repeat_count"); do
|
||||
for index in "${!names[@]}"; do
|
||||
name="${names[$index]}"
|
||||
method="${methods[$index]}"
|
||||
url="http://127.0.0.1:3001${paths[$index]}"
|
||||
for concurrency in "${concurrencies[@]}"; do
|
||||
phase="${name}-c${concurrency}-r${repeat}"
|
||||
printf '%s\n' "$phase" > "$phase_file"
|
||||
command=(
|
||||
python3 "${repo_root}/benchmark/run-http-benchmark.py"
|
||||
--url "$url"
|
||||
--method "$method"
|
||||
--requests "$request_count"
|
||||
--concurrency "$concurrency"
|
||||
--warmup 5
|
||||
--timeout 10
|
||||
--expected-status 200
|
||||
--headers-file "$headers_file"
|
||||
--allow-failures
|
||||
--output "${output_root}/${phase}.json"
|
||||
)
|
||||
if [[ "$method" == POST ]]; then
|
||||
command+=(--body-file "${repo_root}/benchmark/workflow-validate-body.json")
|
||||
fi
|
||||
"${command[@]}" > /dev/null
|
||||
echo "Completed ${phase}."
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
printf '%s\n' snapshot-after > "$phase_file"
|
||||
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
|
||||
> "${output_root}/snapshot-after.txt"
|
||||
|
||||
touch "$sampler_stop"
|
||||
if ! wait "$sampler_pid"; then
|
||||
echo "Resource sampler failed during the benchmark." >&2
|
||||
exit 1
|
||||
fi
|
||||
unset sampler_pid
|
||||
|
||||
expected_results="$(( ${#names[@]} * ${#concurrencies[@]} * repeat_count ))"
|
||||
actual_results="$(find "$output_root" -maxdepth 1 -type f -name '*-c*-r*.json' | wc -l)"
|
||||
if [[ "$actual_results" -ne "$expected_results" ]]; then
|
||||
echo "Expected ${expected_results} benchmark result files, found ${actual_results}." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! awk -F, 'NR > 1 && $2 != "" { found = 1 } END { exit(found ? 0 : 1) }' \
|
||||
"${output_root}/resources-during-load.csv"; then
|
||||
echo "Resource sampler did not capture any phase-tagged rows." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 "${repo_root}/benchmark/summarize-business-results.py" "$output_root" \
|
||||
--markdown-output "${output_root}/summary.md" \
|
||||
--json-output "${output_root}/summary.json" \
|
||||
--require-concurrency-one-success
|
||||
|
||||
echo "Authenticated business benchmark completed: $output_root"
|
||||
@@ -3,19 +3,24 @@ set -Eeuo pipefail
|
||||
|
||||
stop_file="${1:?stop file is required}"
|
||||
interval_seconds="${2:-1}"
|
||||
phase_file="${3:-}"
|
||||
|
||||
if ! [[ "$interval_seconds" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "Sampling interval must be a positive integer." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' 'captured_at,load_1m,mem_available_kib,container,cpu_percent,memory_usage,network_io,block_io,pids'
|
||||
printf '%s\n' 'captured_at,phase,load_1m,mem_available_kib,container,cpu_percent,memory_usage,network_io,block_io,pids'
|
||||
while [[ ! -e "$stop_file" ]]; do
|
||||
captured_at="$(date --iso-8601=seconds)"
|
||||
phase=""
|
||||
if [[ -n "$phase_file" && -r "$phase_file" ]]; then
|
||||
phase="$(tr -cd 'A-Za-z0-9_.-' < "$phase_file")"
|
||||
phase="${phase:0:120}"
|
||||
fi
|
||||
load_1m="$(awk '{print $1}' /proc/loadavg)"
|
||||
mem_available_kib="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)"
|
||||
docker stats --no-stream \
|
||||
--format "${captured_at},${load_1m},${mem_available_kib},{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" \
|
||||
|| true
|
||||
--format "${captured_at},${phase},${load_1m},${mem_available_kib},{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}"
|
||||
sleep "$interval_seconds"
|
||||
done
|
||||
Reference in new issue
Block a user