feat: add zero-cost performance benchmark workflow
This commit is contained in:
1 parent
54851fc191
commit
5293d93a30
17 files changed
+1366
-8
No files matched your search
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import math
|
||||
import pathlib
|
||||
import socket
|
||||
import statistics
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Run a bounded HTTP benchmark against a local service.")
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--method", default="GET", choices=("GET", "HEAD", "POST"))
|
||||
parser.add_argument("--requests", type=int, default=100)
|
||||
parser.add_argument("--concurrency", type=int, default=1)
|
||||
parser.add_argument("--timeout", type=float, default=10.0)
|
||||
parser.add_argument("--warmup", type=int, default=0)
|
||||
parser.add_argument("--expected-status", type=int, action="append")
|
||||
parser.add_argument("--body-file")
|
||||
parser.add_argument("--headers-file")
|
||||
parser.add_argument("--output")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_target(raw_url):
|
||||
parsed = urllib.parse.urlsplit(raw_url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("URL must use http or https and include a host.")
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("Credentials, query strings, and fragments are not allowed in benchmark URLs.")
|
||||
if parsed.hostname == "localhost":
|
||||
return
|
||||
try:
|
||||
addresses = {item[4][0] for item in socket.getaddrinfo(parsed.hostname, parsed.port)}
|
||||
except socket.gaierror as exc:
|
||||
raise ValueError(f"Unable to resolve benchmark host: {parsed.hostname}") from exc
|
||||
for address in addresses:
|
||||
candidate = ipaddress.ip_address(address)
|
||||
if not (candidate.is_loopback or candidate.is_private or candidate.is_link_local):
|
||||
raise ValueError(f"Refusing to benchmark non-local address: {address}")
|
||||
|
||||
|
||||
def percentile(values, percentage):
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
rank = max(1, math.ceil((percentage / 100) * len(ordered)))
|
||||
return round(ordered[rank - 1], 3)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if not 1 <= args.requests <= 1_000_000:
|
||||
raise SystemExit("--requests must be between 1 and 1000000")
|
||||
if not 1 <= args.concurrency <= 256:
|
||||
raise SystemExit("--concurrency must be between 1 and 256")
|
||||
if args.warmup < 0 or args.timeout <= 0:
|
||||
raise SystemExit("--warmup must be non-negative and --timeout must be positive")
|
||||
|
||||
validate_target(args.url)
|
||||
body = pathlib.Path(args.body_file).read_bytes() if args.body_file else None
|
||||
headers = {}
|
||||
if args.headers_file:
|
||||
headers = json.loads(pathlib.Path(args.headers_file).read_text(encoding="utf-8"))
|
||||
if not isinstance(headers, dict) or not all(isinstance(key, str) and isinstance(value, str) for key, value in headers.items()):
|
||||
raise SystemExit("--headers-file must contain a JSON object of string values")
|
||||
|
||||
expected_statuses = set(args.expected_status or range(200, 400))
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({}),
|
||||
NoRedirectHandler(),
|
||||
)
|
||||
|
||||
def request_once():
|
||||
request = urllib.request.Request(args.url, data=body, headers=headers, method=args.method)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with opener.open(request, timeout=args.timeout) as response:
|
||||
response_body = response.read()
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
return response.status, (time.perf_counter() - started) * 1000, None, response_body, retry_after
|
||||
except urllib.error.HTTPError as exc:
|
||||
response_body = exc.read()
|
||||
retry_after = exc.headers.get("Retry-After")
|
||||
return exc.code, (time.perf_counter() - started) * 1000, None, response_body, retry_after
|
||||
except Exception as exc:
|
||||
return None, (time.perf_counter() - started) * 1000, type(exc).__name__, b"", None
|
||||
|
||||
for _ in range(args.warmup):
|
||||
request_once()
|
||||
|
||||
started_at = datetime.datetime.now(datetime.timezone.utc)
|
||||
started = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
results = list(executor.map(lambda _: request_once(), range(args.requests)))
|
||||
duration_seconds = time.perf_counter() - started
|
||||
|
||||
latencies = [latency for _, latency, _, _, _ in results]
|
||||
statuses = {}
|
||||
errors = {}
|
||||
response_codes = {}
|
||||
retry_after_values = {}
|
||||
successful = 0
|
||||
for status, _, error, response_body, retry_after in results:
|
||||
if status is not None:
|
||||
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
||||
if status in expected_statuses:
|
||||
successful += 1
|
||||
if error:
|
||||
errors[error] = errors.get(error, 0) + 1
|
||||
if retry_after:
|
||||
retry_after_values[retry_after] = retry_after_values.get(retry_after, 0) + 1
|
||||
if response_body:
|
||||
try:
|
||||
parsed_body = json.loads(response_body)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
parsed_body = None
|
||||
if isinstance(parsed_body, dict) and parsed_body.get("code") is not None:
|
||||
code = str(parsed_body["code"])
|
||||
response_codes[code] = response_codes.get(code, 0) + 1
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"started_at_utc": started_at.isoformat(),
|
||||
"target": args.url,
|
||||
"method": args.method,
|
||||
"requests": args.requests,
|
||||
"concurrency": args.concurrency,
|
||||
"timeout_seconds": args.timeout,
|
||||
"warmup_requests": args.warmup,
|
||||
"body_bytes": len(body or b""),
|
||||
"custom_headers_supplied": bool(headers),
|
||||
"duration_seconds": round(duration_seconds, 6),
|
||||
"requests_per_second": round(args.requests / duration_seconds, 3),
|
||||
"successful_requests": successful,
|
||||
"failed_requests": args.requests - successful,
|
||||
"status_counts": statuses,
|
||||
"transport_error_counts": errors,
|
||||
"application_code_counts": response_codes,
|
||||
"retry_after_counts": retry_after_values,
|
||||
"latency_ms": {
|
||||
"min": round(min(latencies), 3),
|
||||
"mean": round(statistics.fmean(latencies), 3),
|
||||
"p50": percentile(latencies, 50),
|
||||
"p95": percentile(latencies, 95),
|
||||
"p99": percentile(latencies, 99),
|
||||
"max": round(max(latencies), 3),
|
||||
},
|
||||
}
|
||||
|
||||
rendered = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
print(rendered)
|
||||
if args.output:
|
||||
output_path = pathlib.Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered + "\n", encoding="utf-8")
|
||||
|
||||
if successful != args.requests:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -13,11 +13,30 @@ hardware_profile:
|
||||
|
||||
software:
|
||||
deployment_commit: ""
|
||||
server_commit: "24769465d3ac2cb840c5266c8c0def2aacb641aa"
|
||||
admin_commit: "40bd686128010fbbc6c33f6156a7a3cda5336b3a"
|
||||
server_commit: ""
|
||||
admin_commit: ""
|
||||
client_commit: null
|
||||
image_digests: {}
|
||||
database_migration: ""
|
||||
tripo_mode: "mock"
|
||||
external_paid_calls_allowed: false
|
||||
|
||||
tripo_mock:
|
||||
profile: "replace-with-compose-env-value"
|
||||
api_generation: "v2-poc"
|
||||
account_usage_tier: "unknown"
|
||||
source_url: "https://developers.tripo3d.com/zh/docs/rate-limits"
|
||||
source_checked_at: "2026-08-12"
|
||||
account_concurrency:
|
||||
h_series_3d: 10
|
||||
p_series_3d: 5
|
||||
image_generation: 1
|
||||
animation: 10
|
||||
model_postprocess: 5
|
||||
mesh_operations: 10
|
||||
queue_delay_seconds: 2
|
||||
run_delay_seconds: 8
|
||||
local_application_queue_enabled: false
|
||||
|
||||
workload:
|
||||
scenario_id: ""
|
||||
@@ -27,12 +46,13 @@ workload:
|
||||
request_mix: {}
|
||||
object_sizes: []
|
||||
includes_tripo: false
|
||||
uses_tripo_mock: true
|
||||
|
||||
configuration:
|
||||
application_profile: ""
|
||||
container_limits: {}
|
||||
database_pool_size: null
|
||||
worker_count: null
|
||||
worker_count: 1
|
||||
changed_parameters: []
|
||||
|
||||
result:
|
||||
@@ -49,4 +69,3 @@ result:
|
||||
artifacts:
|
||||
raw_directory: ""
|
||||
report_path: ""
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import pathlib
|
||||
import secrets
|
||||
import socket
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
||||
TERMINAL_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"}
|
||||
|
||||
|
||||
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
class SmokeFailure(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run one zero-credit user registration and text-to-3D smoke flow."
|
||||
)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:3001")
|
||||
parser.add_argument("--timeout", type=float, default=10.0)
|
||||
parser.add_argument("--poll-interval", type=float, default=1.0)
|
||||
parser.add_argument("--deadline", type=float, default=90.0)
|
||||
parser.add_argument("--output")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def private_addresses(hostname, port):
|
||||
if hostname == "localhost":
|
||||
return [ipaddress.ip_address("127.0.0.1")]
|
||||
try:
|
||||
return [
|
||||
ipaddress.ip_address(item[4][0])
|
||||
for item in socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
|
||||
]
|
||||
except (socket.gaierror, ValueError) as exc:
|
||||
raise SmokeFailure(f"Unable to resolve local target host: {hostname}") from exc
|
||||
|
||||
|
||||
def is_private_address(address):
|
||||
return address.is_loopback or address.is_private or address.is_link_local
|
||||
|
||||
|
||||
def validate_base_url(raw_url):
|
||||
parsed = urllib.parse.urlsplit(raw_url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise SmokeFailure("Base URL must use http or https and include a host.")
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise SmokeFailure("Credentials, query strings, and fragments are forbidden in the base URL.")
|
||||
if parsed.path not in {"", "/"}:
|
||||
raise SmokeFailure("Base URL must not include an API path.")
|
||||
addresses = private_addresses(parsed.hostname, parsed.port)
|
||||
if not addresses or any(not is_private_address(address) for address in addresses):
|
||||
raise SmokeFailure("Refusing to run the smoke flow against a non-local address.")
|
||||
return raw_url.rstrip("/")
|
||||
|
||||
|
||||
def read_json_response(response):
|
||||
raw_body = response.read(MAX_RESPONSE_BYTES + 1)
|
||||
if len(raw_body) > MAX_RESPONSE_BYTES:
|
||||
raise SmokeFailure("Server response exceeded the 2 MiB safety limit.")
|
||||
try:
|
||||
payload = json.loads(raw_body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise SmokeFailure("Server returned a non-JSON response.") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise SmokeFailure("Server returned a non-object JSON response.")
|
||||
return payload
|
||||
|
||||
|
||||
def storage_location(raw_url):
|
||||
value = str(raw_url or "").strip()
|
||||
result = {"present": bool(value), "scheme": None, "host": None, "local": False}
|
||||
if not value:
|
||||
return result
|
||||
if value.startswith("oss://") or value.startswith("models/"):
|
||||
result.update({"scheme": "oss", "host": "internal", "local": True})
|
||||
return result
|
||||
parsed = urllib.parse.urlsplit(value)
|
||||
result["scheme"] = parsed.scheme or "relative"
|
||||
result["host"] = parsed.hostname
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
return result
|
||||
if "tripo" in parsed.hostname.lower():
|
||||
return result
|
||||
try:
|
||||
addresses = private_addresses(parsed.hostname, parsed.port)
|
||||
except SmokeFailure:
|
||||
return result
|
||||
result["local"] = bool(addresses) and all(is_private_address(address) for address in addresses)
|
||||
return result
|
||||
|
||||
|
||||
class LocalApi:
|
||||
def __init__(self, base_url, timeout):
|
||||
self.base_url = base_url
|
||||
self.timeout = timeout
|
||||
self.opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({}), NoRedirectHandler()
|
||||
)
|
||||
|
||||
def request(self, method, path, body=None, bearer=None):
|
||||
if not path.startswith("/"):
|
||||
raise SmokeFailure("Internal request path must start with a slash.")
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "ai-soulmate-zero-credit-smoke/1",
|
||||
}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body, separators=(",", ":")).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
if bearer:
|
||||
headers["Authorization"] = f"Bearer {bearer}"
|
||||
request = urllib.request.Request(
|
||||
self.base_url + path, data=data, headers=headers, method=method
|
||||
)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with self.opener.open(request, timeout=self.timeout) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
raise SmokeFailure(f"{method} {path} returned HTTP {response.status}.")
|
||||
return response.status, read_json_response(response), round(
|
||||
(time.perf_counter() - started) * 1000, 3
|
||||
)
|
||||
except urllib.error.HTTPError as exc:
|
||||
exc.close()
|
||||
raise SmokeFailure(f"{method} {path} returned HTTP {exc.code}.") from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise SmokeFailure(f"{method} {path} failed at the transport layer.") from exc
|
||||
|
||||
|
||||
def numeric_credits(payload):
|
||||
value = payload.get("credits")
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise SmokeFailure("User profile did not contain numeric credits.")
|
||||
return int(value)
|
||||
|
||||
|
||||
def write_report(report, output):
|
||||
rendered = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
print(rendered)
|
||||
if output:
|
||||
output_path = pathlib.Path(output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if args.timeout <= 0 or args.poll_interval <= 0 or args.deadline <= 0:
|
||||
raise SystemExit("Timeout, poll interval, and deadline must be positive.")
|
||||
if args.timeout > 60 or args.deadline > 600:
|
||||
raise SystemExit("Timeout must be at most 60 seconds and deadline at most 600 seconds.")
|
||||
|
||||
started_at = datetime.datetime.now(datetime.timezone.utc)
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"started_at_utc": started_at.isoformat(),
|
||||
"base_url": None,
|
||||
"valid": False,
|
||||
"zero_credit_verified": False,
|
||||
"local_storage_verified": False,
|
||||
"steps": {},
|
||||
}
|
||||
current_step = "validate_target"
|
||||
try:
|
||||
base_url = validate_base_url(args.base_url)
|
||||
report["base_url"] = base_url
|
||||
api = LocalApi(base_url, args.timeout)
|
||||
|
||||
current_step = "verify_mock_mode"
|
||||
health_status, health, health_ms = api.request("GET", "/api/health")
|
||||
tripo_mode = str(health.get("tripo_mode") or "").strip().lower()
|
||||
mock_profile = str(health.get("tripo_mock_profile") or "").strip()
|
||||
report["steps"]["verify_mock_mode"] = {
|
||||
"http_status": health_status,
|
||||
"latency_ms": health_ms,
|
||||
"service_status": health.get("status"),
|
||||
"tripo_mode": tripo_mode,
|
||||
"tripo_mock_profile": mock_profile,
|
||||
}
|
||||
if tripo_mode != "mock" or not mock_profile or mock_profile == "not-selected":
|
||||
raise SmokeFailure("Backend health did not confirm an explicit Tripo mock profile.")
|
||||
|
||||
username = f"bench_{int(time.time())}_{secrets.token_hex(4)}"
|
||||
password = "Bm!9" + secrets.token_urlsafe(18)
|
||||
current_step = "register"
|
||||
register_status, registration, register_ms = api.request(
|
||||
"POST",
|
||||
"/api/auth/register",
|
||||
{
|
||||
"username": username,
|
||||
"password": password,
|
||||
"password_confirm": password,
|
||||
"register_via": "account",
|
||||
},
|
||||
)
|
||||
token = registration.get("access_token")
|
||||
user = registration.get("user")
|
||||
if not isinstance(token, str) or not token or not isinstance(user, dict):
|
||||
raise SmokeFailure("Registration response omitted the access token or user object.")
|
||||
user_id = str(user.get("id") or "")
|
||||
if not user_id:
|
||||
raise SmokeFailure("Registration response omitted the user ID.")
|
||||
report["steps"]["register"] = {
|
||||
"http_status": register_status,
|
||||
"latency_ms": register_ms,
|
||||
"user_id": user_id,
|
||||
}
|
||||
|
||||
current_step = "profile_before"
|
||||
profile_status, profile_before, profile_before_ms = api.request(
|
||||
"GET", "/api/user/profile", bearer=token
|
||||
)
|
||||
credits_before = numeric_credits(profile_before)
|
||||
report["steps"]["profile_before"] = {
|
||||
"http_status": profile_status,
|
||||
"latency_ms": profile_before_ms,
|
||||
"credits": credits_before,
|
||||
"assets_count": profile_before.get("assets_count"),
|
||||
}
|
||||
|
||||
current_step = "submit"
|
||||
submit_status, submitted, submit_ms = api.request(
|
||||
"POST",
|
||||
"/api/studio/submit-text-3d",
|
||||
{
|
||||
"prompt": "A small geometric calibration object for local benchmark validation",
|
||||
"with_texture": True,
|
||||
"model_version": "v3.1-20260211",
|
||||
"render_image": True,
|
||||
"result_format": "GLB",
|
||||
},
|
||||
bearer=token,
|
||||
)
|
||||
job_id = str(submitted.get("job_id") or "")
|
||||
if not job_id:
|
||||
raise SmokeFailure("Submission response omitted the job ID.")
|
||||
submit_credits = submitted.get("credits_used")
|
||||
report["steps"]["submit"] = {
|
||||
"http_status": submit_status,
|
||||
"latency_ms": submit_ms,
|
||||
"job_id": job_id,
|
||||
"credits_used": submit_credits,
|
||||
}
|
||||
|
||||
current_step = "poll_job"
|
||||
polling_started = time.monotonic()
|
||||
observations = []
|
||||
poll_count = 0
|
||||
terminal = None
|
||||
last_poll_ms = None
|
||||
while time.monotonic() - polling_started < args.deadline:
|
||||
_, polled, last_poll_ms = api.request(
|
||||
"GET", f"/api/studio/job/{urllib.parse.quote(job_id, safe='')}", bearer=token
|
||||
)
|
||||
poll_count += 1
|
||||
status_name = str(polled.get("status") or "UNKNOWN").upper()
|
||||
if not observations or observations[-1]["status"] != status_name:
|
||||
observations.append(
|
||||
{
|
||||
"status": status_name,
|
||||
"elapsed_seconds": round(time.monotonic() - polling_started, 3),
|
||||
}
|
||||
)
|
||||
if status_name in TERMINAL_STATUSES:
|
||||
terminal = polled
|
||||
break
|
||||
time.sleep(args.poll_interval)
|
||||
if terminal is None:
|
||||
raise SmokeFailure("Job did not reach a terminal state before the deadline.")
|
||||
if str(terminal.get("status") or "").upper() != "COMPLETED":
|
||||
raise SmokeFailure("Job reached a non-success terminal state.")
|
||||
|
||||
model_storage = storage_location(
|
||||
terminal.get("selected_model_url") or terminal.get("model_url")
|
||||
)
|
||||
preview_storage = storage_location(terminal.get("preview_url"))
|
||||
terminal_credits = terminal.get("credits_used")
|
||||
report["steps"]["poll_job"] = {
|
||||
"poll_count": poll_count,
|
||||
"duration_seconds": round(time.monotonic() - polling_started, 3),
|
||||
"last_poll_latency_ms": last_poll_ms,
|
||||
"status_observations": observations,
|
||||
"terminal_status": terminal.get("status"),
|
||||
"credits_used": terminal_credits,
|
||||
"asset_id": terminal.get("asset_id"),
|
||||
"model_storage": model_storage,
|
||||
"preview_storage": preview_storage,
|
||||
}
|
||||
|
||||
current_step = "profile_after"
|
||||
profile_after_status, profile_after, profile_after_ms = api.request(
|
||||
"GET", "/api/user/profile", bearer=token
|
||||
)
|
||||
credits_after = numeric_credits(profile_after)
|
||||
report["steps"]["profile_after"] = {
|
||||
"http_status": profile_after_status,
|
||||
"latency_ms": profile_after_ms,
|
||||
"credits": credits_after,
|
||||
"credits_delta": credits_after - credits_before,
|
||||
"assets_count": profile_after.get("assets_count"),
|
||||
}
|
||||
|
||||
report["zero_credit_verified"] = (
|
||||
submit_credits == 0 and terminal_credits == 0 and credits_after == credits_before
|
||||
)
|
||||
report["local_storage_verified"] = (
|
||||
model_storage["present"]
|
||||
and model_storage["local"]
|
||||
and preview_storage["present"]
|
||||
and preview_storage["local"]
|
||||
)
|
||||
report["valid"] = report["zero_credit_verified"] and report["local_storage_verified"]
|
||||
if not report["zero_credit_verified"]:
|
||||
raise SmokeFailure("The business flow did not preserve the user's credit balance.")
|
||||
if not report["local_storage_verified"]:
|
||||
raise SmokeFailure("The completed job did not return local model and preview storage.")
|
||||
except SmokeFailure as exc:
|
||||
report["failed_step"] = current_step
|
||||
report["error"] = str(exc)
|
||||
|
||||
report["duration_seconds"] = round(
|
||||
(datetime.datetime.now(datetime.timezone.utc) - started_at).total_seconds(), 3
|
||||
)
|
||||
write_report(report, args.output)
|
||||
if not report["valid"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Summarize HTTP benchmark JSON files as Markdown.")
|
||||
parser.add_argument("directory")
|
||||
parser.add_argument("--output")
|
||||
args = parser.parse_args()
|
||||
|
||||
directory = pathlib.Path(args.directory)
|
||||
rows = []
|
||||
for path in sorted(directory.glob("*.json")):
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
latency = payload.get("latency_ms") or {}
|
||||
rows.append(
|
||||
(
|
||||
path.stem,
|
||||
payload.get("concurrency"),
|
||||
payload.get("requests"),
|
||||
payload.get("successful_requests"),
|
||||
payload.get("failed_requests"),
|
||||
payload.get("requests_per_second"),
|
||||
latency.get("p50"),
|
||||
latency.get("p95"),
|
||||
latency.get("p99"),
|
||||
latency.get("max"),
|
||||
)
|
||||
)
|
||||
|
||||
if not rows:
|
||||
raise SystemExit(f"No benchmark JSON files found in: {directory}")
|
||||
|
||||
lines = [
|
||||
"| 场景 | 并发 | 请求数 | 成功 | 失败 | RPS | P50 ms | P95 ms | P99 ms | 最大 ms |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for row in rows:
|
||||
lines.append("| " + " | ".join(str(value) for value in row) + " |")
|
||||
rendered = "\n".join(lines) + "\n"
|
||||
print(rendered, end="")
|
||||
if args.output:
|
||||
output_path = pathlib.Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered, encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user