feat: add zero-cost performance benchmark workflow

This commit is contained in:
wlx committed 2026-08-13 12:02:36 +08:00
1 parent 54851fc191
commit 5293d93a30
17 files changed
+1366 -8

No files matched your search

+345
View File
@@ -0,0 +1,345 @@
#!/usr/bin/env python3
import argparse
import datetime
import ipaddress
import json
import pathlib
import secrets
import socket
import time
import urllib.error
import urllib.parse
import urllib.request
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
TERMINAL_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"}
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
class SmokeFailure(RuntimeError):
pass
def parse_args():
parser = argparse.ArgumentParser(
description="Run one zero-credit user registration and text-to-3D smoke flow."
)
parser.add_argument("--base-url", default="http://127.0.0.1:3001")
parser.add_argument("--timeout", type=float, default=10.0)
parser.add_argument("--poll-interval", type=float, default=1.0)
parser.add_argument("--deadline", type=float, default=90.0)
parser.add_argument("--output")
return parser.parse_args()
def private_addresses(hostname, port):
if hostname == "localhost":
return [ipaddress.ip_address("127.0.0.1")]
try:
return [
ipaddress.ip_address(item[4][0])
for item in socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
]
except (socket.gaierror, ValueError) as exc:
raise SmokeFailure(f"Unable to resolve local target host: {hostname}") from exc
def is_private_address(address):
return address.is_loopback or address.is_private or address.is_link_local
def validate_base_url(raw_url):
parsed = urllib.parse.urlsplit(raw_url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise SmokeFailure("Base URL must use http or https and include a host.")
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise SmokeFailure("Credentials, query strings, and fragments are forbidden in the base URL.")
if parsed.path not in {"", "/"}:
raise SmokeFailure("Base URL must not include an API path.")
addresses = private_addresses(parsed.hostname, parsed.port)
if not addresses or any(not is_private_address(address) for address in addresses):
raise SmokeFailure("Refusing to run the smoke flow against a non-local address.")
return raw_url.rstrip("/")
def read_json_response(response):
raw_body = response.read(MAX_RESPONSE_BYTES + 1)
if len(raw_body) > MAX_RESPONSE_BYTES:
raise SmokeFailure("Server response exceeded the 2 MiB safety limit.")
try:
payload = json.loads(raw_body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise SmokeFailure("Server returned a non-JSON response.") from exc
if not isinstance(payload, dict):
raise SmokeFailure("Server returned a non-object JSON response.")
return payload
def storage_location(raw_url):
value = str(raw_url or "").strip()
result = {"present": bool(value), "scheme": None, "host": None, "local": False}
if not value:
return result
if value.startswith("oss://") or value.startswith("models/"):
result.update({"scheme": "oss", "host": "internal", "local": True})
return result
parsed = urllib.parse.urlsplit(value)
result["scheme"] = parsed.scheme or "relative"
result["host"] = parsed.hostname
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
return result
if "tripo" in parsed.hostname.lower():
return result
try:
addresses = private_addresses(parsed.hostname, parsed.port)
except SmokeFailure:
return result
result["local"] = bool(addresses) and all(is_private_address(address) for address in addresses)
return result
class LocalApi:
def __init__(self, base_url, timeout):
self.base_url = base_url
self.timeout = timeout
self.opener = urllib.request.build_opener(
urllib.request.ProxyHandler({}), NoRedirectHandler()
)
def request(self, method, path, body=None, bearer=None):
if not path.startswith("/"):
raise SmokeFailure("Internal request path must start with a slash.")
headers = {
"Accept": "application/json",
"User-Agent": "ai-soulmate-zero-credit-smoke/1",
}
data = None
if body is not None:
data = json.dumps(body, separators=(",", ":")).encode("utf-8")
headers["Content-Type"] = "application/json"
if bearer:
headers["Authorization"] = f"Bearer {bearer}"
request = urllib.request.Request(
self.base_url + path, data=data, headers=headers, method=method
)
started = time.perf_counter()
try:
with self.opener.open(request, timeout=self.timeout) as response:
if not 200 <= response.status < 300:
raise SmokeFailure(f"{method} {path} returned HTTP {response.status}.")
return response.status, read_json_response(response), round(
(time.perf_counter() - started) * 1000, 3
)
except urllib.error.HTTPError as exc:
exc.close()
raise SmokeFailure(f"{method} {path} returned HTTP {exc.code}.") from exc
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise SmokeFailure(f"{method} {path} failed at the transport layer.") from exc
def numeric_credits(payload):
value = payload.get("credits")
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise SmokeFailure("User profile did not contain numeric credits.")
return int(value)
def write_report(report, output):
rendered = json.dumps(report, ensure_ascii=False, indent=2)
print(rendered)
if output:
output_path = pathlib.Path(output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(rendered + "\n", encoding="utf-8")
def main():
args = parse_args()
if args.timeout <= 0 or args.poll_interval <= 0 or args.deadline <= 0:
raise SystemExit("Timeout, poll interval, and deadline must be positive.")
if args.timeout > 60 or args.deadline > 600:
raise SystemExit("Timeout must be at most 60 seconds and deadline at most 600 seconds.")
started_at = datetime.datetime.now(datetime.timezone.utc)
report = {
"schema_version": 1,
"started_at_utc": started_at.isoformat(),
"base_url": None,
"valid": False,
"zero_credit_verified": False,
"local_storage_verified": False,
"steps": {},
}
current_step = "validate_target"
try:
base_url = validate_base_url(args.base_url)
report["base_url"] = base_url
api = LocalApi(base_url, args.timeout)
current_step = "verify_mock_mode"
health_status, health, health_ms = api.request("GET", "/api/health")
tripo_mode = str(health.get("tripo_mode") or "").strip().lower()
mock_profile = str(health.get("tripo_mock_profile") or "").strip()
report["steps"]["verify_mock_mode"] = {
"http_status": health_status,
"latency_ms": health_ms,
"service_status": health.get("status"),
"tripo_mode": tripo_mode,
"tripo_mock_profile": mock_profile,
}
if tripo_mode != "mock" or not mock_profile or mock_profile == "not-selected":
raise SmokeFailure("Backend health did not confirm an explicit Tripo mock profile.")
username = f"bench_{int(time.time())}_{secrets.token_hex(4)}"
password = "Bm!9" + secrets.token_urlsafe(18)
current_step = "register"
register_status, registration, register_ms = api.request(
"POST",
"/api/auth/register",
{
"username": username,
"password": password,
"password_confirm": password,
"register_via": "account",
},
)
token = registration.get("access_token")
user = registration.get("user")
if not isinstance(token, str) or not token or not isinstance(user, dict):
raise SmokeFailure("Registration response omitted the access token or user object.")
user_id = str(user.get("id") or "")
if not user_id:
raise SmokeFailure("Registration response omitted the user ID.")
report["steps"]["register"] = {
"http_status": register_status,
"latency_ms": register_ms,
"user_id": user_id,
}
current_step = "profile_before"
profile_status, profile_before, profile_before_ms = api.request(
"GET", "/api/user/profile", bearer=token
)
credits_before = numeric_credits(profile_before)
report["steps"]["profile_before"] = {
"http_status": profile_status,
"latency_ms": profile_before_ms,
"credits": credits_before,
"assets_count": profile_before.get("assets_count"),
}
current_step = "submit"
submit_status, submitted, submit_ms = api.request(
"POST",
"/api/studio/submit-text-3d",
{
"prompt": "A small geometric calibration object for local benchmark validation",
"with_texture": True,
"model_version": "v3.1-20260211",
"render_image": True,
"result_format": "GLB",
},
bearer=token,
)
job_id = str(submitted.get("job_id") or "")
if not job_id:
raise SmokeFailure("Submission response omitted the job ID.")
submit_credits = submitted.get("credits_used")
report["steps"]["submit"] = {
"http_status": submit_status,
"latency_ms": submit_ms,
"job_id": job_id,
"credits_used": submit_credits,
}
current_step = "poll_job"
polling_started = time.monotonic()
observations = []
poll_count = 0
terminal = None
last_poll_ms = None
while time.monotonic() - polling_started < args.deadline:
_, polled, last_poll_ms = api.request(
"GET", f"/api/studio/job/{urllib.parse.quote(job_id, safe='')}", bearer=token
)
poll_count += 1
status_name = str(polled.get("status") or "UNKNOWN").upper()
if not observations or observations[-1]["status"] != status_name:
observations.append(
{
"status": status_name,
"elapsed_seconds": round(time.monotonic() - polling_started, 3),
}
)
if status_name in TERMINAL_STATUSES:
terminal = polled
break
time.sleep(args.poll_interval)
if terminal is None:
raise SmokeFailure("Job did not reach a terminal state before the deadline.")
if str(terminal.get("status") or "").upper() != "COMPLETED":
raise SmokeFailure("Job reached a non-success terminal state.")
model_storage = storage_location(
terminal.get("selected_model_url") or terminal.get("model_url")
)
preview_storage = storage_location(terminal.get("preview_url"))
terminal_credits = terminal.get("credits_used")
report["steps"]["poll_job"] = {
"poll_count": poll_count,
"duration_seconds": round(time.monotonic() - polling_started, 3),
"last_poll_latency_ms": last_poll_ms,
"status_observations": observations,
"terminal_status": terminal.get("status"),
"credits_used": terminal_credits,
"asset_id": terminal.get("asset_id"),
"model_storage": model_storage,
"preview_storage": preview_storage,
}
current_step = "profile_after"
profile_after_status, profile_after, profile_after_ms = api.request(
"GET", "/api/user/profile", bearer=token
)
credits_after = numeric_credits(profile_after)
report["steps"]["profile_after"] = {
"http_status": profile_after_status,
"latency_ms": profile_after_ms,
"credits": credits_after,
"credits_delta": credits_after - credits_before,
"assets_count": profile_after.get("assets_count"),
}
report["zero_credit_verified"] = (
submit_credits == 0 and terminal_credits == 0 and credits_after == credits_before
)
report["local_storage_verified"] = (
model_storage["present"]
and model_storage["local"]
and preview_storage["present"]
and preview_storage["local"]
)
report["valid"] = report["zero_credit_verified"] and report["local_storage_verified"]
if not report["zero_credit_verified"]:
raise SmokeFailure("The business flow did not preserve the user's credit balance.")
if not report["local_storage_verified"]:
raise SmokeFailure("The completed job did not return local model and preview storage.")
except SmokeFailure as exc:
report["failed_step"] = current_step
report["error"] = str(exc)
report["duration_seconds"] = round(
(datetime.datetime.now(datetime.timezone.utc) - started_at).total_seconds(), 3
)
write_report(report, args.output)
if not report["valid"]:
raise SystemExit(2)
if __name__ == "__main__":
main()