fix: classify upstream failures and prevent empty billed responses

This commit is contained in:
wlx committed 2026-09-28 04:55:00 +08:00
1 parent b455d8d79f
commit 9acc69d4c7
28 files changed
+1449 -131

No files matched your search

+306
View File
@@ -0,0 +1,306 @@
#!/usr/bin/env python3
"""Audit every catalog route through the public MOYI API without exporting secrets.
Inventory is read-only by default. --run-calls explicitly creates one temporary
platform key, authorizes ready chat routes, makes bounded billed requests, then
disables/deletes only that key. An upstream HTTP 200 is recorded separately from
a nonempty completion. Neither catalog presence nor configuration is a live pass.
"""
import argparse
import collections
import datetime
import http.cookiejar
import importlib.util
import json
import os
from pathlib import Path
import time
import urllib.error
import urllib.request
TEXT_INPUT_TYPES = frozenset(("text", "string", "prompt"))
def normalize_types(value):
"""Return a stable list of declared input/output types from catalog JSON."""
if isinstance(value, str):
value = value.split(",")
if not isinstance(value, list):
return []
return [str(item).strip().lower() for item in value if str(item).strip()]
def chat_test_reasons(item, visible_route=True):
"""Explain why a route must not be sent a generic text chat request."""
reasons = []
if not visible_route:
reasons.append("inactive_route_or_channel")
if not item.get("pricingConfigured"):
reasons.append("price_missing")
if not item.get("billingReady"):
reasons.append("billing_not_ready")
if not item.get("executionReady"):
reasons.append("executor_not_ready")
if item.get("mode") == "async" or item.get("type") not in ("text", "code"):
reasons.append("requires_modality_specific_test")
input_types = normalize_types(item.get("inputTypes"))
if not input_types:
reasons.append("input_type_unknown")
elif not any(item_type in TEXT_INPUT_TYPES for item_type in input_types):
reasons.append("input_type_not_text")
elif any(item_type not in TEXT_INPUT_TYPES for item_type in input_types):
reasons.append("requires_modality_specific_test")
return list(dict.fromkeys(reasons))
def estimate_reservation(models, max_tokens):
"""Estimate the maximum reservation for the bounded chat probes."""
estimated = 0.0
for model in models:
pricing = model.get("pricing") or {}
unit = str(pricing.get("unit", "")).lower()
if "token" not in unit and pricing.get("price") is not None:
# Fixed-unit routes (for example CNY / task) charge once per probe.
estimated += float(pricing["price"])
else:
estimated += (
float(pricing.get("input", 0)) * 1024
+ float(pricing.get("output", 0)) * max_tokens
) / 1_000_000
return estimated
def load_retry_route_ids(path):
"""Read route IDs with a non-pass outcome from a previous audit report."""
document = json.loads(path.read_text(encoding="utf-8"))
results = document.get("results")
if not isinstance(results, list):
raise ValueError("Retry report does not contain a results list")
return {
str(result["id"])
for result in results
if isinstance(result, dict) and result.get("id") and result.get("outcome") != "pass"
}
def outcome_summary(results):
outcomes = collections.Counter(result.get("outcome", "unknown") for result in results)
passed = outcomes.get("pass", 0)
tested = len(results)
return {
"testedRoutes": tested,
"passedRoutes": passed,
"nonPassRoutes": tested - passed,
"passRate": round(passed / tested, 4) if tested else None,
"outcomes": dict(outcomes),
}
def cleanup_temporary_key(request, key_id):
"""Disable and delete a temporary key, attempting deletion after any failure."""
result = {"disabled": False, "deleted": False, "errors": []}
try:
status, _ = request("/api/keys/" + key_id + "/status", {"status": "disabled"}, method="PATCH")
result["disabled"] = status == 204
except Exception as error: # cleanup must continue to deletion
result["errors"].append({"operation": "disable", "type": type(error).__name__})
try:
status, _ = request("/api/keys/" + key_id, method="DELETE")
result["deleted"] = status == 204
except Exception as error:
result["errors"].append({"operation": "delete", "type": type(error).__name__})
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--deployment-dir", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--run-calls", action="store_true")
parser.add_argument("--max-estimated-cny", type=float, default=10)
parser.add_argument("--max-tokens", type=int, default=16)
parser.add_argument("--request-timeout", "--timeout", dest="request_timeout", type=float, default=120)
parser.add_argument("--only-model", action="append", dest="only_models", default=[])
parser.add_argument("--retry-report", type=Path,
help="Only re-test routes whose prior report outcome was not pass")
args = parser.parse_args()
if args.max_estimated_cny < 0:
parser.error("--max-estimated-cny must be non-negative")
if args.max_tokens < 1:
parser.error("--max-tokens must be at least 1")
if args.request_timeout < 1:
parser.error("--request-timeout must be at least 1 second")
os.umask(0o077)
retry_route_ids = load_retry_route_ids(args.retry_report) if args.retry_report else set()
spec = importlib.util.spec_from_file_location("moyi_deploy", args.deployment_dir / "deploy.py")
deploy = importlib.util.module_from_spec(spec)
spec.loader.exec_module(deploy)
config = deploy.load_config(args.deployment_dir / ".env.cloud", "https")
env = config["services"]["moyi-api"]["environment"]
base = "https://" + config["services"]["frontend"]["environment"]["MOYI_DOMAIN"]
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
def request(path, body=None, method=None, key=None, timeout=None):
timeout = args.request_timeout if timeout is None else timeout
headers = {"Content-Type": "application/json", "User-Agent": "moyi-catalog-acceptance/1"}
if key:
headers["Authorization"] = "Bearer " + key
req = urllib.request.Request(base + path, data=json.dumps(body).encode() if body is not None else None,
method=method, headers=headers)
try:
with opener.open(req, timeout=timeout) as response:
raw = response.read(2_000_000)
return response.status, json.loads(raw) if raw else {}
except urllib.error.HTTPError as error:
try:
payload = json.loads(error.read(8192))
except ValueError:
payload = {}
return error.code, payload
report = {
"startedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"baseUrl": base,
"parameters": {
"runCalls": args.run_calls,
"maxEstimatedCny": args.max_estimated_cny,
"maxTokens": args.max_tokens,
"requestTimeoutSeconds": args.request_timeout,
"onlyModels": args.only_models,
"retryReport": str(args.retry_report) if args.retry_report else None,
},
"retrySelection": {
"requestedNonPassRouteIds": sorted(retry_route_ids),
"matchedInventoryRouteIds": [],
"eligibleRouteIds": [],
"skippedRouteIds": [],
},
"inventory": [], "results": [], "temporaryKeyDeleted": False,
}
def save():
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
key_id = None
try:
status, _ = request("/api/auth/login", {"account": env["ADMIN_ACCOUNT"], "password": env["ADMIN_PASSWORD"]})
if status != 200:
raise RuntimeError("Admin login failed: HTTP " + str(status))
status, catalog = request("/api/admin/models")
if status != 200:
raise RuntimeError("Catalog read failed")
status, public = request("/api/models")
if status != 200:
raise RuntimeError("User catalog read failed")
active_ids = {item["id"] for item in public["models"]}
provider_names = {item["id"]: item["code"] for item in public["providers"]}
candidates = []
for item in catalog["models"]:
row = {field: item.get(field) for field in ("id", "logicalModelId", "modelKey", "publicName", "sourceLabel", "providerId", "upstreamModel", "type", "mode", "status", "maxConcurrency", "pricingConfigured", "billingReady", "executionReady")}
row["inputTypes"] = normalize_types(item.get("inputTypes"))
row["outputTypes"] = normalize_types(item.get("outputTypes"))
row["provider"] = provider_names.get(item["providerId"], item["providerId"])
row["visibleRoute"] = item["id"] in active_ids
row["pricing"] = item.get("pricing")
reasons = chat_test_reasons(item, row["visibleRoute"])
row["notChatTestedReasons"] = reasons
row["chatInputCompatible"] = not any(reason in reasons for reason in ("input_type_unknown", "input_type_not_text", "requires_modality_specific_test"))
row["retrySelected"] = item["id"] in retry_route_ids
report["inventory"].append(row)
if item["id"] in retry_route_ids:
report["retrySelection"]["matchedInventoryRouteIds"].append(item["id"])
selected_by_name = not args.only_models or item["id"] in args.only_models or item.get("upstreamModel") in args.only_models
selected_by_retry = not args.retry_report or item["id"] in retry_route_ids
if not reasons and selected_by_name and selected_by_retry:
candidates.append(item)
report["retrySelection"]["eligibleRouteIds"] = [item["id"] for item in candidates if item["id"] in retry_route_ids]
report["retrySelection"]["skippedRouteIds"] = sorted(retry_route_ids - set(report["retrySelection"]["eligibleRouteIds"]))
report["counts"] = {"adminRoutes": len(catalog["models"]), "userVisibleRoutes": len(active_ids),
"logicalDefinitions": len(catalog["logicalModels"]), "chatCandidates": len(candidates)}
report["channelCounts"] = dict(collections.Counter(row["provider"] for row in report["inventory"]))
print(json.dumps(report["counts"]), flush=True)
save()
if not args.run_calls:
return
estimated = estimate_reservation(candidates, args.max_tokens)
report["estimatedReservationCny"] = estimated
if estimated > args.max_estimated_cny:
raise RuntimeError("Estimated test reservation exceeds configured budget")
name = "catalog-audit-" + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S")
status, created = request("/api/keys", {"name": name, "modelIds": [m["id"] for m in candidates]})
if status != 201:
raise RuntimeError("Temporary test key creation failed: HTTP " + str(status))
key_id = created["apiKey"]["id"]
key = created["key"]
report["temporaryKeyName"] = name
status, models = request("/v1/models", key=key)
report["externalModelListStatus"] = status
if isinstance(models, dict):
report["externalModelListCount"] = len(models.get("data", []))
else:
report["externalModelListCount"] = None
report["externalModelListError"] = "response_not_object"
for index, item in enumerate(candidates):
start = time.monotonic()
result = {"id": item["id"], "model": item["upstreamModel"], "group": item["sourceLabel"],
"provider": provider_names.get(item["providerId"], item["providerId"]),
"request": {"endpoint": "/v1/chat/completions", "modelId": item["id"],
"maxTokens": args.max_tokens, "stream": False,
"timeoutSeconds": args.request_timeout}}
try:
status, payload = request("/v1/chat/completions", {"model": item["id"], "messages": [{"role": "user", "content": "Reply with OK."}], "max_tokens": args.max_tokens, "stream": False}, key=key)
result["status"] = status
if status == 200:
if not isinstance(payload, dict):
result["outcome"] = "invalid_payload"
result["errorCode"] = "RESPONSE_NOT_OBJECT"
else:
choices = payload.get("choices") or []
message = choices[0].get("message", {}) if choices and isinstance(choices[0], dict) else {}
content = message.get("content")
result["contentType"] = type(content).__name__ if content is not None else None
result["completionPresent"] = bool(content or message.get("tool_calls"))
# Reasoning-only responses are valid upstream responses when
# a reasoning model reaches the caller's output budget before
# writing its final answer. Keep them distinct from a truly
# empty response so the audit does not misdiagnose a budget
# outcome as a transport or gateway failure.
result["reasoningPresent"] = bool(message.get("reasoning_content") or message.get("reasoning"))
result["finishReason"] = choices[0].get("finish_reason") if choices and isinstance(choices[0], dict) else None
usage = payload.get("usage", {})
result["tokens"] = {k: usage.get(k) for k in ("prompt_tokens", "completion_tokens", "total_tokens")} if isinstance(usage, dict) else None
result["outcome"] = "pass" if result["completionPresent"] else "reasoning_only" if result["reasoningPresent"] else "empty_completion"
else:
error = payload.get("error", {})
result["errorCode"] = str(error.get("code", "HTTP_" + str(status))) if isinstance(error, dict) else "HTTP_" + str(status)
result["outcome"] = "failed"
except Exception as error:
result.update(outcome="transport_error", errorType=type(error).__name__)
result["elapsedSeconds"] = round(time.monotonic() - start, 3)
report["results"].append(result)
save()
print(json.dumps({"index": index + 1, "total": len(candidates), **result}, ensure_ascii=False), flush=True)
# Prevent short upstream failures from flooding the gateway RPM limit.
time.sleep(max(0, 1.1 - (time.monotonic() - start)))
finally:
if key_id:
cleanup = cleanup_temporary_key(request, key_id)
report["temporaryKeyDisabled"] = cleanup["disabled"]
report["temporaryKeyDeleted"] = cleanup["deleted"]
if cleanup["errors"]:
report["cleanupErrors"] = cleanup["errors"]
report["finishedAt"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
report.update(outcome_summary(report["results"]))
save()
try:
request("/api/auth/logout", {}, method="POST")
except Exception as error:
report["logoutError"] = type(error).__name__
save()
print(json.dumps({"outcomes": report["outcomes"], "temporaryKeyDeleted": report["temporaryKeyDeleted"]}), flush=True)
if __name__ == "__main__":
main()
+65
View File
@@ -0,0 +1,65 @@
import importlib.util
import json
import tempfile
import unittest
from pathlib import Path
SCRIPT = Path(__file__).with_name("live-catalog-audit.py")
SPEC = importlib.util.spec_from_file_location("live_catalog_audit", SCRIPT)
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
class LiveCatalogAuditTests(unittest.TestCase):
def test_estimate_uses_requested_max_tokens(self):
route = {"pricing": {"input": 1, "output": 2}}
self.assertAlmostEqual(MODULE.estimate_reservation([route], 16), (1024 + 32) / 1_000_000)
self.assertAlmostEqual(MODULE.estimate_reservation([route], 128), (1024 + 256) / 1_000_000)
self.assertEqual(MODULE.estimate_reservation([{"pricing": {"price": 1.44, "unit": "CNY / task"}}], 16), 1.44)
def test_input_type_check_excludes_ocr_from_text_probe(self):
route = {
"pricingConfigured": True, "billingReady": True, "executionReady": True,
"mode": "sync", "type": "text", "inputTypes": ["image"],
}
reasons = MODULE.chat_test_reasons(route)
self.assertIn("input_type_not_text", reasons)
def test_unknown_and_mixed_input_types_are_explicit(self):
unknown = {"type": "text", "mode": "sync", "inputTypes": []}
mixed = {"type": "text", "mode": "sync", "inputTypes": ["text", "image"]}
self.assertIn("input_type_unknown", MODULE.chat_test_reasons(unknown))
self.assertIn("requires_modality_specific_test", MODULE.chat_test_reasons(mixed))
def test_retry_report_selects_only_non_pass_route_ids(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "previous.json"
path.write_text(json.dumps({"results": [
{"id": "pass-route", "outcome": "pass"},
{"id": "failed-route", "outcome": "failed"},
{"id": "reasoning-route", "outcome": "reasoning_only"},
]}), encoding="utf-8")
self.assertEqual(MODULE.load_retry_route_ids(path), {"failed-route", "reasoning-route"})
def test_cleanup_attempts_delete_when_disable_fails(self):
calls = []
def request(path, body=None, method=None):
calls.append((path, method))
if method == "PATCH":
raise TimeoutError("simulated timeout")
return 204, {}
result = MODULE.cleanup_temporary_key(request, "temporary")
self.assertFalse(result["disabled"])
self.assertTrue(result["deleted"])
self.assertEqual(calls, [
("/api/keys/temporary/status", "PATCH"),
("/api/keys/temporary", "DELETE"),
])
self.assertEqual(result["errors"][0]["operation"], "disable")
if __name__ == "__main__":
unittest.main()