307 lines
16 KiB
Python
307 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit every catalog route through the public MOYI API without exporting secrets.
|
|
|
|
Inventory is read-only by default. --run-calls explicitly creates one temporary
|
|
platform key, authorizes ready chat routes, makes bounded billed requests, then
|
|
disables/deletes only that key. An upstream HTTP 200 is recorded separately from
|
|
a nonempty completion. Neither catalog presence nor configuration is a live pass.
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import datetime
|
|
import http.cookiejar
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
|
|
TEXT_INPUT_TYPES = frozenset(("text", "string", "prompt"))
|
|
|
|
|
|
def normalize_types(value):
|
|
"""Return a stable list of declared input/output types from catalog JSON."""
|
|
if isinstance(value, str):
|
|
value = value.split(",")
|
|
if not isinstance(value, list):
|
|
return []
|
|
return [str(item).strip().lower() for item in value if str(item).strip()]
|
|
|
|
|
|
def chat_test_reasons(item, visible_route=True):
|
|
"""Explain why a route must not be sent a generic text chat request."""
|
|
reasons = []
|
|
if not visible_route:
|
|
reasons.append("inactive_route_or_channel")
|
|
if not item.get("pricingConfigured"):
|
|
reasons.append("price_missing")
|
|
if not item.get("billingReady"):
|
|
reasons.append("billing_not_ready")
|
|
if not item.get("executionReady"):
|
|
reasons.append("executor_not_ready")
|
|
if item.get("mode") == "async" or item.get("type") not in ("text", "code"):
|
|
reasons.append("requires_modality_specific_test")
|
|
input_types = normalize_types(item.get("inputTypes"))
|
|
if not input_types:
|
|
reasons.append("input_type_unknown")
|
|
elif not any(item_type in TEXT_INPUT_TYPES for item_type in input_types):
|
|
reasons.append("input_type_not_text")
|
|
elif any(item_type not in TEXT_INPUT_TYPES for item_type in input_types):
|
|
reasons.append("requires_modality_specific_test")
|
|
return list(dict.fromkeys(reasons))
|
|
|
|
|
|
def estimate_reservation(models, max_tokens):
|
|
"""Estimate the maximum reservation for the bounded chat probes."""
|
|
estimated = 0.0
|
|
for model in models:
|
|
pricing = model.get("pricing") or {}
|
|
unit = str(pricing.get("unit", "")).lower()
|
|
if "token" not in unit and pricing.get("price") is not None:
|
|
# Fixed-unit routes (for example CNY / task) charge once per probe.
|
|
estimated += float(pricing["price"])
|
|
else:
|
|
estimated += (
|
|
float(pricing.get("input", 0)) * 1024
|
|
+ float(pricing.get("output", 0)) * max_tokens
|
|
) / 1_000_000
|
|
return estimated
|
|
|
|
|
|
def load_retry_route_ids(path):
|
|
"""Read route IDs with a non-pass outcome from a previous audit report."""
|
|
document = json.loads(path.read_text(encoding="utf-8"))
|
|
results = document.get("results")
|
|
if not isinstance(results, list):
|
|
raise ValueError("Retry report does not contain a results list")
|
|
return {
|
|
str(result["id"])
|
|
for result in results
|
|
if isinstance(result, dict) and result.get("id") and result.get("outcome") != "pass"
|
|
}
|
|
|
|
|
|
def outcome_summary(results):
|
|
outcomes = collections.Counter(result.get("outcome", "unknown") for result in results)
|
|
passed = outcomes.get("pass", 0)
|
|
tested = len(results)
|
|
return {
|
|
"testedRoutes": tested,
|
|
"passedRoutes": passed,
|
|
"nonPassRoutes": tested - passed,
|
|
"passRate": round(passed / tested, 4) if tested else None,
|
|
"outcomes": dict(outcomes),
|
|
}
|
|
|
|
|
|
def cleanup_temporary_key(request, key_id):
|
|
"""Disable and delete a temporary key, attempting deletion after any failure."""
|
|
result = {"disabled": False, "deleted": False, "errors": []}
|
|
try:
|
|
status, _ = request("/api/keys/" + key_id + "/status", {"status": "disabled"}, method="PATCH")
|
|
result["disabled"] = status == 204
|
|
except Exception as error: # cleanup must continue to deletion
|
|
result["errors"].append({"operation": "disable", "type": type(error).__name__})
|
|
try:
|
|
status, _ = request("/api/keys/" + key_id, method="DELETE")
|
|
result["deleted"] = status == 204
|
|
except Exception as error:
|
|
result["errors"].append({"operation": "delete", "type": type(error).__name__})
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--deployment-dir", required=True, type=Path)
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
parser.add_argument("--run-calls", action="store_true")
|
|
parser.add_argument("--max-estimated-cny", type=float, default=10)
|
|
parser.add_argument("--max-tokens", type=int, default=16)
|
|
parser.add_argument("--request-timeout", "--timeout", dest="request_timeout", type=float, default=120)
|
|
parser.add_argument("--only-model", action="append", dest="only_models", default=[])
|
|
parser.add_argument("--retry-report", type=Path,
|
|
help="Only re-test routes whose prior report outcome was not pass")
|
|
args = parser.parse_args()
|
|
if args.max_estimated_cny < 0:
|
|
parser.error("--max-estimated-cny must be non-negative")
|
|
if args.max_tokens < 1:
|
|
parser.error("--max-tokens must be at least 1")
|
|
if args.request_timeout < 1:
|
|
parser.error("--request-timeout must be at least 1 second")
|
|
os.umask(0o077)
|
|
retry_route_ids = load_retry_route_ids(args.retry_report) if args.retry_report else set()
|
|
spec = importlib.util.spec_from_file_location("moyi_deploy", args.deployment_dir / "deploy.py")
|
|
deploy = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(deploy)
|
|
config = deploy.load_config(args.deployment_dir / ".env.cloud", "https")
|
|
env = config["services"]["moyi-api"]["environment"]
|
|
base = "https://" + config["services"]["frontend"]["environment"]["MOYI_DOMAIN"]
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
|
|
|
|
def request(path, body=None, method=None, key=None, timeout=None):
|
|
timeout = args.request_timeout if timeout is None else timeout
|
|
headers = {"Content-Type": "application/json", "User-Agent": "moyi-catalog-acceptance/1"}
|
|
if key:
|
|
headers["Authorization"] = "Bearer " + key
|
|
req = urllib.request.Request(base + path, data=json.dumps(body).encode() if body is not None else None,
|
|
method=method, headers=headers)
|
|
try:
|
|
with opener.open(req, timeout=timeout) as response:
|
|
raw = response.read(2_000_000)
|
|
return response.status, json.loads(raw) if raw else {}
|
|
except urllib.error.HTTPError as error:
|
|
try:
|
|
payload = json.loads(error.read(8192))
|
|
except ValueError:
|
|
payload = {}
|
|
return error.code, payload
|
|
|
|
report = {
|
|
"startedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"baseUrl": base,
|
|
"parameters": {
|
|
"runCalls": args.run_calls,
|
|
"maxEstimatedCny": args.max_estimated_cny,
|
|
"maxTokens": args.max_tokens,
|
|
"requestTimeoutSeconds": args.request_timeout,
|
|
"onlyModels": args.only_models,
|
|
"retryReport": str(args.retry_report) if args.retry_report else None,
|
|
},
|
|
"retrySelection": {
|
|
"requestedNonPassRouteIds": sorted(retry_route_ids),
|
|
"matchedInventoryRouteIds": [],
|
|
"eligibleRouteIds": [],
|
|
"skippedRouteIds": [],
|
|
},
|
|
"inventory": [], "results": [], "temporaryKeyDeleted": False,
|
|
}
|
|
|
|
def save():
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
key_id = None
|
|
try:
|
|
status, _ = request("/api/auth/login", {"account": env["ADMIN_ACCOUNT"], "password": env["ADMIN_PASSWORD"]})
|
|
if status != 200:
|
|
raise RuntimeError("Admin login failed: HTTP " + str(status))
|
|
status, catalog = request("/api/admin/models")
|
|
if status != 200:
|
|
raise RuntimeError("Catalog read failed")
|
|
status, public = request("/api/models")
|
|
if status != 200:
|
|
raise RuntimeError("User catalog read failed")
|
|
active_ids = {item["id"] for item in public["models"]}
|
|
provider_names = {item["id"]: item["code"] for item in public["providers"]}
|
|
candidates = []
|
|
for item in catalog["models"]:
|
|
row = {field: item.get(field) for field in ("id", "logicalModelId", "modelKey", "publicName", "sourceLabel", "providerId", "upstreamModel", "type", "mode", "status", "maxConcurrency", "pricingConfigured", "billingReady", "executionReady")}
|
|
row["inputTypes"] = normalize_types(item.get("inputTypes"))
|
|
row["outputTypes"] = normalize_types(item.get("outputTypes"))
|
|
row["provider"] = provider_names.get(item["providerId"], item["providerId"])
|
|
row["visibleRoute"] = item["id"] in active_ids
|
|
row["pricing"] = item.get("pricing")
|
|
reasons = chat_test_reasons(item, row["visibleRoute"])
|
|
row["notChatTestedReasons"] = reasons
|
|
row["chatInputCompatible"] = not any(reason in reasons for reason in ("input_type_unknown", "input_type_not_text", "requires_modality_specific_test"))
|
|
row["retrySelected"] = item["id"] in retry_route_ids
|
|
report["inventory"].append(row)
|
|
if item["id"] in retry_route_ids:
|
|
report["retrySelection"]["matchedInventoryRouteIds"].append(item["id"])
|
|
selected_by_name = not args.only_models or item["id"] in args.only_models or item.get("upstreamModel") in args.only_models
|
|
selected_by_retry = not args.retry_report or item["id"] in retry_route_ids
|
|
if not reasons and selected_by_name and selected_by_retry:
|
|
candidates.append(item)
|
|
report["retrySelection"]["eligibleRouteIds"] = [item["id"] for item in candidates if item["id"] in retry_route_ids]
|
|
report["retrySelection"]["skippedRouteIds"] = sorted(retry_route_ids - set(report["retrySelection"]["eligibleRouteIds"]))
|
|
report["counts"] = {"adminRoutes": len(catalog["models"]), "userVisibleRoutes": len(active_ids),
|
|
"logicalDefinitions": len(catalog["logicalModels"]), "chatCandidates": len(candidates)}
|
|
report["channelCounts"] = dict(collections.Counter(row["provider"] for row in report["inventory"]))
|
|
print(json.dumps(report["counts"]), flush=True)
|
|
save()
|
|
if not args.run_calls:
|
|
return
|
|
estimated = estimate_reservation(candidates, args.max_tokens)
|
|
report["estimatedReservationCny"] = estimated
|
|
if estimated > args.max_estimated_cny:
|
|
raise RuntimeError("Estimated test reservation exceeds configured budget")
|
|
name = "catalog-audit-" + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S")
|
|
status, created = request("/api/keys", {"name": name, "modelIds": [m["id"] for m in candidates]})
|
|
if status != 201:
|
|
raise RuntimeError("Temporary test key creation failed: HTTP " + str(status))
|
|
key_id = created["apiKey"]["id"]
|
|
key = created["key"]
|
|
report["temporaryKeyName"] = name
|
|
status, models = request("/v1/models", key=key)
|
|
report["externalModelListStatus"] = status
|
|
if isinstance(models, dict):
|
|
report["externalModelListCount"] = len(models.get("data", []))
|
|
else:
|
|
report["externalModelListCount"] = None
|
|
report["externalModelListError"] = "response_not_object"
|
|
for index, item in enumerate(candidates):
|
|
start = time.monotonic()
|
|
result = {"id": item["id"], "model": item["upstreamModel"], "group": item["sourceLabel"],
|
|
"provider": provider_names.get(item["providerId"], item["providerId"]),
|
|
"request": {"endpoint": "/v1/chat/completions", "modelId": item["id"],
|
|
"maxTokens": args.max_tokens, "stream": False,
|
|
"timeoutSeconds": args.request_timeout}}
|
|
try:
|
|
status, payload = request("/v1/chat/completions", {"model": item["id"], "messages": [{"role": "user", "content": "Reply with OK."}], "max_tokens": args.max_tokens, "stream": False}, key=key)
|
|
result["status"] = status
|
|
if status == 200:
|
|
if not isinstance(payload, dict):
|
|
result["outcome"] = "invalid_payload"
|
|
result["errorCode"] = "RESPONSE_NOT_OBJECT"
|
|
else:
|
|
choices = payload.get("choices") or []
|
|
message = choices[0].get("message", {}) if choices and isinstance(choices[0], dict) else {}
|
|
content = message.get("content")
|
|
result["contentType"] = type(content).__name__ if content is not None else None
|
|
result["completionPresent"] = bool(content or message.get("tool_calls"))
|
|
# Reasoning-only responses are valid upstream responses when
|
|
# a reasoning model reaches the caller's output budget before
|
|
# writing its final answer. Keep them distinct from a truly
|
|
# empty response so the audit does not misdiagnose a budget
|
|
# outcome as a transport or gateway failure.
|
|
result["reasoningPresent"] = bool(message.get("reasoning_content") or message.get("reasoning"))
|
|
result["finishReason"] = choices[0].get("finish_reason") if choices and isinstance(choices[0], dict) else None
|
|
usage = payload.get("usage", {})
|
|
result["tokens"] = {k: usage.get(k) for k in ("prompt_tokens", "completion_tokens", "total_tokens")} if isinstance(usage, dict) else None
|
|
result["outcome"] = "pass" if result["completionPresent"] else "reasoning_only" if result["reasoningPresent"] else "empty_completion"
|
|
else:
|
|
error = payload.get("error", {})
|
|
result["errorCode"] = str(error.get("code", "HTTP_" + str(status))) if isinstance(error, dict) else "HTTP_" + str(status)
|
|
result["outcome"] = "failed"
|
|
except Exception as error:
|
|
result.update(outcome="transport_error", errorType=type(error).__name__)
|
|
result["elapsedSeconds"] = round(time.monotonic() - start, 3)
|
|
report["results"].append(result)
|
|
save()
|
|
print(json.dumps({"index": index + 1, "total": len(candidates), **result}, ensure_ascii=False), flush=True)
|
|
# Prevent short upstream failures from flooding the gateway RPM limit.
|
|
time.sleep(max(0, 1.1 - (time.monotonic() - start)))
|
|
finally:
|
|
if key_id:
|
|
cleanup = cleanup_temporary_key(request, key_id)
|
|
report["temporaryKeyDisabled"] = cleanup["disabled"]
|
|
report["temporaryKeyDeleted"] = cleanup["deleted"]
|
|
if cleanup["errors"]:
|
|
report["cleanupErrors"] = cleanup["errors"]
|
|
report["finishedAt"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
report.update(outcome_summary(report["results"]))
|
|
save()
|
|
try:
|
|
request("/api/auth/logout", {}, method="POST")
|
|
except Exception as error:
|
|
report["logoutError"] = type(error).__name__
|
|
save()
|
|
print(json.dumps({"outcomes": report["outcomes"], "temporaryKeyDeleted": report["temporaryKeyDeleted"]}), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|