#!/usr/bin/env python3 """Audit every catalog route through the public MOYI API without exporting secrets. Inventory is read-only by default. --run-calls explicitly creates one temporary platform key, authorizes ready chat routes, makes bounded billed requests, then disables/deletes only that key. An upstream HTTP 200 is recorded separately from a nonempty completion. Neither catalog presence nor configuration is a live pass. """ import argparse import collections import datetime import http.cookiejar import importlib.util import json import os from pathlib import Path import time import urllib.error import urllib.request TEXT_INPUT_TYPES = frozenset(("text", "string", "prompt")) def normalize_types(value): """Return a stable list of declared input/output types from catalog JSON.""" if isinstance(value, str): value = value.split(",") if not isinstance(value, list): return [] return [str(item).strip().lower() for item in value if str(item).strip()] def chat_test_reasons(item, visible_route=True): """Explain why a route must not be sent a generic text chat request.""" reasons = [] if not visible_route: reasons.append("inactive_route_or_channel") if not item.get("pricingConfigured"): reasons.append("price_missing") if not item.get("billingReady"): reasons.append("billing_not_ready") if not item.get("executionReady"): reasons.append("executor_not_ready") if item.get("mode") == "async" or item.get("type") not in ("text", "code"): reasons.append("requires_modality_specific_test") input_types = normalize_types(item.get("inputTypes")) if not input_types: reasons.append("input_type_unknown") elif not any(item_type in TEXT_INPUT_TYPES for item_type in input_types): reasons.append("input_type_not_text") elif any(item_type not in TEXT_INPUT_TYPES for item_type in input_types): reasons.append("requires_modality_specific_test") return list(dict.fromkeys(reasons)) def estimate_reservation(models, max_tokens): """Estimate the maximum reservation for the bounded chat probes.""" estimated = 0.0 for model in models: pricing = model.get("pricing") or {} unit = str(pricing.get("unit", "")).lower() if "token" not in unit and pricing.get("price") is not None: # Fixed-unit routes (for example CNY / task) charge once per probe. estimated += float(pricing["price"]) else: estimated += ( float(pricing.get("input", 0)) * 1024 + float(pricing.get("output", 0)) * max_tokens ) / 1_000_000 return estimated def load_retry_route_ids(path): """Read route IDs with a non-pass outcome from a previous audit report.""" document = json.loads(path.read_text(encoding="utf-8")) results = document.get("results") if not isinstance(results, list): raise ValueError("Retry report does not contain a results list") return { str(result["id"]) for result in results if isinstance(result, dict) and result.get("id") and result.get("outcome") != "pass" } def outcome_summary(results): outcomes = collections.Counter(result.get("outcome", "unknown") for result in results) passed = outcomes.get("pass", 0) tested = len(results) return { "testedRoutes": tested, "passedRoutes": passed, "nonPassRoutes": tested - passed, "passRate": round(passed / tested, 4) if tested else None, "outcomes": dict(outcomes), } def cleanup_temporary_key(request, key_id): """Disable and delete a temporary key, attempting deletion after any failure.""" result = {"disabled": False, "deleted": False, "errors": []} try: status, _ = request("/api/keys/" + key_id + "/status", {"status": "disabled"}, method="PATCH") result["disabled"] = status == 204 except Exception as error: # cleanup must continue to deletion result["errors"].append({"operation": "disable", "type": type(error).__name__}) try: status, _ = request("/api/keys/" + key_id, method="DELETE") result["deleted"] = status == 204 except Exception as error: result["errors"].append({"operation": "delete", "type": type(error).__name__}) return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--deployment-dir", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--run-calls", action="store_true") parser.add_argument("--max-estimated-cny", type=float, default=10) parser.add_argument("--max-tokens", type=int, default=16) parser.add_argument("--request-timeout", "--timeout", dest="request_timeout", type=float, default=120) parser.add_argument("--only-model", action="append", dest="only_models", default=[]) parser.add_argument("--retry-report", type=Path, help="Only re-test routes whose prior report outcome was not pass") args = parser.parse_args() if args.max_estimated_cny < 0: parser.error("--max-estimated-cny must be non-negative") if args.max_tokens < 1: parser.error("--max-tokens must be at least 1") if args.request_timeout < 1: parser.error("--request-timeout must be at least 1 second") os.umask(0o077) retry_route_ids = load_retry_route_ids(args.retry_report) if args.retry_report else set() spec = importlib.util.spec_from_file_location("moyi_deploy", args.deployment_dir / "deploy.py") deploy = importlib.util.module_from_spec(spec) spec.loader.exec_module(deploy) config = deploy.load_config(args.deployment_dir / ".env.cloud", "https") env = config["services"]["moyi-api"]["environment"] base = "https://" + config["services"]["frontend"]["environment"]["MOYI_DOMAIN"] opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())) def request(path, body=None, method=None, key=None, timeout=None): timeout = args.request_timeout if timeout is None else timeout headers = {"Content-Type": "application/json", "User-Agent": "moyi-catalog-acceptance/1"} if key: headers["Authorization"] = "Bearer " + key req = urllib.request.Request(base + path, data=json.dumps(body).encode() if body is not None else None, method=method, headers=headers) try: with opener.open(req, timeout=timeout) as response: raw = response.read(2_000_000) return response.status, json.loads(raw) if raw else {} except urllib.error.HTTPError as error: try: payload = json.loads(error.read(8192)) except ValueError: payload = {} return error.code, payload report = { "startedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), "baseUrl": base, "parameters": { "runCalls": args.run_calls, "maxEstimatedCny": args.max_estimated_cny, "maxTokens": args.max_tokens, "requestTimeoutSeconds": args.request_timeout, "onlyModels": args.only_models, "retryReport": str(args.retry_report) if args.retry_report else None, }, "retrySelection": { "requestedNonPassRouteIds": sorted(retry_route_ids), "matchedInventoryRouteIds": [], "eligibleRouteIds": [], "skippedRouteIds": [], }, "inventory": [], "results": [], "temporaryKeyDeleted": False, } def save(): args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") key_id = None try: status, _ = request("/api/auth/login", {"account": env["ADMIN_ACCOUNT"], "password": env["ADMIN_PASSWORD"]}) if status != 200: raise RuntimeError("Admin login failed: HTTP " + str(status)) status, catalog = request("/api/admin/models") if status != 200: raise RuntimeError("Catalog read failed") status, public = request("/api/models") if status != 200: raise RuntimeError("User catalog read failed") active_ids = {item["id"] for item in public["models"]} provider_names = {item["id"]: item["code"] for item in public["providers"]} candidates = [] for item in catalog["models"]: row = {field: item.get(field) for field in ("id", "logicalModelId", "modelKey", "publicName", "sourceLabel", "providerId", "upstreamModel", "type", "mode", "status", "maxConcurrency", "pricingConfigured", "billingReady", "executionReady")} row["inputTypes"] = normalize_types(item.get("inputTypes")) row["outputTypes"] = normalize_types(item.get("outputTypes")) row["provider"] = provider_names.get(item["providerId"], item["providerId"]) row["visibleRoute"] = item["id"] in active_ids row["pricing"] = item.get("pricing") reasons = chat_test_reasons(item, row["visibleRoute"]) row["notChatTestedReasons"] = reasons row["chatInputCompatible"] = not any(reason in reasons for reason in ("input_type_unknown", "input_type_not_text", "requires_modality_specific_test")) row["retrySelected"] = item["id"] in retry_route_ids report["inventory"].append(row) if item["id"] in retry_route_ids: report["retrySelection"]["matchedInventoryRouteIds"].append(item["id"]) selected_by_name = not args.only_models or item["id"] in args.only_models or item.get("upstreamModel") in args.only_models selected_by_retry = not args.retry_report or item["id"] in retry_route_ids if not reasons and selected_by_name and selected_by_retry: candidates.append(item) report["retrySelection"]["eligibleRouteIds"] = [item["id"] for item in candidates if item["id"] in retry_route_ids] report["retrySelection"]["skippedRouteIds"] = sorted(retry_route_ids - set(report["retrySelection"]["eligibleRouteIds"])) report["counts"] = {"adminRoutes": len(catalog["models"]), "userVisibleRoutes": len(active_ids), "logicalDefinitions": len(catalog["logicalModels"]), "chatCandidates": len(candidates)} report["channelCounts"] = dict(collections.Counter(row["provider"] for row in report["inventory"])) print(json.dumps(report["counts"]), flush=True) save() if not args.run_calls: return estimated = estimate_reservation(candidates, args.max_tokens) report["estimatedReservationCny"] = estimated if estimated > args.max_estimated_cny: raise RuntimeError("Estimated test reservation exceeds configured budget") name = "catalog-audit-" + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S") status, created = request("/api/keys", {"name": name, "modelIds": [m["id"] for m in candidates]}) if status != 201: raise RuntimeError("Temporary test key creation failed: HTTP " + str(status)) key_id = created["apiKey"]["id"] key = created["key"] report["temporaryKeyName"] = name status, models = request("/v1/models", key=key) report["externalModelListStatus"] = status if isinstance(models, dict): report["externalModelListCount"] = len(models.get("data", [])) else: report["externalModelListCount"] = None report["externalModelListError"] = "response_not_object" for index, item in enumerate(candidates): start = time.monotonic() result = {"id": item["id"], "model": item["upstreamModel"], "group": item["sourceLabel"], "provider": provider_names.get(item["providerId"], item["providerId"]), "request": {"endpoint": "/v1/chat/completions", "modelId": item["id"], "maxTokens": args.max_tokens, "stream": False, "timeoutSeconds": args.request_timeout}} try: status, payload = request("/v1/chat/completions", {"model": item["id"], "messages": [{"role": "user", "content": "Reply with OK."}], "max_tokens": args.max_tokens, "stream": False}, key=key) result["status"] = status if status == 200: if not isinstance(payload, dict): result["outcome"] = "invalid_payload" result["errorCode"] = "RESPONSE_NOT_OBJECT" else: choices = payload.get("choices") or [] message = choices[0].get("message", {}) if choices and isinstance(choices[0], dict) else {} content = message.get("content") result["contentType"] = type(content).__name__ if content is not None else None result["completionPresent"] = bool(content or message.get("tool_calls")) # Reasoning-only responses are valid upstream responses when # a reasoning model reaches the caller's output budget before # writing its final answer. Keep them distinct from a truly # empty response so the audit does not misdiagnose a budget # outcome as a transport or gateway failure. result["reasoningPresent"] = bool(message.get("reasoning_content") or message.get("reasoning")) result["finishReason"] = choices[0].get("finish_reason") if choices and isinstance(choices[0], dict) else None usage = payload.get("usage", {}) result["tokens"] = {k: usage.get(k) for k in ("prompt_tokens", "completion_tokens", "total_tokens")} if isinstance(usage, dict) else None result["outcome"] = "pass" if result["completionPresent"] else "reasoning_only" if result["reasoningPresent"] else "empty_completion" else: error = payload.get("error", {}) result["errorCode"] = str(error.get("code", "HTTP_" + str(status))) if isinstance(error, dict) else "HTTP_" + str(status) result["outcome"] = "failed" except Exception as error: result.update(outcome="transport_error", errorType=type(error).__name__) result["elapsedSeconds"] = round(time.monotonic() - start, 3) report["results"].append(result) save() print(json.dumps({"index": index + 1, "total": len(candidates), **result}, ensure_ascii=False), flush=True) # Prevent short upstream failures from flooding the gateway RPM limit. time.sleep(max(0, 1.1 - (time.monotonic() - start))) finally: if key_id: cleanup = cleanup_temporary_key(request, key_id) report["temporaryKeyDisabled"] = cleanup["disabled"] report["temporaryKeyDeleted"] = cleanup["deleted"] if cleanup["errors"]: report["cleanupErrors"] = cleanup["errors"] report["finishedAt"] = datetime.datetime.now(datetime.timezone.utc).isoformat() report.update(outcome_summary(report["results"])) save() try: request("/api/auth/logout", {}, method="POST") except Exception as error: report["logoutError"] = type(error).__name__ save() print(json.dumps({"outcomes": report["outcomes"], "temporaryKeyDeleted": report["temporaryKeyDeleted"]}), flush=True) if __name__ == "__main__": main()