fix: classify upstream failures and prevent empty billed responses
This commit is contained in:
1 parent
b455d8d79f
commit
9acc69d4c7
28 files changed
+1449
-131
No files matched your search
@@ -0,0 +1,65 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("live-catalog-audit.py")
|
||||
SPEC = importlib.util.spec_from_file_location("live_catalog_audit", SCRIPT)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class LiveCatalogAuditTests(unittest.TestCase):
|
||||
def test_estimate_uses_requested_max_tokens(self):
|
||||
route = {"pricing": {"input": 1, "output": 2}}
|
||||
self.assertAlmostEqual(MODULE.estimate_reservation([route], 16), (1024 + 32) / 1_000_000)
|
||||
self.assertAlmostEqual(MODULE.estimate_reservation([route], 128), (1024 + 256) / 1_000_000)
|
||||
self.assertEqual(MODULE.estimate_reservation([{"pricing": {"price": 1.44, "unit": "CNY / task"}}], 16), 1.44)
|
||||
|
||||
def test_input_type_check_excludes_ocr_from_text_probe(self):
|
||||
route = {
|
||||
"pricingConfigured": True, "billingReady": True, "executionReady": True,
|
||||
"mode": "sync", "type": "text", "inputTypes": ["image"],
|
||||
}
|
||||
reasons = MODULE.chat_test_reasons(route)
|
||||
self.assertIn("input_type_not_text", reasons)
|
||||
|
||||
def test_unknown_and_mixed_input_types_are_explicit(self):
|
||||
unknown = {"type": "text", "mode": "sync", "inputTypes": []}
|
||||
mixed = {"type": "text", "mode": "sync", "inputTypes": ["text", "image"]}
|
||||
self.assertIn("input_type_unknown", MODULE.chat_test_reasons(unknown))
|
||||
self.assertIn("requires_modality_specific_test", MODULE.chat_test_reasons(mixed))
|
||||
|
||||
def test_retry_report_selects_only_non_pass_route_ids(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "previous.json"
|
||||
path.write_text(json.dumps({"results": [
|
||||
{"id": "pass-route", "outcome": "pass"},
|
||||
{"id": "failed-route", "outcome": "failed"},
|
||||
{"id": "reasoning-route", "outcome": "reasoning_only"},
|
||||
]}), encoding="utf-8")
|
||||
self.assertEqual(MODULE.load_retry_route_ids(path), {"failed-route", "reasoning-route"})
|
||||
|
||||
def test_cleanup_attempts_delete_when_disable_fails(self):
|
||||
calls = []
|
||||
|
||||
def request(path, body=None, method=None):
|
||||
calls.append((path, method))
|
||||
if method == "PATCH":
|
||||
raise TimeoutError("simulated timeout")
|
||||
return 204, {}
|
||||
|
||||
result = MODULE.cleanup_temporary_key(request, "temporary")
|
||||
self.assertFalse(result["disabled"])
|
||||
self.assertTrue(result["deleted"])
|
||||
self.assertEqual(calls, [
|
||||
("/api/keys/temporary/status", "PATCH"),
|
||||
("/api/keys/temporary", "DELETE"),
|
||||
])
|
||||
self.assertEqual(result["errors"][0]["operation"], "disable")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user