fix: keep upstream token budgets billing-safe

This commit is contained in:
wlx committed 2026-09-28 08:39:04 +08:00
1 parent 4ec47cdb23
commit e0c926104e
4 files changed
+131 -44

No files matched your search

@@ -86,9 +86,7 @@ public class GatewayController {
String providerId=String.valueOf(model.get("providerId")); String providerId=String.valueOf(model.get("providerId"));
boolean mock="provider-mock".equals(providerId); boolean mock="provider-mock".equals(providerId);
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503); if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
Map<String,Object> normalizedInput = new LinkedHashMap<>(input); ChatMeter.Request request=ChatMeter.request(input,json);
OpenAiCompatibleProvider.prepareRequestForModel(String.valueOf(model.get("upstreamModel")), normalizedInput);
ChatMeter.Request request=ChatMeter.request(normalizedInput,json);
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false); guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
// Reserve both limits atomically. A queued request never consumes a // Reserve both limits atomically. A queued request never consumes a
// model slot while waiting for the API-Key slot (or vice versa). // model slot while waiting for the API-Key slot (or vice versa).
@@ -83,22 +83,6 @@ public class OpenAiCompatibleProvider {
return open(providerId,upstreamModel,input,true); return open(providerId,upstreamModel,input,true);
} }
/**
* Normalize the small set of provider-specific request constraints that
* are known before the request is reserved and billed. Claude manual
* thinking requires a larger output budget than the common 512-token
* playground default; changing it here keeps the billing reservation and
* the upstream request aligned.
*/
public static void prepareRequestForModel(String upstreamModel, Map<String, Object> body) {
if (upstreamModel == null || body == null || !upstreamModel.toLowerCase(java.util.Locale.ROOT).contains("claude")
|| !upstreamModel.toLowerCase(java.util.Locale.ROOT).contains("thinking")) return;
String field = body.containsKey("max_completion_tokens") ? "max_completion_tokens" : "max_tokens";
if (!body.containsKey(field)) return;
int current = numberValue(body.get(field), 0);
if (current > 0 && current < 2048) body.put(field, 2048);
}
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) { private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
ProviderConfig provider = resolve(providerId); ProviderConfig provider = resolve(providerId);
if (provider == null || !provider.hasKey()) { if (provider == null || !provider.hasKey()) {
@@ -124,10 +108,12 @@ public class OpenAiCompatibleProvider {
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
if (response.statusCode() < 200 || response.statusCode() >= 300) { if (response.statusCode() < 200 || response.statusCode() >= 300) {
String details = readErrorBody(response.body()); String details = readErrorBody(response.body());
if (requiresLargerOutputBudget(details)) {
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
}
// Some OpenAI-compatible relays use the newer spelling for // Some OpenAI-compatible relays use the newer spelling for
// reasoning-model budgets, and Claude manual thinking // reasoning-model budgets. Retry only this deterministic
// rejects small budgets. Retry only these deterministic // GPT request-shape error; other failures remain bill-safe.
// request-shape errors; other failures remain bill-safe.
if (attempt == 0 && prepareCompatibilityRetry(body, details)) continue; if (attempt == 0 && prepareCompatibilityRetry(body, details)) continue;
throw classifyUpstreamError(response.statusCode(), details); throw classifyUpstreamError(response.statusCode(), details);
} }
@@ -153,23 +139,20 @@ public class OpenAiCompatibleProvider {
/** Return true only for a deterministic request-shape rejection that is safe to retry. */ /** Return true only for a deterministic request-shape rejection that is safe to retry. */
static boolean prepareCompatibilityRetry(Map<String, Object> body, String details) { static boolean prepareCompatibilityRetry(Map<String, Object> body, String details) {
String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT); String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT);
if (text.matches(".*max_tokens.{0,80}max_completion_tokens.*") && body.containsKey("max_tokens")) { String model = String.valueOf(body.getOrDefault("model", "")).toLowerCase(java.util.Locale.ROOT);
if (model.startsWith("gpt-") && text.matches(".*max_tokens.{0,80}max_completion_tokens.*") && body.containsKey("max_tokens")) {
Object value = body.remove("max_tokens"); Object value = body.remove("max_tokens");
body.put("max_completion_tokens", value); body.put("max_completion_tokens", value);
return true; return true;
} }
String model = String.valueOf(body.getOrDefault("model", "")).toLowerCase(java.util.Locale.ROOT);
if (text.matches(".*max_tokens must be greater than 1024.*") && body.containsKey("max_tokens")
&& model.contains("claude") && model.contains("thinking")) {
int current = numberValue(body.get("max_tokens"), 0);
if (current < 2048) {
body.put("max_tokens", 2048);
return true;
}
}
return false; return false;
} }
static boolean requiresLargerOutputBudget(String details) {
String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT);
return text.matches(".*max_tokens.{0,80}(must be greater than|minimum is|at least).{0,20}1024.*");
}
private static int numberValue(Object value, int fallback) { private static int numberValue(Object value, int fallback) {
if (value instanceof Number number) return number.intValue(); if (value instanceof Number number) return number.intValue();
try { return Integer.parseInt(String.valueOf(value)); } catch (RuntimeException ignored) { return fallback; } try { return Integer.parseInt(String.valueOf(value)); } catch (RuntimeException ignored) { return fallback; }
@@ -0,0 +1,112 @@
package com.moyi.controlplane.provider;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.security.SecretCipher;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.mock.env.MockEnvironment;
class OpenAiCompatibleProviderHttpTest {
private final ObjectMapper json = new ObjectMapper();
private HttpServer server;
private OpenAiCompatibleProvider provider;
@BeforeEach
void setUp() throws Exception {
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
var environment = new MockEnvironment()
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
.withProperty("moyi.gateway.upstream-timeout-seconds", "5");
provider = new OpenAiCompatibleProvider(json, environment, org.mockito.Mockito.mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
}
@AfterEach
void tearDown() {
provider.close();
server.stop(0);
}
@Test
void retriesGptBudgetParameterWithoutChangingUserBudget() throws Exception {
var requests = new ArrayList<Map<String, Object>>();
server.createContext("/v1/chat/completions", exchange -> {
Map<String, Object> body = json.readValue(exchange.getRequestBody(), Map.class);
synchronized (requests) {
requests.add(body);
}
if (requests.size() == 1) {
write(exchange, 400, "{\"error\":{\"message\":\"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead.\"}}");
return;
}
write(exchange, 200, "{\"id\":\"ok\",\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}");
});
server.start();
var input = textInput(512);
var response = provider.chat("provider-deepseek", "gpt-5", input);
assertEquals("OK", ((Map<?, ?>) ((Map<?, ?>) ((List<?>) response.get("choices")).getFirst()).get("message")).get("content"));
assertEquals(2, requests.size());
assertEquals(512, requests.get(0).get("max_tokens"));
assertFalse(requests.get(0).containsKey("max_completion_tokens"));
assertEquals(512, requests.get(1).get("max_completion_tokens"));
assertFalse(requests.get(1).containsKey("max_tokens"));
assertEquals(512, input.get("max_tokens"));
}
@Test
void rejectsClaudeMinimumBudgetWithoutSecondCallOrBudgetRewrite() throws Exception {
var requestCount = new AtomicInteger();
server.createContext("/v1/chat/completions", exchange -> {
requestCount.incrementAndGet();
write(exchange, 400, "{\"error\":{\"message\":\"max_tokens must be greater than 1024 for manual Claude thinking\"}}");
});
server.start();
var input = textInput(512);
ApiException error = assertThrows(ApiException.class,
() -> provider.chat("provider-deepseek", "claude-sonnet-4-5-20250929-thinking", input));
assertEquals("INVALID_MAX_TOKENS", error.getCode());
assertEquals(400, error.getStatus());
assertTrue(error.getMessage().contains("大于 1024"));
assertEquals(1, requestCount.get());
assertEquals(512, input.get("max_tokens"));
}
private Map<String, Object> textInput(int maxTokens) {
var input = new LinkedHashMap<String, Object>();
input.put("messages", List.of(Map.of("role", "user", "content", "Reply with OK.")));
input.put("max_tokens", maxTokens);
input.put("stream", false);
return input;
}
private void write(HttpExchange exchange, int status, String body) throws IOException {
byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
exchange.getResponseHeaders().set("Content-Type", "application/json");
exchange.sendResponseHeaders(status, bytes.length);
try (var output = exchange.getResponseBody()) {
output.write(bytes);
}
}
}
@@ -75,7 +75,7 @@ class ProviderErrorClassificationTest {
@Test @Test
void retriesNewCompletionBudgetParameter() { void retriesNewCompletionBudgetParameter() {
var body = new LinkedHashMap<String, Object>(); var body = new LinkedHashMap<String, Object>();
body.put("model", "claude-sonnet-4-5-20250929-thinking"); body.put("model", "gpt-5");
body.put("max_tokens", 512); body.put("max_tokens", 512);
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body, assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead.")); "Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead."));
@@ -84,20 +84,14 @@ class ProviderErrorClassificationTest {
} }
@Test @Test
void raisesManualClaudeThinkingBudget() { void doesNotRewriteClaudeBudget() {
var body = new LinkedHashMap<String, Object>(); var body = new LinkedHashMap<String, Object>();
body.put("model", "claude-sonnet-4-5-20250929-thinking"); body.put("model", "claude-sonnet-4-5-20250929-thinking");
body.put("max_tokens", 512); body.put("max_tokens", 512);
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body, assertTrue(!OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
"max_tokens must be greater than 1024 for manual Claude thinking"));
assertEquals(512, body.get("max_tokens"));
assertTrue(OpenAiCompatibleProvider.requiresLargerOutputBudget(
"max_tokens must be greater than 1024 for manual Claude thinking")); "max_tokens must be greater than 1024 for manual Claude thinking"));
assertEquals(2048, body.get("max_tokens"));
}
@Test
void normalizesClaudeThinkingBudgetBeforeBilling() {
var body = new LinkedHashMap<String, Object>();
body.put("max_tokens", 512);
OpenAiCompatibleProvider.prepareRequestForModel("claude-sonnet-4-5-20250929-thinking", body);
assertEquals(2048, body.get("max_tokens"));
} }
} }