fix: keep upstream token budgets billing-safe
This commit is contained in:
1 parent
4ec47cdb23
commit
e0c926104e
4 files changed
+131
-44
No files matched your search
+1
-3
@@ -86,9 +86,7 @@ public class GatewayController {
|
||||
String providerId=String.valueOf(model.get("providerId"));
|
||||
boolean mock="provider-mock".equals(providerId);
|
||||
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||
Map<String,Object> normalizedInput = new LinkedHashMap<>(input);
|
||||
OpenAiCompatibleProvider.prepareRequestForModel(String.valueOf(model.get("upstreamModel")), normalizedInput);
|
||||
ChatMeter.Request request=ChatMeter.request(normalizedInput,json);
|
||||
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||
// Reserve both limits atomically. A queued request never consumes a
|
||||
// model slot while waiting for the API-Key slot (or vice versa).
|
||||
|
||||
+12
-29
@@ -83,22 +83,6 @@ public class OpenAiCompatibleProvider {
|
||||
return open(providerId,upstreamModel,input,true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize the small set of provider-specific request constraints that
|
||||
* are known before the request is reserved and billed. Claude manual
|
||||
* thinking requires a larger output budget than the common 512-token
|
||||
* playground default; changing it here keeps the billing reservation and
|
||||
* the upstream request aligned.
|
||||
*/
|
||||
public static void prepareRequestForModel(String upstreamModel, Map<String, Object> body) {
|
||||
if (upstreamModel == null || body == null || !upstreamModel.toLowerCase(java.util.Locale.ROOT).contains("claude")
|
||||
|| !upstreamModel.toLowerCase(java.util.Locale.ROOT).contains("thinking")) return;
|
||||
String field = body.containsKey("max_completion_tokens") ? "max_completion_tokens" : "max_tokens";
|
||||
if (!body.containsKey(field)) return;
|
||||
int current = numberValue(body.get(field), 0);
|
||||
if (current > 0 && current < 2048) body.put(field, 2048);
|
||||
}
|
||||
|
||||
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
|
||||
ProviderConfig provider = resolve(providerId);
|
||||
if (provider == null || !provider.hasKey()) {
|
||||
@@ -124,10 +108,12 @@ public class OpenAiCompatibleProvider {
|
||||
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String details = readErrorBody(response.body());
|
||||
if (requiresLargerOutputBudget(details)) {
|
||||
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
|
||||
}
|
||||
// Some OpenAI-compatible relays use the newer spelling for
|
||||
// reasoning-model budgets, and Claude manual thinking
|
||||
// rejects small budgets. Retry only these deterministic
|
||||
// request-shape errors; other failures remain bill-safe.
|
||||
// reasoning-model budgets. Retry only this deterministic
|
||||
// GPT request-shape error; other failures remain bill-safe.
|
||||
if (attempt == 0 && prepareCompatibilityRetry(body, details)) continue;
|
||||
throw classifyUpstreamError(response.statusCode(), details);
|
||||
}
|
||||
@@ -153,23 +139,20 @@ public class OpenAiCompatibleProvider {
|
||||
/** Return true only for a deterministic request-shape rejection that is safe to retry. */
|
||||
static boolean prepareCompatibilityRetry(Map<String, Object> body, String details) {
|
||||
String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT);
|
||||
if (text.matches(".*max_tokens.{0,80}max_completion_tokens.*") && body.containsKey("max_tokens")) {
|
||||
String model = String.valueOf(body.getOrDefault("model", "")).toLowerCase(java.util.Locale.ROOT);
|
||||
if (model.startsWith("gpt-") && text.matches(".*max_tokens.{0,80}max_completion_tokens.*") && body.containsKey("max_tokens")) {
|
||||
Object value = body.remove("max_tokens");
|
||||
body.put("max_completion_tokens", value);
|
||||
return true;
|
||||
}
|
||||
String model = String.valueOf(body.getOrDefault("model", "")).toLowerCase(java.util.Locale.ROOT);
|
||||
if (text.matches(".*max_tokens must be greater than 1024.*") && body.containsKey("max_tokens")
|
||||
&& model.contains("claude") && model.contains("thinking")) {
|
||||
int current = numberValue(body.get("max_tokens"), 0);
|
||||
if (current < 2048) {
|
||||
body.put("max_tokens", 2048);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static boolean requiresLargerOutputBudget(String details) {
|
||||
String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT);
|
||||
return text.matches(".*max_tokens.{0,80}(must be greater than|minimum is|at least).{0,20}1024.*");
|
||||
}
|
||||
|
||||
private static int numberValue(Object value, int fallback) {
|
||||
if (value instanceof Number number) return number.intValue();
|
||||
try { return Integer.parseInt(String.valueOf(value)); } catch (RuntimeException ignored) { return fallback; }
|
||||
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package com.moyi.controlplane.provider;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.sun.net.httpserver.HttpExchange;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.mock.env.MockEnvironment;
|
||||
|
||||
class OpenAiCompatibleProviderHttpTest {
|
||||
private final ObjectMapper json = new ObjectMapper();
|
||||
private HttpServer server;
|
||||
private OpenAiCompatibleProvider provider;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
var environment = new MockEnvironment()
|
||||
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||
.withProperty("moyi.gateway.upstream-timeout-seconds", "5");
|
||||
provider = new OpenAiCompatibleProvider(json, environment, org.mockito.Mockito.mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
provider.close();
|
||||
server.stop(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void retriesGptBudgetParameterWithoutChangingUserBudget() throws Exception {
|
||||
var requests = new ArrayList<Map<String, Object>>();
|
||||
server.createContext("/v1/chat/completions", exchange -> {
|
||||
Map<String, Object> body = json.readValue(exchange.getRequestBody(), Map.class);
|
||||
synchronized (requests) {
|
||||
requests.add(body);
|
||||
}
|
||||
if (requests.size() == 1) {
|
||||
write(exchange, 400, "{\"error\":{\"message\":\"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead.\"}}");
|
||||
return;
|
||||
}
|
||||
write(exchange, 200, "{\"id\":\"ok\",\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}");
|
||||
});
|
||||
server.start();
|
||||
|
||||
var input = textInput(512);
|
||||
var response = provider.chat("provider-deepseek", "gpt-5", input);
|
||||
|
||||
assertEquals("OK", ((Map<?, ?>) ((Map<?, ?>) ((List<?>) response.get("choices")).getFirst()).get("message")).get("content"));
|
||||
assertEquals(2, requests.size());
|
||||
assertEquals(512, requests.get(0).get("max_tokens"));
|
||||
assertFalse(requests.get(0).containsKey("max_completion_tokens"));
|
||||
assertEquals(512, requests.get(1).get("max_completion_tokens"));
|
||||
assertFalse(requests.get(1).containsKey("max_tokens"));
|
||||
assertEquals(512, input.get("max_tokens"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsClaudeMinimumBudgetWithoutSecondCallOrBudgetRewrite() throws Exception {
|
||||
var requestCount = new AtomicInteger();
|
||||
server.createContext("/v1/chat/completions", exchange -> {
|
||||
requestCount.incrementAndGet();
|
||||
write(exchange, 400, "{\"error\":{\"message\":\"max_tokens must be greater than 1024 for manual Claude thinking\"}}");
|
||||
});
|
||||
server.start();
|
||||
|
||||
var input = textInput(512);
|
||||
ApiException error = assertThrows(ApiException.class,
|
||||
() -> provider.chat("provider-deepseek", "claude-sonnet-4-5-20250929-thinking", input));
|
||||
|
||||
assertEquals("INVALID_MAX_TOKENS", error.getCode());
|
||||
assertEquals(400, error.getStatus());
|
||||
assertTrue(error.getMessage().contains("大于 1024"));
|
||||
assertEquals(1, requestCount.get());
|
||||
assertEquals(512, input.get("max_tokens"));
|
||||
}
|
||||
|
||||
private Map<String, Object> textInput(int maxTokens) {
|
||||
var input = new LinkedHashMap<String, Object>();
|
||||
input.put("messages", List.of(Map.of("role", "user", "content", "Reply with OK.")));
|
||||
input.put("max_tokens", maxTokens);
|
||||
input.put("stream", false);
|
||||
return input;
|
||||
}
|
||||
|
||||
private void write(HttpExchange exchange, int status, String body) throws IOException {
|
||||
byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
|
||||
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||
exchange.sendResponseHeaders(status, bytes.length);
|
||||
try (var output = exchange.getResponseBody()) {
|
||||
output.write(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-12
@@ -75,7 +75,7 @@ class ProviderErrorClassificationTest {
|
||||
@Test
|
||||
void retriesNewCompletionBudgetParameter() {
|
||||
var body = new LinkedHashMap<String, Object>();
|
||||
body.put("model", "claude-sonnet-4-5-20250929-thinking");
|
||||
body.put("model", "gpt-5");
|
||||
body.put("max_tokens", 512);
|
||||
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||
"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead."));
|
||||
@@ -84,20 +84,14 @@ class ProviderErrorClassificationTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void raisesManualClaudeThinkingBudget() {
|
||||
void doesNotRewriteClaudeBudget() {
|
||||
var body = new LinkedHashMap<String, Object>();
|
||||
body.put("model", "claude-sonnet-4-5-20250929-thinking");
|
||||
body.put("max_tokens", 512);
|
||||
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||
assertTrue(!OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||
"max_tokens must be greater than 1024 for manual Claude thinking"));
|
||||
assertEquals(512, body.get("max_tokens"));
|
||||
assertTrue(OpenAiCompatibleProvider.requiresLargerOutputBudget(
|
||||
"max_tokens must be greater than 1024 for manual Claude thinking"));
|
||||
assertEquals(2048, body.get("max_tokens"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesClaudeThinkingBudgetBeforeBilling() {
|
||||
var body = new LinkedHashMap<String, Object>();
|
||||
body.put("max_tokens", 512);
|
||||
OpenAiCompatibleProvider.prepareRequestForModel("claude-sonnet-4-5-20250929-thinking", body);
|
||||
assertEquals(2048, body.get("max_tokens"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user