fix: keep upstream token budgets billing-safe
This commit is contained in:
1 parent
4ec47cdb23
commit
e0c926104e
4 files changed
+131
-44
No files matched your search
+1
-3
@@ -86,9 +86,7 @@ public class GatewayController {
|
|||||||
String providerId=String.valueOf(model.get("providerId"));
|
String providerId=String.valueOf(model.get("providerId"));
|
||||||
boolean mock="provider-mock".equals(providerId);
|
boolean mock="provider-mock".equals(providerId);
|
||||||
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||||
Map<String,Object> normalizedInput = new LinkedHashMap<>(input);
|
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||||
OpenAiCompatibleProvider.prepareRequestForModel(String.valueOf(model.get("upstreamModel")), normalizedInput);
|
|
||||||
ChatMeter.Request request=ChatMeter.request(normalizedInput,json);
|
|
||||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||||
// Reserve both limits atomically. A queued request never consumes a
|
// Reserve both limits atomically. A queued request never consumes a
|
||||||
// model slot while waiting for the API-Key slot (or vice versa).
|
// model slot while waiting for the API-Key slot (or vice versa).
|
||||||
|
|||||||
+12
-29
@@ -83,22 +83,6 @@ public class OpenAiCompatibleProvider {
|
|||||||
return open(providerId,upstreamModel,input,true);
|
return open(providerId,upstreamModel,input,true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Normalize the small set of provider-specific request constraints that
|
|
||||||
* are known before the request is reserved and billed. Claude manual
|
|
||||||
* thinking requires a larger output budget than the common 512-token
|
|
||||||
* playground default; changing it here keeps the billing reservation and
|
|
||||||
* the upstream request aligned.
|
|
||||||
*/
|
|
||||||
public static void prepareRequestForModel(String upstreamModel, Map<String, Object> body) {
|
|
||||||
if (upstreamModel == null || body == null || !upstreamModel.toLowerCase(java.util.Locale.ROOT).contains("claude")
|
|
||||||
|| !upstreamModel.toLowerCase(java.util.Locale.ROOT).contains("thinking")) return;
|
|
||||||
String field = body.containsKey("max_completion_tokens") ? "max_completion_tokens" : "max_tokens";
|
|
||||||
if (!body.containsKey(field)) return;
|
|
||||||
int current = numberValue(body.get(field), 0);
|
|
||||||
if (current > 0 && current < 2048) body.put(field, 2048);
|
|
||||||
}
|
|
||||||
|
|
||||||
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
|
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
|
||||||
ProviderConfig provider = resolve(providerId);
|
ProviderConfig provider = resolve(providerId);
|
||||||
if (provider == null || !provider.hasKey()) {
|
if (provider == null || !provider.hasKey()) {
|
||||||
@@ -124,10 +108,12 @@ public class OpenAiCompatibleProvider {
|
|||||||
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||||
String details = readErrorBody(response.body());
|
String details = readErrorBody(response.body());
|
||||||
|
if (requiresLargerOutputBudget(details)) {
|
||||||
|
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
|
||||||
|
}
|
||||||
// Some OpenAI-compatible relays use the newer spelling for
|
// Some OpenAI-compatible relays use the newer spelling for
|
||||||
// reasoning-model budgets, and Claude manual thinking
|
// reasoning-model budgets. Retry only this deterministic
|
||||||
// rejects small budgets. Retry only these deterministic
|
// GPT request-shape error; other failures remain bill-safe.
|
||||||
// request-shape errors; other failures remain bill-safe.
|
|
||||||
if (attempt == 0 && prepareCompatibilityRetry(body, details)) continue;
|
if (attempt == 0 && prepareCompatibilityRetry(body, details)) continue;
|
||||||
throw classifyUpstreamError(response.statusCode(), details);
|
throw classifyUpstreamError(response.statusCode(), details);
|
||||||
}
|
}
|
||||||
@@ -153,23 +139,20 @@ public class OpenAiCompatibleProvider {
|
|||||||
/** Return true only for a deterministic request-shape rejection that is safe to retry. */
|
/** Return true only for a deterministic request-shape rejection that is safe to retry. */
|
||||||
static boolean prepareCompatibilityRetry(Map<String, Object> body, String details) {
|
static boolean prepareCompatibilityRetry(Map<String, Object> body, String details) {
|
||||||
String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT);
|
String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT);
|
||||||
if (text.matches(".*max_tokens.{0,80}max_completion_tokens.*") && body.containsKey("max_tokens")) {
|
String model = String.valueOf(body.getOrDefault("model", "")).toLowerCase(java.util.Locale.ROOT);
|
||||||
|
if (model.startsWith("gpt-") && text.matches(".*max_tokens.{0,80}max_completion_tokens.*") && body.containsKey("max_tokens")) {
|
||||||
Object value = body.remove("max_tokens");
|
Object value = body.remove("max_tokens");
|
||||||
body.put("max_completion_tokens", value);
|
body.put("max_completion_tokens", value);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
String model = String.valueOf(body.getOrDefault("model", "")).toLowerCase(java.util.Locale.ROOT);
|
|
||||||
if (text.matches(".*max_tokens must be greater than 1024.*") && body.containsKey("max_tokens")
|
|
||||||
&& model.contains("claude") && model.contains("thinking")) {
|
|
||||||
int current = numberValue(body.get("max_tokens"), 0);
|
|
||||||
if (current < 2048) {
|
|
||||||
body.put("max_tokens", 2048);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static boolean requiresLargerOutputBudget(String details) {
|
||||||
|
String text = details == null ? "" : details.toLowerCase(java.util.Locale.ROOT);
|
||||||
|
return text.matches(".*max_tokens.{0,80}(must be greater than|minimum is|at least).{0,20}1024.*");
|
||||||
|
}
|
||||||
|
|
||||||
private static int numberValue(Object value, int fallback) {
|
private static int numberValue(Object value, int fallback) {
|
||||||
if (value instanceof Number number) return number.intValue();
|
if (value instanceof Number number) return number.intValue();
|
||||||
try { return Integer.parseInt(String.valueOf(value)); } catch (RuntimeException ignored) { return fallback; }
|
try { return Integer.parseInt(String.valueOf(value)); } catch (RuntimeException ignored) { return fallback; }
|
||||||
|
|||||||
+112
@@ -0,0 +1,112 @@
|
|||||||
|
package com.moyi.controlplane.provider;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.moyi.controlplane.common.ApiException;
|
||||||
|
import com.moyi.controlplane.security.SecretCipher;
|
||||||
|
import com.sun.net.httpserver.HttpExchange;
|
||||||
|
import com.sun.net.httpserver.HttpServer;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import org.junit.jupiter.api.AfterEach;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
import org.springframework.mock.env.MockEnvironment;
|
||||||
|
|
||||||
|
class OpenAiCompatibleProviderHttpTest {
|
||||||
|
private final ObjectMapper json = new ObjectMapper();
|
||||||
|
private HttpServer server;
|
||||||
|
private OpenAiCompatibleProvider provider;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() throws Exception {
|
||||||
|
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
|
var environment = new MockEnvironment()
|
||||||
|
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||||
|
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||||
|
.withProperty("moyi.gateway.upstream-timeout-seconds", "5");
|
||||||
|
provider = new OpenAiCompatibleProvider(json, environment, org.mockito.Mockito.mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterEach
|
||||||
|
void tearDown() {
|
||||||
|
provider.close();
|
||||||
|
server.stop(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void retriesGptBudgetParameterWithoutChangingUserBudget() throws Exception {
|
||||||
|
var requests = new ArrayList<Map<String, Object>>();
|
||||||
|
server.createContext("/v1/chat/completions", exchange -> {
|
||||||
|
Map<String, Object> body = json.readValue(exchange.getRequestBody(), Map.class);
|
||||||
|
synchronized (requests) {
|
||||||
|
requests.add(body);
|
||||||
|
}
|
||||||
|
if (requests.size() == 1) {
|
||||||
|
write(exchange, 400, "{\"error\":{\"message\":\"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead.\"}}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
write(exchange, 200, "{\"id\":\"ok\",\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}");
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
|
||||||
|
var input = textInput(512);
|
||||||
|
var response = provider.chat("provider-deepseek", "gpt-5", input);
|
||||||
|
|
||||||
|
assertEquals("OK", ((Map<?, ?>) ((Map<?, ?>) ((List<?>) response.get("choices")).getFirst()).get("message")).get("content"));
|
||||||
|
assertEquals(2, requests.size());
|
||||||
|
assertEquals(512, requests.get(0).get("max_tokens"));
|
||||||
|
assertFalse(requests.get(0).containsKey("max_completion_tokens"));
|
||||||
|
assertEquals(512, requests.get(1).get("max_completion_tokens"));
|
||||||
|
assertFalse(requests.get(1).containsKey("max_tokens"));
|
||||||
|
assertEquals(512, input.get("max_tokens"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void rejectsClaudeMinimumBudgetWithoutSecondCallOrBudgetRewrite() throws Exception {
|
||||||
|
var requestCount = new AtomicInteger();
|
||||||
|
server.createContext("/v1/chat/completions", exchange -> {
|
||||||
|
requestCount.incrementAndGet();
|
||||||
|
write(exchange, 400, "{\"error\":{\"message\":\"max_tokens must be greater than 1024 for manual Claude thinking\"}}");
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
|
||||||
|
var input = textInput(512);
|
||||||
|
ApiException error = assertThrows(ApiException.class,
|
||||||
|
() -> provider.chat("provider-deepseek", "claude-sonnet-4-5-20250929-thinking", input));
|
||||||
|
|
||||||
|
assertEquals("INVALID_MAX_TOKENS", error.getCode());
|
||||||
|
assertEquals(400, error.getStatus());
|
||||||
|
assertTrue(error.getMessage().contains("大于 1024"));
|
||||||
|
assertEquals(1, requestCount.get());
|
||||||
|
assertEquals(512, input.get("max_tokens"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> textInput(int maxTokens) {
|
||||||
|
var input = new LinkedHashMap<String, Object>();
|
||||||
|
input.put("messages", List.of(Map.of("role", "user", "content", "Reply with OK.")));
|
||||||
|
input.put("max_tokens", maxTokens);
|
||||||
|
input.put("stream", false);
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void write(HttpExchange exchange, int status, String body) throws IOException {
|
||||||
|
byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
|
||||||
|
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||||
|
exchange.sendResponseHeaders(status, bytes.length);
|
||||||
|
try (var output = exchange.getResponseBody()) {
|
||||||
|
output.write(bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+6
-12
@@ -75,7 +75,7 @@ class ProviderErrorClassificationTest {
|
|||||||
@Test
|
@Test
|
||||||
void retriesNewCompletionBudgetParameter() {
|
void retriesNewCompletionBudgetParameter() {
|
||||||
var body = new LinkedHashMap<String, Object>();
|
var body = new LinkedHashMap<String, Object>();
|
||||||
body.put("model", "claude-sonnet-4-5-20250929-thinking");
|
body.put("model", "gpt-5");
|
||||||
body.put("max_tokens", 512);
|
body.put("max_tokens", 512);
|
||||||
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||||
"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead."));
|
"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead."));
|
||||||
@@ -84,20 +84,14 @@ class ProviderErrorClassificationTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void raisesManualClaudeThinkingBudget() {
|
void doesNotRewriteClaudeBudget() {
|
||||||
var body = new LinkedHashMap<String, Object>();
|
var body = new LinkedHashMap<String, Object>();
|
||||||
body.put("model", "claude-sonnet-4-5-20250929-thinking");
|
body.put("model", "claude-sonnet-4-5-20250929-thinking");
|
||||||
body.put("max_tokens", 512);
|
body.put("max_tokens", 512);
|
||||||
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
assertTrue(!OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||||
|
"max_tokens must be greater than 1024 for manual Claude thinking"));
|
||||||
|
assertEquals(512, body.get("max_tokens"));
|
||||||
|
assertTrue(OpenAiCompatibleProvider.requiresLargerOutputBudget(
|
||||||
"max_tokens must be greater than 1024 for manual Claude thinking"));
|
"max_tokens must be greater than 1024 for manual Claude thinking"));
|
||||||
assertEquals(2048, body.get("max_tokens"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void normalizesClaudeThinkingBudgetBeforeBilling() {
|
|
||||||
var body = new LinkedHashMap<String, Object>();
|
|
||||||
body.put("max_tokens", 512);
|
|
||||||
OpenAiCompatibleProvider.prepareRequestForModel("claude-sonnet-4-5-20250929-thinking", body);
|
|
||||||
assertEquals(2048, body.get("max_tokens"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user