fix: keep upstream token budgets billing-safe
This commit is contained in:
1 parent
4ec47cdb23
commit
e0c926104e
4 files changed
+131
-44
No files matched your search
+112
@@ -0,0 +1,112 @@
|
||||
package com.moyi.controlplane.provider;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.sun.net.httpserver.HttpExchange;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.mock.env.MockEnvironment;
|
||||
|
||||
class OpenAiCompatibleProviderHttpTest {
|
||||
private final ObjectMapper json = new ObjectMapper();
|
||||
private HttpServer server;
|
||||
private OpenAiCompatibleProvider provider;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
var environment = new MockEnvironment()
|
||||
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||
.withProperty("moyi.gateway.upstream-timeout-seconds", "5");
|
||||
provider = new OpenAiCompatibleProvider(json, environment, org.mockito.Mockito.mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
provider.close();
|
||||
server.stop(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void retriesGptBudgetParameterWithoutChangingUserBudget() throws Exception {
|
||||
var requests = new ArrayList<Map<String, Object>>();
|
||||
server.createContext("/v1/chat/completions", exchange -> {
|
||||
Map<String, Object> body = json.readValue(exchange.getRequestBody(), Map.class);
|
||||
synchronized (requests) {
|
||||
requests.add(body);
|
||||
}
|
||||
if (requests.size() == 1) {
|
||||
write(exchange, 400, "{\"error\":{\"message\":\"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead.\"}}");
|
||||
return;
|
||||
}
|
||||
write(exchange, 200, "{\"id\":\"ok\",\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}");
|
||||
});
|
||||
server.start();
|
||||
|
||||
var input = textInput(512);
|
||||
var response = provider.chat("provider-deepseek", "gpt-5", input);
|
||||
|
||||
assertEquals("OK", ((Map<?, ?>) ((Map<?, ?>) ((List<?>) response.get("choices")).getFirst()).get("message")).get("content"));
|
||||
assertEquals(2, requests.size());
|
||||
assertEquals(512, requests.get(0).get("max_tokens"));
|
||||
assertFalse(requests.get(0).containsKey("max_completion_tokens"));
|
||||
assertEquals(512, requests.get(1).get("max_completion_tokens"));
|
||||
assertFalse(requests.get(1).containsKey("max_tokens"));
|
||||
assertEquals(512, input.get("max_tokens"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsClaudeMinimumBudgetWithoutSecondCallOrBudgetRewrite() throws Exception {
|
||||
var requestCount = new AtomicInteger();
|
||||
server.createContext("/v1/chat/completions", exchange -> {
|
||||
requestCount.incrementAndGet();
|
||||
write(exchange, 400, "{\"error\":{\"message\":\"max_tokens must be greater than 1024 for manual Claude thinking\"}}");
|
||||
});
|
||||
server.start();
|
||||
|
||||
var input = textInput(512);
|
||||
ApiException error = assertThrows(ApiException.class,
|
||||
() -> provider.chat("provider-deepseek", "claude-sonnet-4-5-20250929-thinking", input));
|
||||
|
||||
assertEquals("INVALID_MAX_TOKENS", error.getCode());
|
||||
assertEquals(400, error.getStatus());
|
||||
assertTrue(error.getMessage().contains("大于 1024"));
|
||||
assertEquals(1, requestCount.get());
|
||||
assertEquals(512, input.get("max_tokens"));
|
||||
}
|
||||
|
||||
private Map<String, Object> textInput(int maxTokens) {
|
||||
var input = new LinkedHashMap<String, Object>();
|
||||
input.put("messages", List.of(Map.of("role", "user", "content", "Reply with OK.")));
|
||||
input.put("max_tokens", maxTokens);
|
||||
input.put("stream", false);
|
||||
return input;
|
||||
}
|
||||
|
||||
private void write(HttpExchange exchange, int status, String body) throws IOException {
|
||||
byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
|
||||
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||
exchange.sendResponseHeaders(status, bytes.length);
|
||||
try (var output = exchange.getResponseBody()) {
|
||||
output.write(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-12
@@ -75,7 +75,7 @@ class ProviderErrorClassificationTest {
|
||||
@Test
|
||||
void retriesNewCompletionBudgetParameter() {
|
||||
var body = new LinkedHashMap<String, Object>();
|
||||
body.put("model", "claude-sonnet-4-5-20250929-thinking");
|
||||
body.put("model", "gpt-5");
|
||||
body.put("max_tokens", 512);
|
||||
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||
"Unsupported parameter: max_tokens is not supported. Use max_completion_tokens instead."));
|
||||
@@ -84,20 +84,14 @@ class ProviderErrorClassificationTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void raisesManualClaudeThinkingBudget() {
|
||||
void doesNotRewriteClaudeBudget() {
|
||||
var body = new LinkedHashMap<String, Object>();
|
||||
body.put("model", "claude-sonnet-4-5-20250929-thinking");
|
||||
body.put("max_tokens", 512);
|
||||
assertTrue(OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||
assertTrue(!OpenAiCompatibleProvider.prepareCompatibilityRetry(body,
|
||||
"max_tokens must be greater than 1024 for manual Claude thinking"));
|
||||
assertEquals(512, body.get("max_tokens"));
|
||||
assertTrue(OpenAiCompatibleProvider.requiresLargerOutputBudget(
|
||||
"max_tokens must be greater than 1024 for manual Claude thinking"));
|
||||
assertEquals(2048, body.get("max_tokens"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesClaudeThinkingBudgetBeforeBilling() {
|
||||
var body = new LinkedHashMap<String, Object>();
|
||||
body.put("max_tokens", 512);
|
||||
OpenAiCompatibleProvider.prepareRequestForModel("claude-sonnet-4-5-20250929-thinking", body);
|
||||
assertEquals(2048, body.get("max_tokens"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user