fix: classify upstream failures and prevent empty billed responses

This commit is contained in:
wlx committed 2026-09-28 04:55:00 +08:00
1 parent b455d8d79f
commit 9acc69d4c7
28 files changed
+1449 -131

No files matched your search

@@ -68,7 +68,8 @@ class ChatStreamingTest {
// Leave the body open: the client hard deadline must terminate the read.
}); server.start();
long started=System.nanoTime();
assertThrows(Exception.class,()->{ try(var stream=provider.stream("provider-deepseek","alias",request)) { stream.events(chunk->{}); } });
var error=assertThrows(ApiException.class,()->{ try(var stream=provider.stream("provider-deepseek","alias",request)) { stream.events(chunk->{}); } });
assertEquals("UPSTREAM_PROVIDER_TIMEOUT",error.getCode());
assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)<4000);
}
@Test void upstreamHttpFailureNeverBecomesSuccessfulStream() {
@@ -84,6 +85,31 @@ class ChatStreamingTest {
assertThrows(ApiException.class,()->ChatMeter.tokens(Map.of("usage",Map.of("prompt_tokens",1.5,"completion_tokens",2))));
assertThrows(ApiException.class,()->ChatMeter.tokens(Map.of("usage",Map.of("prompt_tokens",1,"completion_tokens",2,"prompt_cache_hit_tokens",3))));
}
@Test void reasoningAliasAndContentPartsAreCollectedSeparately() {
var collector=new ChatStreamCollector("public-route",System.currentTimeMillis());
collector.accept(Map.of("choices",List.of(Map.of("delta",Map.of(
"content",List.of(Map.of("type","text","text","OK")),
"reasoning", "thinking"
)))));
collector.accept(Map.of("choices",List.of(Map.of("delta",Map.of(),"finish_reason","stop"))));
var message=(Map<?,?>)((Map<?,?>)((List<?>)collector.result().get("choices")).getFirst()).get("message");
assertEquals("OK",message.get("content"));
assertEquals("thinking",message.get("reasoning_content"));
}
@Test void finishedWhitespaceOnlyStreamIsRejectedBeforeSettlement() {
var collector=new ChatStreamCollector("public-route",System.currentTimeMillis());
collector.accept(Map.of("choices",List.of(Map.of("delta",Map.of("content"," \n")))));
collector.accept(Map.of("choices",List.of(Map.of("delta",Map.of(),"finish_reason","stop"))));
var error=assertThrows(ApiException.class,collector::result);
assertEquals("UPSTREAM_RESPONSE_INVALID",error.getCode());
}
@Test void completedStreamWithoutContentOrReasoningIsNotBillable() {
var collector=new ChatStreamCollector("public-route",System.currentTimeMillis());
collector.accept(Map.of("choices",List.of(Map.of("index",0,"delta",Map.of(),"finish_reason","stop"))));
assertEquals("UPSTREAM_RESPONSE_INVALID",assertThrows(ApiException.class,collector::result).getCode());
}
@Test void tokenBoundsAndUnsupportedMultimodalAreRejectedBeforeUpstream() {
assertThrows(ApiException.class,()->ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content",List.of(Map.of("type","image_url"))))),json));
assertThrows(ApiException.class,()->ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content","test")),"max_tokens",0),json));
@@ -20,6 +20,32 @@ import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
class GatewayDeploymentTest {
@Test
void capacitySnapshotReadsBothLiveAndWaitingBucketsAndDoesNotInventZeroOnFailure() {
StringRedisTemplate redis = mock(StringRedisTemplate.class);
doReturn(List.of(2L, 3L, 0L, 1L)).when(redis).execute(any(RedisScript.class), anyList());
GatewayGuard guard = new GatewayGuard(redis, 60, 300, 4, "moyi:test:");
var snapshot = guard.capacitySnapshot(List.of("model-a", "model-b"));
assertTrue(snapshot.available());
assertEquals(new GatewayGuard.Capacity(2, 3), snapshot.groups().get("model-a"));
assertEquals(new GatewayGuard.Capacity(0, 1), snapshot.groups().get("model-b"));
verify(redis).execute(any(RedisScript.class), eq(List.of(
"moyi:test:gateway:slots:model:model-a", "moyi:test:gateway:waiting:model:model-a",
"moyi:test:gateway:slots:model:model-b", "moyi:test:gateway:waiting:model:model-b")));
doThrow(new org.springframework.dao.DataAccessResourceFailureException("offline")).when(redis).execute(any(RedisScript.class), anyList());
assertFalse(guard.capacitySnapshot(List.of("model-a")).available());
assertTrue(guard.capacitySnapshot(List.of("model-a")).groups().isEmpty());
}
@Test
void queueTimeoutCleansItsWaitingEntry() {
StringRedisTemplate redis = mock(StringRedisTemplate.class);
doReturn(0L).when(redis).execute(any(RedisScript.class), anyList(), any(Object[].class));
GatewayGuard guard = new GatewayGuard(redis, 60, 300, 1, 1, 15, 10, "moyi:test:");
assertThrows(GatewayLimitException.class, () -> guard.acquire(Map.of("id", "model-a"), UUID.randomUUID(), 15));
verify(redis).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:waiting:model:model-a")), any(Object[].class));
}
@Test
void redisRateKeysAreEnvironmentScoped() {
StringRedisTemplate redis = mock(StringRedisTemplate.class);
@@ -108,6 +134,65 @@ class GatewayDeploymentTest {
assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock","messages",List.of(Map.of("role","user","content","test"))),null,new MockHttpServletResponse()).getStatusCode().value());
}
@Test
void syncResponsePreservesReasoningAndFinishReasonWhenContentIsEmpty() {
CatalogService catalog = mock(CatalogService.class);
GatewayGuard guard = mock(GatewayGuard.class);
OpenAiCompatibleProvider provider = mock(OpenAiCompatibleProvider.class);
ChatBillingService billing = mock(ChatBillingService.class);
String userId = UUID.randomUUID().toString();
Map<String, Object> key = Map.of("id", UUID.randomUUID().toString(), "userId", userId);
Map<String, Object> model = Map.of("id", "reasoning-route", "providerId", "provider-relay", "upstreamModel", "reasoning-model", "mode", "sync");
when(catalog.authenticateKey("test-key")).thenReturn(key);
when(catalog.activeModel("reasoning-route")).thenReturn(model);
when(catalog.keyCanInvoke(key, "reasoning-route")).thenReturn(true);
when(provider.isConfigured("provider-relay")).thenReturn(true);
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
"id", "upstream-response",
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", "", "reasoning_content", "private reasoning"), "finish_reason", "length")),
"usage", Map.of("prompt_tokens", 2, "completion_tokens", 16, "total_tokens", 18)
));
when(provider.chat(eq("provider-relay"), eq("reasoning-model"), anyMap())).thenReturn(upstreamResponse);
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
when(billing.settle(any(), anyMap(), anyInt(), isNull())).thenReturn(java.math.BigDecimal.ZERO);
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
var response = controller.chatCompletions("Bearer test-key", Map.of("model", "reasoning-route", "messages", List.of(Map.of("role", "user", "content", "hello"))), null, new MockHttpServletResponse());
assertEquals(200, response.getStatusCode().value());
Map<?, ?> choice = (Map<?, ?>)((List<?>)response.getBody().get("choices")).getFirst();
Map<?, ?> message = (Map<?, ?>)choice.get("message");
assertEquals("", message.get("content"));
assertEquals("private reasoning", message.get("reasoning_content"));
assertEquals("length", choice.get("finish_reason"));
}
@Test
void trulyEmptySyncResponseIsRejectedBeforeSettlement() {
CatalogService catalog = mock(CatalogService.class);
GatewayGuard guard = mock(GatewayGuard.class);
OpenAiCompatibleProvider provider = mock(OpenAiCompatibleProvider.class);
ChatBillingService billing = mock(ChatBillingService.class);
String userId = UUID.randomUUID().toString();
Map<String, Object> key = Map.of("id", UUID.randomUUID().toString(), "userId", userId);
Map<String, Object> model = Map.of("id", "empty-route", "providerId", "provider-relay", "upstreamModel", "empty-model", "mode", "sync");
when(catalog.authenticateKey("test-key")).thenReturn(key);
when(catalog.activeModel("empty-route")).thenReturn(model);
when(catalog.keyCanInvoke(key, "empty-route")).thenReturn(true);
when(provider.isConfigured("provider-relay")).thenReturn(true);
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
"id", "upstream-empty",
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", ""), "finish_reason", "stop"))
));
when(provider.chat(eq("provider-relay"), eq("empty-model"), anyMap())).thenReturn(upstreamResponse);
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "empty-route", "messages", List.of(Map.of("role", "user", "content", "hello"))), null, new MockHttpServletResponse()));
assertEquals("UPSTREAM_RESPONSE_INVALID", error.getCode());
verify(billing, never()).settle(any(), anyMap(), anyInt(), any());
verify(billing).fail(any(), eq("UPSTREAM_RESPONSE_INVALID"), anyString(), anyInt());
}
@Test
void unimplementedMediaProviderDoesNotCreateFakeTask() {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
@@ -26,4 +26,25 @@ class ProviderErrorClassificationTest {
assertEquals("UPSTREAM_RATE_LIMITED", error.getCode());
assertEquals(429, error.getStatus());
}
@Test
void classifiesMissingUpstreamRouteWithoutEchoingProviderDetails() {
var error = OpenAiCompatibleProvider.classifyUpstreamError(502, "No available channel for model gemini-3-flash");
assertEquals("UPSTREAM_MODEL_UNAVAILABLE", error.getCode());
assertEquals(502, error.getStatus());
assertEquals("上游渠道没有可用的模型路由", error.getMessage());
}
@Test
void classifiesErrorEnvelopeReturnedWithHttp200() {
var error = OpenAiCompatibleProvider.classifyUpstreamError(200, "{\"error\":{\"code\":\"invalid_api_key\",\"message\":\"unauthorized\"}}");
assertEquals("UPSTREAM_AUTH_FAILED", error.getCode());
}
@Test
void classifiesTimeoutResponsesWithoutExposingProviderBody() {
var error = OpenAiCompatibleProvider.classifyUpstreamError(504, "gateway timeout at upstream");
assertEquals("UPSTREAM_PROVIDER_TIMEOUT", error.getCode());
assertEquals(504, error.getStatus());
}
}
@@ -0,0 +1,77 @@
package com.moyi.controlplane.task;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.security.SecretCipher;
import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.springframework.core.env.Environment;
import org.springframework.jdbc.core.JdbcTemplate;
class AsyncProviderClientTest {
@Test
void upstreamErrorDoesNotExposeProviderMessage() throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/", exchange -> {
byte[] body = "{\"code\":1001,\"message\":\"secret account and API key details\"}".getBytes(StandardCharsets.UTF_8);
exchange.getResponseHeaders().set("Content-Type", "application/json");
exchange.sendResponseHeaders(200, body.length);
try (var output = exchange.getResponseBody()) { output.write(body); }
});
server.start();
try {
String baseUrl = "http://127.0.0.1:" + server.getAddress().getPort() + "/tripo";
AsyncProviderClient client = clientFor(baseUrl);
ApiException error = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
assertEquals("UPSTREAM_PROVIDER_ERROR", error.getCode());
assertEquals("上游 3D 任务创建失败", error.getMessage());
} finally {
server.stop(0);
}
}
@ParameterizedTest
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "503,UPSTREAM_PROVIDER_ERROR"})
void httpErrorsExposeOnlySafeCodes(int status, String code) throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/", exchange -> {
byte[] body = "private account details".getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(status, body.length);
try (var output = exchange.getResponseBody()) { output.write(body); }
});
server.start();
try {
AsyncProviderClient client = clientFor("http://127.0.0.1:" + server.getAddress().getPort() + "/tripo");
ApiException submit = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
assertEquals(code, submit.getCode());
assertEquals("上游 3D 任务创建失败(HTTP " + status + ")", submit.getMessage());
ApiException poll = assertThrows(ApiException.class, () -> client.poll("provider-vast", "task-test"));
assertEquals(code, poll.getCode());
assertEquals("上游异步任务查询失败(HTTP " + status + ")", poll.getMessage());
} finally {
server.stop(0);
}
}
private AsyncProviderClient clientFor(String baseUrl) {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
SecretCipher cipher = new SecretCipher("test-encryption");
when(jdbc.queryForMap(any(String.class), eq("provider-vast"))).thenReturn(Map.of(
"code", "vast-tripo", "base_url", baseUrl, "credential_ciphertext", cipher.encrypt("test-key")
));
return new AsyncProviderClient(jdbc, cipher, new ObjectMapper(), mock(Environment.class));
}
}
@@ -0,0 +1,186 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.catalog.CatalogService;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.UUID;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.test.util.ReflectionTestUtils;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
@EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+")
class AsyncTaskPostgresTest {
private final ObjectMapper json = new ObjectMapper().findAndRegisterModules();
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID();
private final String taskId = "async-transaction-test";
private String schema;
private DriverManagerDataSource base;
private JdbcTemplate jdbc;
private UsageAuditService usage;
private AsyncProviderClient provider;
private AsyncTaskService tasks;
@BeforeEach void setup() {
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
schema = "async_task_test_" + UUID.randomUUID().toString().replace("-", "");
base = new DriverManagerDataSource(url, "moyi_role_test", "");
var source = new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", "");
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).target("25").load().migrate();
jdbc = new JdbcTemplate(source);
usage = spy(new UsageAuditService(jdbc, json));
provider = mock(AsyncProviderClient.class);
tasks = new AsyncTaskService(jdbc, json, mock(CatalogService.class), 100, 4, 4, 120_000,
mock(GatewayGuard.class), usage, provider, new DataSourceTransactionManager(source));
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'async-test','username','not-a-password','Test')", userId);
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-prefix',?)", keyId, userId, "b".repeat(64));
jdbc.update("INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)", userId);
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
}
@AfterEach void cleanup() {
if (base != null && schema != null) new JdbcTemplate(base).execute("DROP SCHEMA IF EXISTS " + schema + " CASCADE");
}
private void fail() {
ReflectionTestUtils.invokeMethod(tasks,"failTask",taskId,"PROVIDER_NOT_CONFIGURED","异步渠道尚未配置 API Key");
}
private void poll() {
Map<String,Object> row = jdbc.queryForMap("SELECT id,provider_task_id,provider_deadline_at,created_at FROM model_tasks WHERE id=?", taskId);
row.put("provider_id", "provider-vast");
ReflectionTestUtils.invokeMethod(tasks,"pollProviderTask",row);
}
private BigDecimal wallet(String column) {
return jdbc.queryForObject("SELECT " + column + " FROM user_wallets WHERE user_id=?",BigDecimal.class,userId);
}
private String status() {
return jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?",String.class,taskId);
}
private BigDecimal cost() {
return jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?",BigDecimal.class,taskId);
}
@ParameterizedTest
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId);
when(provider.submit(anyString(),anyString(),anyString(),anyString())).thenThrow(new ApiException(code,"上游任务未接受",503));
tasks.completeMockTasks();
tasks.completeMockTasks();
assertEquals("failed",status());
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
assertEquals(0,wallet("reserved_balance").signum());
assertEquals(0,cost().signum());
assertEquals(0,jdbc.queryForObject("SELECT cost_amount FROM usage_events WHERE provider_task_id=?",BigDecimal.class,taskId).signum());
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId));
verify(provider,times(1)).submit(anyString(),anyString(),anyString(),anyString());
}
@Test void refundAndTaskStateRollBackTogetherWhenAuditFails() {
doThrow(new IllegalStateException("simulated database write failure")).when(usage).updateTask(eq(taskId),eq("failed"),isNull(),anyString(),anyString());
assertThrows(IllegalStateException.class,this::fail);
assertEquals("running",status());
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
assertNull(cost());
doCallRealMethod().when(usage).updateTask(eq(taskId),eq("failed"),isNull(),anyString(),anyString());
fail(); fail();
assertEquals("failed",status());
assertEquals(0,wallet("reserved_balance").signum());
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
}
@Test void successfulSettlementRollsBackForRetryAndCannotBeRefundedByLateFailure() {
when(provider.poll("provider-vast","upstream-task")).thenReturn(Map.of("data",Map.of("status","success","progress",100,"output",Map.of("model_url","https://example.test/model.glb"))));
doThrow(new IllegalStateException("simulated database write failure")).when(usage).updateTask(eq(taskId),eq("succeeded"),anyMap(),isNull(),isNull());
assertThrows(IllegalStateException.class,this::poll);
assertEquals("running",status());
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
assertNull(cost());
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId));
doCallRealMethod().when(usage).updateTask(eq(taskId),eq("succeeded"),anyMap(),isNull(),isNull());
poll(); poll(); fail();
assertEquals("succeeded",status());
assertEquals(new BigDecimal("8.56000000"),wallet("balance"));
assertEquals(0,wallet("reserved_balance").signum());
assertEquals(new BigDecimal("1.44000000"),cost());
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId));
}
@ParameterizedTest
@ValueSource(strings={"UPSTREAM_PROVIDER_ERROR", "UPSTREAM_AUTH_FAILED", "UPSTREAM_RATE_LIMITED"})
void pollingErrorsKeepTheHoldAndClearOnRecovery(String code) {
when(provider.poll("provider-vast","upstream-task"))
.thenThrow(new ApiException(code,"private upstream details",502))
.thenReturn(Map.of("data",Map.of("status","running","progress",75)));
poll();
assertEquals("running",status());
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
assertNull(cost());
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
assertTrue(error.contains("UPSTREAM_TASK_POLL_RETRY"));
assertFalse(error.contains("private upstream details"));
poll();
assertEquals("running",status());
assertEquals(75,jdbc.queryForObject("SELECT progress FROM model_tasks WHERE id=?",Integer.class,taskId));
assertNull(jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId));
assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId));
}
@Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed() {
jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId);
poll(); poll();
assertEquals("failed",status());
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
assertEquals(0,wallet("reserved_balance").signum());
assertEquals(0,cost().signum());
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT"));
assertTrue(error.contains("执行结果未知"));
verifyNoInteractions(provider);
}
@Test void confirmedUpstreamFailureReleasesHoldImmediately() {
when(provider.poll("provider-vast","upstream-task")).thenReturn(Map.of("data",Map.of("status","failed")));
poll();
assertEquals("failed",status());
assertEquals(0,wallet("reserved_balance").signum());
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
assertEquals(0,cost().signum());
}
@Test void migrationBackfillsOldAcceptedTasksAndNewSubmissionsUseConfiguredDeadline() {
OffsetDateTime createdAt = jdbc.queryForObject("SELECT created_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
OffsetDateTime migratedDeadline = jdbc.queryForObject("SELECT provider_deadline_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
assertEquals(Duration.ofHours(1),Duration.between(createdAt,migratedDeadline));
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL,provider_deadline_at=NULL WHERE id=?",taskId);
when(provider.submit(anyString(),anyString(),anyString(),anyString())).thenReturn(Map.of("data",Map.of("task_id","upstream-task")));
tasks.completeMockTasks();
OffsetDateTime submittedAt = jdbc.queryForObject("SELECT updated_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
OffsetDateTime deadline = jdbc.queryForObject("SELECT provider_deadline_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
assertEquals(Duration.ofMinutes(2),Duration.between(submittedAt,deadline));
tasks.completeMockTasks();
verify(provider,times(1)).submit(anyString(),anyString(),anyString(),anyString());
verify(provider,never()).poll(anyString(),anyString());
assertEquals("running",status());
}
}
@@ -0,0 +1,83 @@
package com.moyi.controlplane.usage;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.auth.AuthService;
import com.moyi.controlplane.gateway.GatewayGuard;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.*;
@EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+")
class UsageCapacityPostgresTest {
private JdbcTemplate jdbc;
private JdbcTemplate base;
private String schema;
@BeforeEach void setup() {
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
schema = "capacity_test_" + UUID.randomUUID().toString().replace("-", "");
var source = new DriverManagerDataSource(url, "moyi_role_test", "");
base = new JdbcTemplate(source);
Flyway.configure().dataSource(source).schemas(schema).defaultSchema(schema).load().migrate();
jdbc = new JdbcTemplate(new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", ""));
jdbc.update("UPDATE model_providers SET status='disabled'");
jdbc.update("INSERT INTO model_providers(id,code,name,protocol,status) VALUES ('capacity-provider','capacity-provider','Capacity provider','http','active')");
jdbc.update("INSERT INTO model_definitions(id,public_name,model_type,call_mode) VALUES ('capacity-model','Capacity model','text','sync')");
jdbc.update("INSERT INTO model_catalog(id,provider_id,logical_model_id,public_name,source_label,upstream_model,model_type,call_mode) SELECT 'capacity-route-'||n,'capacity-provider','capacity-model','Capacity model','Group '||n,'test','text','sync' FROM generate_series(1,105) n");
jdbc.update("UPDATE model_catalog SET call_mode='async',model_type='3d' WHERE id='capacity-route-105'");
UUID user = UUID.randomUUID(), key = UUID.randomUUID();
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'capacity-user','username','not-loginable','Capacity user')", user);
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Capacity test','local-test',?)", key, user, "c".repeat(64));
for (String state : List.of("running", "queued", "succeeded")) {
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,created_at) VALUES (?,?,?,'capacity-route-105','3d',?,now()-interval '1 day')", "task-" + state, user, key, state);
}
}
@AfterEach void cleanup() {
if (base != null && schema != null) base.execute("DROP SCHEMA " + schema + " CASCADE");
}
@Test void liveCapacitySeparatesQueuedWorkAndKeepsEveryGroupAndLongRunningTask() {
GatewayGuard guard = mock(GatewayGuard.class);
when(guard.defaultModelConcurrency()).thenReturn(7);
when(guard.defaultKeyConcurrency()).thenReturn(3);
when(guard.queueWaitMs()).thenReturn(4500L);
when(guard.capacitySnapshot(anyList())).thenAnswer(invocation -> {
List<String> ids = invocation.getArgument(0);
Map<String, GatewayGuard.Capacity> states = new LinkedHashMap<>();
ids.forEach(id -> states.put(id, new GatewayGuard.Capacity(id.equals("capacity-route-1") ? 2 : 0, id.equals("capacity-route-1") ? 3 : 0)));
return new GatewayGuard.CapacitySnapshot(true, states);
});
var service = new UsageAuditService(jdbc, new ObjectMapper(), guard, 77, 2500);
Map<String, Object> summary = service.adminSummary("test", mock(AuthService.class));
List<Map<String, Object>> groups = (List<Map<String, Object>>) summary.get("capacityGroups");
assertEquals(105, groups.size(), "Capacity listing must not truncate at 100 groups");
Map<String, Object> async = groups.stream().filter(group -> group.get("id").equals("capacity-route-105")).findFirst().orElseThrow();
assertEquals(1L, async.get("running"), "A task older than ten minutes must remain in live capacity");
assertEquals(1L, async.get("queued"), "Running tasks must not also count as queued");
assertEquals(7, async.get("limit"), "Null route limits use the real configured model default");
assertEquals(3L, summary.get("running"));
assertEquals(4L, summary.get("waiting"));
Map<?, ?> config = (Map<?, ?>) summary.get("capacityConfig");
assertEquals(4500L, config.get("queueWaitMs"));
assertEquals(3, config.get("defaultKeyConcurrency"));
assertEquals(77, config.get("asyncQueueLimit"));
assertEquals(2500L, config.get("asyncWorkerIntervalMs"));
when(guard.capacitySnapshot(anyList())).thenReturn(new GatewayGuard.CapacitySnapshot(false, Map.of()));
var unavailable = service.adminSummary("test", mock(AuthService.class));
assertNull(unavailable.get("running"), "Unavailable Redis must not be reported as zero load");
assertNull(unavailable.get("waiting"));
}
}