fix: classify upstream failures and prevent empty billed responses
This commit is contained in:
1 parent
b455d8d79f
commit
9acc69d4c7
28 files changed
+1449
-131
No files matched your search
+77
@@ -0,0 +1,77 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
|
||||
class AsyncProviderClientTest {
|
||||
@Test
|
||||
void upstreamErrorDoesNotExposeProviderMessage() throws Exception {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/", exchange -> {
|
||||
byte[] body = "{\"code\":1001,\"message\":\"secret account and API key details\"}".getBytes(StandardCharsets.UTF_8);
|
||||
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||
exchange.sendResponseHeaders(200, body.length);
|
||||
try (var output = exchange.getResponseBody()) { output.write(body); }
|
||||
});
|
||||
server.start();
|
||||
try {
|
||||
String baseUrl = "http://127.0.0.1:" + server.getAddress().getPort() + "/tripo";
|
||||
AsyncProviderClient client = clientFor(baseUrl);
|
||||
|
||||
ApiException error = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
|
||||
assertEquals("UPSTREAM_PROVIDER_ERROR", error.getCode());
|
||||
assertEquals("上游 3D 任务创建失败", error.getMessage());
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "503,UPSTREAM_PROVIDER_ERROR"})
|
||||
void httpErrorsExposeOnlySafeCodes(int status, String code) throws Exception {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/", exchange -> {
|
||||
byte[] body = "private account details".getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(status, body.length);
|
||||
try (var output = exchange.getResponseBody()) { output.write(body); }
|
||||
});
|
||||
server.start();
|
||||
try {
|
||||
AsyncProviderClient client = clientFor("http://127.0.0.1:" + server.getAddress().getPort() + "/tripo");
|
||||
ApiException submit = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
|
||||
assertEquals(code, submit.getCode());
|
||||
assertEquals("上游 3D 任务创建失败(HTTP " + status + ")", submit.getMessage());
|
||||
ApiException poll = assertThrows(ApiException.class, () -> client.poll("provider-vast", "task-test"));
|
||||
assertEquals(code, poll.getCode());
|
||||
assertEquals("上游异步任务查询失败(HTTP " + status + ")", poll.getMessage());
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
private AsyncProviderClient clientFor(String baseUrl) {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
SecretCipher cipher = new SecretCipher("test-encryption");
|
||||
when(jdbc.queryForMap(any(String.class), eq("provider-vast"))).thenReturn(Map.of(
|
||||
"code", "vast-tripo", "base_url", baseUrl, "credential_ciphertext", cipher.encrypt("test-key")
|
||||
));
|
||||
return new AsyncProviderClient(jdbc, cipher, new ObjectMapper(), mock(Environment.class));
|
||||
}
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.catalog.CatalogService;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.gateway.GatewayGuard;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import java.math.BigDecimal;
|
||||
import java.time.Duration;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+")
|
||||
class AsyncTaskPostgresTest {
|
||||
private final ObjectMapper json = new ObjectMapper().findAndRegisterModules();
|
||||
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID();
|
||||
private final String taskId = "async-transaction-test";
|
||||
private String schema;
|
||||
private DriverManagerDataSource base;
|
||||
private JdbcTemplate jdbc;
|
||||
private UsageAuditService usage;
|
||||
private AsyncProviderClient provider;
|
||||
private AsyncTaskService tasks;
|
||||
|
||||
@BeforeEach void setup() {
|
||||
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
|
||||
schema = "async_task_test_" + UUID.randomUUID().toString().replace("-", "");
|
||||
base = new DriverManagerDataSource(url, "moyi_role_test", "");
|
||||
var source = new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", "");
|
||||
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).target("25").load().migrate();
|
||||
jdbc = new JdbcTemplate(source);
|
||||
usage = spy(new UsageAuditService(jdbc, json));
|
||||
provider = mock(AsyncProviderClient.class);
|
||||
tasks = new AsyncTaskService(jdbc, json, mock(CatalogService.class), 100, 4, 4, 120_000,
|
||||
mock(GatewayGuard.class), usage, provider, new DataSourceTransactionManager(source));
|
||||
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'async-test','username','not-a-password','Test')", userId);
|
||||
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-prefix',?)", keyId, userId, "b".repeat(64));
|
||||
jdbc.update("INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)", userId);
|
||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
|
||||
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
|
||||
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() {
|
||||
if (base != null && schema != null) new JdbcTemplate(base).execute("DROP SCHEMA IF EXISTS " + schema + " CASCADE");
|
||||
}
|
||||
|
||||
private void fail() {
|
||||
ReflectionTestUtils.invokeMethod(tasks,"failTask",taskId,"PROVIDER_NOT_CONFIGURED","异步渠道尚未配置 API Key");
|
||||
}
|
||||
private void poll() {
|
||||
Map<String,Object> row = jdbc.queryForMap("SELECT id,provider_task_id,provider_deadline_at,created_at FROM model_tasks WHERE id=?", taskId);
|
||||
row.put("provider_id", "provider-vast");
|
||||
ReflectionTestUtils.invokeMethod(tasks,"pollProviderTask",row);
|
||||
}
|
||||
private BigDecimal wallet(String column) {
|
||||
return jdbc.queryForObject("SELECT " + column + " FROM user_wallets WHERE user_id=?",BigDecimal.class,userId);
|
||||
}
|
||||
private String status() {
|
||||
return jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?",String.class,taskId);
|
||||
}
|
||||
private BigDecimal cost() {
|
||||
return jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?",BigDecimal.class,taskId);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
|
||||
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
|
||||
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId);
|
||||
when(provider.submit(anyString(),anyString(),anyString(),anyString())).thenThrow(new ApiException(code,"上游任务未接受",503));
|
||||
tasks.completeMockTasks();
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("failed",status());
|
||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||
assertEquals(0,wallet("reserved_balance").signum());
|
||||
assertEquals(0,cost().signum());
|
||||
assertEquals(0,jdbc.queryForObject("SELECT cost_amount FROM usage_events WHERE provider_task_id=?",BigDecimal.class,taskId).signum());
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId));
|
||||
verify(provider,times(1)).submit(anyString(),anyString(),anyString(),anyString());
|
||||
}
|
||||
|
||||
@Test void refundAndTaskStateRollBackTogetherWhenAuditFails() {
|
||||
doThrow(new IllegalStateException("simulated database write failure")).when(usage).updateTask(eq(taskId),eq("failed"),isNull(),anyString(),anyString());
|
||||
assertThrows(IllegalStateException.class,this::fail);
|
||||
assertEquals("running",status());
|
||||
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
|
||||
assertNull(cost());
|
||||
doCallRealMethod().when(usage).updateTask(eq(taskId),eq("failed"),isNull(),anyString(),anyString());
|
||||
fail(); fail();
|
||||
assertEquals("failed",status());
|
||||
assertEquals(0,wallet("reserved_balance").signum());
|
||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||
}
|
||||
|
||||
@Test void successfulSettlementRollsBackForRetryAndCannotBeRefundedByLateFailure() {
|
||||
when(provider.poll("provider-vast","upstream-task")).thenReturn(Map.of("data",Map.of("status","success","progress",100,"output",Map.of("model_url","https://example.test/model.glb"))));
|
||||
doThrow(new IllegalStateException("simulated database write failure")).when(usage).updateTask(eq(taskId),eq("succeeded"),anyMap(),isNull(),isNull());
|
||||
assertThrows(IllegalStateException.class,this::poll);
|
||||
assertEquals("running",status());
|
||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
|
||||
assertNull(cost());
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId));
|
||||
doCallRealMethod().when(usage).updateTask(eq(taskId),eq("succeeded"),anyMap(),isNull(),isNull());
|
||||
poll(); poll(); fail();
|
||||
assertEquals("succeeded",status());
|
||||
assertEquals(new BigDecimal("8.56000000"),wallet("balance"));
|
||||
assertEquals(0,wallet("reserved_balance").signum());
|
||||
assertEquals(new BigDecimal("1.44000000"),cost());
|
||||
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings={"UPSTREAM_PROVIDER_ERROR", "UPSTREAM_AUTH_FAILED", "UPSTREAM_RATE_LIMITED"})
|
||||
void pollingErrorsKeepTheHoldAndClearOnRecovery(String code) {
|
||||
when(provider.poll("provider-vast","upstream-task"))
|
||||
.thenThrow(new ApiException(code,"private upstream details",502))
|
||||
.thenReturn(Map.of("data",Map.of("status","running","progress",75)));
|
||||
poll();
|
||||
assertEquals("running",status());
|
||||
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
|
||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||
assertNull(cost());
|
||||
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
|
||||
assertTrue(error.contains("UPSTREAM_TASK_POLL_RETRY"));
|
||||
assertFalse(error.contains("private upstream details"));
|
||||
poll();
|
||||
assertEquals("running",status());
|
||||
assertEquals(75,jdbc.queryForObject("SELECT progress FROM model_tasks WHERE id=?",Integer.class,taskId));
|
||||
assertNull(jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId));
|
||||
assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId));
|
||||
}
|
||||
|
||||
@Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed() {
|
||||
jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId);
|
||||
poll(); poll();
|
||||
assertEquals("failed",status());
|
||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||
assertEquals(0,wallet("reserved_balance").signum());
|
||||
assertEquals(0,cost().signum());
|
||||
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
|
||||
assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT"));
|
||||
assertTrue(error.contains("执行结果未知"));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void confirmedUpstreamFailureReleasesHoldImmediately() {
|
||||
when(provider.poll("provider-vast","upstream-task")).thenReturn(Map.of("data",Map.of("status","failed")));
|
||||
poll();
|
||||
assertEquals("failed",status());
|
||||
assertEquals(0,wallet("reserved_balance").signum());
|
||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||
assertEquals(0,cost().signum());
|
||||
}
|
||||
|
||||
@Test void migrationBackfillsOldAcceptedTasksAndNewSubmissionsUseConfiguredDeadline() {
|
||||
OffsetDateTime createdAt = jdbc.queryForObject("SELECT created_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
|
||||
OffsetDateTime migratedDeadline = jdbc.queryForObject("SELECT provider_deadline_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
|
||||
assertEquals(Duration.ofHours(1),Duration.between(createdAt,migratedDeadline));
|
||||
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL,provider_deadline_at=NULL WHERE id=?",taskId);
|
||||
when(provider.submit(anyString(),anyString(),anyString(),anyString())).thenReturn(Map.of("data",Map.of("task_id","upstream-task")));
|
||||
tasks.completeMockTasks();
|
||||
OffsetDateTime submittedAt = jdbc.queryForObject("SELECT updated_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
|
||||
OffsetDateTime deadline = jdbc.queryForObject("SELECT provider_deadline_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId);
|
||||
assertEquals(Duration.ofMinutes(2),Duration.between(submittedAt,deadline));
|
||||
tasks.completeMockTasks();
|
||||
verify(provider,times(1)).submit(anyString(),anyString(),anyString(),anyString());
|
||||
verify(provider,never()).poll(anyString(),anyString());
|
||||
assertEquals("running",status());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user