fix: retain async holds for unknown provider outcomes

This commit is contained in:
xpp committed 2026-10-21 18:57:35 +08:00
1 parent 76265845dc
commit 5be06a97c1
18 files changed
+574 -108

No files matched your search

@@ -11,6 +11,7 @@ import static org.mockito.Mockito.when;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.security.SecretCipher;
import com.moyi.controlplane.provider.transport.ProviderRequestException;
import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
@@ -207,7 +208,7 @@ class AsyncProviderClientTest {
}
@ParameterizedTest
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "503,UPSTREAM_PROVIDER_ERROR"})
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "408,UPSTREAM_PROVIDER_ERROR", "409,UPSTREAM_PROVIDER_ERROR", "503,UPSTREAM_PROVIDER_ERROR"})
void httpErrorsExposeOnlySafeCodes(int status, String code) throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/", exchange -> {
@@ -221,6 +222,7 @@ class AsyncProviderClientTest {
ApiException submit = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
assertEquals(code, submit.getCode());
assertEquals("上游 3D 任务创建失败(HTTP " + status + ")", submit.getMessage());
assertEquals(status != 408 && status != 409 && status < 500, ((ProviderRequestException) submit).definitelyNotAccepted());
ApiException poll = assertThrows(ApiException.class, () -> client.poll("provider-vast", "task-test"));
assertEquals(code, poll.getCode());
assertEquals("上游异步任务查询失败(HTTP " + status + ")", poll.getMessage());
@@ -229,6 +231,71 @@ class AsyncProviderClientTest {
}
}
@Test void successfulHttpResponseWithMalformedBodyHasUnknownAcceptance() throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/", exchange -> {
byte[] body = "not-json; private upstream detail".getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(200, body.length);
try (var output = exchange.getResponseBody()) { output.write(body); }
});
server.start();
try {
var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort());
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair"));
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
org.junit.jupiter.api.Assertions.assertFalse(failure.getMessage().contains("private upstream detail"));
} finally { server.stop(0); }
}
@ParameterizedTest
@CsvSource({"image,gpt-image-2", "video,sora-2", "audio,tts-1"})
void compatibleSubmit5xxRemainsUnknownAndNeverUsesFallbackRoute(String type, String model) throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
var calls = new java.util.concurrent.atomic.AtomicInteger();
server.createContext("/", exchange -> {
calls.incrementAndGet();
exchange.getRequestBody().readAllBytes();
byte[] body = "private account details".getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(503, body.length);
try (var output = exchange.getResponseBody()) { output.write(body); }
});
server.start();
try {
var jdbc = mock(JdbcTemplate.class);
var cipher = new SecretCipher("test-encryption");
when(jdbc.queryForMap(any(String.class), eq("relay"))).thenReturn(Map.of(
"code", "relay", "protocol", "openai-compatible", "base_url", "http://127.0.0.1:" + server.getAddress().getPort(), "credential_ciphertext", cipher.encrypt("test-key")));
var client = new AsyncProviderClient(jdbc, cipher, new ObjectMapper(), mock(Environment.class));
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("relay", model, type, "chair"));
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
assertEquals(1, calls.get());
} finally { server.stop(0); }
}
@Test void acceptedPostWithDroppedResponseIsUnknownAndIsNotTransparentlyRepeated() throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
var calls = new java.util.concurrent.atomic.AtomicInteger();
server.createContext("/", exchange -> {
exchange.getRequestBody().readAllBytes();
calls.incrementAndGet();
exchange.close(); // Simulate acceptance followed by loss of the HTTP response.
});
server.start();
try {
var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort());
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair"));
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
assertEquals(1, calls.get());
} finally { server.stop(0); }
}
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
var client = clientFor("http://127.0.0.1:1");
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
assertEquals(ProviderRequestException.Acceptance.NOT_SENT, failure.acceptance());
assertEquals("INVALID_MEDIA_URL", failure.getCode());
}
private AsyncProviderClient clientFor(String baseUrl) {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
SecretCipher cipher = new SecretCipher("test-encryption");
@@ -15,6 +15,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
import com.moyi.controlplane.provider.transport.ProviderRequestException;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -42,6 +43,7 @@ class AsyncTaskPostgresTest {
private AsyncProviderClient provider;
private AsyncTaskService tasks;
private CatalogService catalog;
private GatewayGuard guard;
@BeforeEach void setup() {
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
@@ -52,7 +54,7 @@ class AsyncTaskPostgresTest {
jdbc = new JdbcTemplate(source);
usage = spy(new UsageAuditService(jdbc, json));
provider = mock(AsyncProviderClient.class);
GatewayGuard guard=mock(GatewayGuard.class);
guard=mock(GatewayGuard.class);
when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true);
catalog = mock(CatalogService.class);
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000,
@@ -152,7 +154,10 @@ class AsyncTaskPostgresTest {
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId);
when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(new ApiException(code,"上游任务未接受",503));
when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(
"PROVIDER_NOT_CONFIGURED".equals(code)
? ProviderRequestException.notSent(code, "上游任务未接受", 503)
: ProviderRequestException.http(401, "上游任务未接受"));
tasks.completeMockTasks();
tasks.completeMockTasks();
assertEquals("failed",status());
@@ -216,19 +221,167 @@ class AsyncTaskPostgresTest {
assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId));
}
@Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed() {
@Test void deadlineRetainsHoldWithoutClaimingTheUpstreamFailed() {
jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId);
poll(); poll();
assertEquals("failed",status());
assertEquals("reconciling",status());
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
assertEquals(0,wallet("reserved_balance").signum());
assertEquals(0,cost().signum());
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
assertNull(cost());
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT"));
assertTrue(error.contains("执行结果未知"));
verifyNoInteractions(provider);
}
@ParameterizedTest
@ValueSource(ints={408,409,500,502,503,504})
void uncertainSubmitResponseNeverRefundsOrResubmits(int httpStatus) {
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
.thenThrow(ProviderRequestException.http(httpStatus, "safe local message"));
tasks.completeMockTasks();
tasks.completeMockTasks();
assertEquals("reconciling", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertNull(cost());
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
assertEquals("pending", jdbc.queryForObject("SELECT cost_status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId));
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString());
}
@Test void lostSubmitResponseAndInterruptedClaimRequireReconciliationWithoutReplay() {
jdbc.update("UPDATE model_tasks SET provider_task_id=NULL,updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
tasks.completeMockTasks();
tasks.completeMockTasks();
assertEquals("reconciling", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertNull(cost());
verifyNoInteractions(provider);
}
@Test void missingTaskReceiptRetainsFinancialAndBudgetHolds() {
var created = create("missing-receipt", Map.of("model", "model-mock-3d", "prompt", "chair"));
String id = String.valueOf(created.get("id"));
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap())).thenReturn(Map.of("accepted", true));
tasks.completeMockTasks();
tasks.completeMockTasks();
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id));
assertEquals(new BigDecimal("2.88000000"), wallet("reserved_balance"));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id));
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap());
}
@Test void genericSubmitFailureIsNotTreatedAsEvidenceOfRejection() {
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
.thenThrow(new IllegalStateException("network connection lost; private details"));
tasks.completeMockTasks();
assertEquals("reconciling", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertFalse(jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?", String.class, taskId).contains("private details"));
}
@Test void missingPollingCredentialDoesNotRefundAnAlreadyAcceptedJob() {
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
.thenThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "sensitive detail", 503));
poll();
assertEquals("reconciling", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertNull(cost());
}
@Test void inlineSuccessSurvivesSettlementFailureAndRecoversWithoutRegenerating() {
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
.thenReturn(Map.of("_moyiInlineResult", Map.of("url", "https://example.test/generated.glb")));
doThrow(new IllegalStateException("simulated audit failure")).when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull());
assertThrows(IllegalStateException.class, () -> tasks.completeMockTasks());
assertEquals("running", status());
assertEquals("succeeded", jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId));
fail(); // A stale error cannot refund an already observed success before the retry settles it.
assertEquals("running", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertNull(cost());
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
doCallRealMethod().when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull());
tasks.completeMockTasks();
tasks.completeMockTasks();
assertEquals("succeeded", status());
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
assertEquals(0, wallet("reserved_balance").signum());
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId));
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString());
}
@ParameterizedTest
@ValueSource(strings={"unsuccessful","not_failed","cancelling","unknown"})
void ambiguousProviderStatusCannotTriggerSettlementOrRefund(String state) {
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
.thenReturn(Map.of("status", state));
poll();
assertEquals("running", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertNull(cost());
}
@Test void interruptedClaimCheckCannotQuarantineALaterAcceptedReceipt() {
var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage);
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
journal.requireReconciliation(taskId, "STALE_WORKER", "stale claim check", true);
assertEquals("running", status(), "An upstream task ID proves this is no longer an interrupted claim");
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
}
@Test void reconciliationAndUsageAuditRollBackTogetherWithoutTouchingTheHold() {
var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage);
doThrow(new IllegalStateException("audit offline")).when(usage).updateTask(eq(taskId), eq("reconciling"), isNull(), anyString(), anyString());
assertThrows(IllegalStateException.class, () -> journal.requireReconciliation(taskId, "UNKNOWN", "result unknown", false));
assertEquals("running", status());
assertEquals("running", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertNull(cost());
}
@Test void failedTaskReplayDoesNotRequeueOrReserveAgain() {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "will be rejected");
var first = create("rejected-task", input);
String id = String.valueOf(first.get("id"));
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap()))
.thenThrow(ProviderRequestException.http(429, "provider rejected"));
tasks.completeMockTasks();
BigDecimal held = wallet("reserved_balance");
var replay = create("rejected-task", input);
assertEquals(id, replay.get("id"));
assertEquals("failed", replay.get("status"));
assertEquals(held, wallet("reserved_balance"));
assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id));
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap());
}
@Test void pendingReconciliationConsumesAsyncConcurrencyRatherThanOpeningAnotherSlot() {
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000, false,
guard, usage, provider, new DataSourceTransactionManager(jdbc.getDataSource()),
new com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy(com.moyi.controlplane.gateway.request.ApiRequestRiskEngine.allowAll()));
jdbc.update("UPDATE model_tasks SET status='reconciling' WHERE id=?", taskId);
jdbc.update("UPDATE model_catalog SET max_concurrency=1 WHERE id='model-mock-3d'");
String id = String.valueOf(create("next-while-unknown", Map.of("model", "model-mock-3d", "prompt", "another chair")).get("id"));
tasks.completeMockTasks();
assertEquals("queued", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id));
verifyNoInteractions(provider);
}
@Test void documentedSucceededStatusSettlesInsteadOfPollingForever() {
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
.thenReturn(Map.of("status", "succeeded", "url", "https://example.test/generated.glb"));
poll();
assertEquals("succeeded", status());
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
}
@Test void confirmedUpstreamFailureReleasesHoldImmediately() {
when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)).thenReturn(Map.of("data",Map.of("status","failed")));
poll();
@@ -39,7 +39,7 @@ class UsageCapacityPostgresTest {
UUID user = UUID.randomUUID(), key = UUID.randomUUID();
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'capacity-user','username','not-loginable','Capacity user')", user);
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Capacity test','local-test',?)", key, user, "c".repeat(64));
for (String state : List.of("running", "queued", "succeeded")) {
for (String state : List.of("running", "queued", "reconciling", "succeeded")) {
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,created_at) VALUES (?,?,?,'capacity-route-105','3d',?,now()-interval '1 day')", "task-" + state, user, key, state);
}
}
@@ -67,9 +67,11 @@ class UsageCapacityPostgresTest {
Map<String, Object> async = groups.stream().filter(group -> group.get("id").equals("capacity-route-105")).findFirst().orElseThrow();
assertEquals(1L, async.get("running"), "A task older than ten minutes must remain in live capacity");
assertEquals(1L, async.get("queued"), "Running tasks must not also count as queued");
assertEquals(1L, async.get("reconciling"), "Unknown upstream outcomes must remain visible, not be counted as failed or idle");
assertEquals(7, async.get("limit"), "Null route limits use the real configured model default");
assertEquals(3L, summary.get("running"));
assertEquals(4L, summary.get("waiting"));
assertEquals(1L, summary.get("reconciling"));
Map<?, ?> config = (Map<?, ?>) summary.get("capacityConfig");
assertEquals(4500L, config.get("queueWaitMs"));
assertEquals(3, config.get("defaultKeyConcurrency"));