feat: make async task creation durably idempotent

This commit is contained in:
xpp committed 2026-10-21 18:41:21 +08:00
1 parent 5533a483d9
commit 76265845dc
8 files changed
+232 -9

No files matched your search

@@ -10,6 +10,11 @@ import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.UUID;
import java.util.ArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -36,6 +41,7 @@ class AsyncTaskPostgresTest {
private UsageAuditService usage;
private AsyncProviderClient provider;
private AsyncTaskService tasks;
private CatalogService catalog;
@BeforeEach void setup() {
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
@@ -48,7 +54,8 @@ class AsyncTaskPostgresTest {
provider = mock(AsyncProviderClient.class);
GatewayGuard guard=mock(GatewayGuard.class);
when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true);
tasks = new AsyncTaskService(jdbc, json, mock(CatalogService.class), 100, 4, 4, 120_000,
catalog = mock(CatalogService.class);
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000,
guard, usage, provider, new DataSourceTransactionManager(source));
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'async-test','username','not-a-password','Test')", userId);
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-prefix',?)", keyId, userId, "b".repeat(64));
@@ -56,6 +63,9 @@ class AsyncTaskPostgresTest {
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
when(catalog.keyOwnerId(keyId)).thenReturn(userId);
when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true);
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
}
@AfterEach void cleanup() {
@@ -81,6 +91,63 @@ class AsyncTaskPostgresTest {
return jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?",BigDecimal.class,taskId);
}
private Map<String,Object> create(String idempotencyKey, Map<String,Object> input) {
return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal());
}
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
var first = create("client-operation-1", input);
var replay = create("client-operation-1", new java.util.LinkedHashMap<>(input));
assertEquals(first.get("id"), replay.get("id"));
String created = String.valueOf(first.get("id"));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, created));
BigDecimal held = wallet("reserved_balance");
ReflectionTestUtils.invokeMethod(tasks, "finishTask", created, "succeeded", 100, Map.of("url", "https://example.test/result.glb"), null, null);
when(catalog.activeModel("model-mock-3d")).thenReturn(null);
var finishedReplay = create("client-operation-1", input);
assertEquals(created, finishedReplay.get("id"));
assertEquals("succeeded", finishedReplay.get("status"));
assertTrue(wallet("reserved_balance").compareTo(held) < 0);
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE request_id IS NOT NULL AND user_id=?", Integer.class, userId));
String stored = jdbc.queryForObject("SELECT idempotency_hash FROM model_tasks WHERE id=?", String.class, created);
assertEquals(64, stored.length());
assertNotEquals("client-operation-1", stored);
}
@Test void sameKeyDifferentRequestConflictsWithoutNewReservation() {
create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "chair"));
BigDecimal held = wallet("reserved_balance");
ApiException conflict = assertThrows(ApiException.class,
() -> create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "table")));
assertEquals("IDEMPOTENCY_CONFLICT", conflict.getCode());
assertEquals(409, conflict.getStatus());
assertEquals(held, wallet("reserved_balance"));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class));
}
@Test void concurrentSameKeyCreatesOneTaskOneAuditAndOneHold() throws Exception {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "concurrent chair");
BigDecimal held = wallet("reserved_balance");
var start = new CountDownLatch(1);
try (var executor = Executors.newFixedThreadPool(8)) {
var results = new ArrayList<java.util.concurrent.Future<Map<String,Object>>>();
for (int i = 0; i < 8; i++) results.add(executor.submit(() -> {
assertTrue(start.await(10, TimeUnit.SECONDS));
return create("concurrent-operation", input);
}));
start.countDown();
var ids = new java.util.HashSet<Object>();
for (var future : results) ids.add(future.get(30, TimeUnit.SECONDS).get("id"));
assertEquals(1, ids.size());
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, ids.iterator().next()));
}
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class));
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance").subtract(held));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds", Integer.class));
verifyNoInteractions(provider);
}
@ParameterizedTest
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {