feat: make async task creation durably idempotent

This commit is contained in:
xpp committed 2026-10-21 18:41:21 +08:00
1 parent 5533a483d9
commit 76265845dc
8 files changed
+232 -9

No files matched your search

@@ -43,8 +43,9 @@ public class PlaygroundController {
@PostMapping(value="/{type:3d|video|image|audio}/tasks", consumes="application/json")
public ResponseEntity<Map<String, Object>> createTask(@PathVariable String keyId, @PathVariable String type,
@RequestBody Map<String, Object> input, HttpServletRequest request) {
return ResponseEntity.accepted().body(Map.of("task", tasks.createForKey(requireKey(request, keyId), type, input,
@RequestBody Map<String, Object> input,
@RequestHeader(value="Idempotency-Key", required=false) String idempotencyKey, HttpServletRequest request) {
return ResponseEntity.accepted().body(Map.of("task", tasks.createForKey(requireKey(request, keyId), type, input, idempotencyKey,
com.moyi.controlplane.gateway.request.ApiRequestMetadata.playground(request.getRemoteAddr()))));
}
@@ -113,7 +113,13 @@ public class AsyncTaskService {
@Transactional
public Map<String, Object> create(String authorization, String taskType, Map<String, Object> input, ApiRequestMetadata metadata) {
return createForKey(requireKey(authorization), taskType, input, metadata);
return create(authorization, taskType, input, null, metadata);
}
@Transactional
public Map<String, Object> create(String authorization, String taskType, Map<String, Object> input,
String idempotencyKey, ApiRequestMetadata metadata) {
return createForKey(requireKey(authorization), taskType, input, idempotencyKey, metadata);
}
@Transactional
@@ -123,8 +129,36 @@ public class AsyncTaskService {
@Transactional
public Map<String, Object> createForKey(Map<String, Object> key, String taskType, Map<String, Object> input, ApiRequestMetadata metadata) {
return createForKey(key, taskType, input, null, metadata);
}
@Transactional
public Map<String, Object> createForKey(Map<String, Object> key, String taskType, Map<String, Object> input,
String idempotencyKey, ApiRequestMetadata metadata) {
// Explicit transaction also protects callers that do not go through the Spring proxy.
return inTransaction(() -> createInTransaction(key, taskType, input, idempotencyKey, metadata));
}
private Map<String,Object> createInTransaction(Map<String,Object> key, String taskType, Map<String,Object> input,
String idempotencyKey, ApiRequestMetadata metadata) {
UUID apiKeyId = UUID.fromString(String.valueOf(key.get("id")));
gatewayGuard.checkRate(apiKeyId, false);
if (input == null) throw new ApiException("INVALID_REQUEST", "请提供任务请求");
TaskCreationIdentity identity = TaskCreationIdentity.of(objectMapper, idempotencyKey, taskType, input);
UUID userId = catalogService.keyOwnerId(apiKeyId);
if (identity.keyHash() != null) {
// Serialize the same user/key before checking capacity, budget or wallet balances.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))",
"moyi:task-idempotency:" + userId + ":" + identity.keyHash());
var existing = jdbc.queryForList("SELECT id,request_hash FROM model_tasks WHERE user_id=? AND idempotency_hash=?", userId, identity.keyHash());
if (!existing.isEmpty()) {
if (!identity.requestHash().equals(existing.getFirst().get("request_hash"))) {
throw new ApiException("IDEMPOTENCY_CONFLICT", "此 Idempotency-Key 已用于不同的任务请求", 409);
}
// Authenticated same-user reads remain valid even if the original route was retired.
return taskView(String.valueOf(existing.getFirst().get("id")));
}
}
String requestedModelId = String.valueOf(input.getOrDefault("model", ""));
Map<String, Object> model = GatewayModelIds.resolve(catalogService, key, requestedModelId);
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线", 404);
@@ -137,7 +171,6 @@ public class AsyncTaskService {
if (prompt.isBlank() && (mediaInput == null || String.valueOf(mediaInput).isBlank())) {
throw new ApiException("PROMPT_REQUIRED", "生成提示词或媒体输入不能为空");
}
UUID userId = catalogService.keyOwnerId(apiKeyId);
requestRisk.enforce(new ApiRequestContext(userId, apiKeyId, ApiRequestProtocol.ASYNC_TASK,
requestedModelId, modelId, false, estimateInputUnits(input), 0, metadata, System.nanoTime()));
// Serialize the queue-capacity check with task insertion for this
@@ -161,8 +194,8 @@ public class AsyncTaskService {
OffsetDateTime now = now();
try {
jdbc.update(
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input == null ? Map.of() : input), reservedAmount, now, now
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash()
);
} catch (RuntimeException exception) {
releaseWallet(userId, reservedAmount);
@@ -27,11 +27,12 @@ public class TaskController {
public ResponseEntity<Map<String, Object>> create3d(
@org.springframework.web.bind.annotation.PathVariable(required = false) String type,
@RequestHeader(value = "Authorization", required = false) String authorization,
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
@RequestBody Map<String, Object> input,
HttpServletRequest request
) {
String taskType = type == null ? String.valueOf(input == null ? "3d" : input.getOrDefault("taskType", "3d")) : type;
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, taskType, input,
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, taskType, input, idempotencyKey,
ApiRequestMetadata.fromHttp(request.getRemoteAddr()))));
}
@@ -39,10 +40,11 @@ public class TaskController {
@PostMapping("/video/tasks")
public ResponseEntity<Map<String, Object>> createVideo(
@RequestHeader(value = "Authorization", required = false) String authorization,
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
@RequestBody Map<String, Object> input,
HttpServletRequest request
) {
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, "video", input,
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, "video", input, idempotencyKey,
ApiRequestMetadata.fromHttp(request.getRemoteAddr()))));
}
@@ -0,0 +1,58 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.DecimalNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.moyi.controlplane.common.ApiException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
import java.util.Map;
import java.util.TreeSet;
/** Stable request identity only; never stores or exposes the caller's raw idempotency key. */
record TaskCreationIdentity(String keyHash, String requestHash) {
static TaskCreationIdentity of(ObjectMapper mapper, String key, String type, Map<String,Object> input) {
if (key == null) return new TaskCreationIdentity(null, null);
if (!key.matches("[!-~]{1,128}")) {
throw new ApiException("INVALID_IDEMPOTENCY_KEY", "Idempotency-Key 须为 1–128 个可见 ASCII 字符");
}
try {
ObjectNode request = mapper.createObjectNode();
request.put("taskType", type);
request.set("input", canonical(mapper, mapper.valueToTree(input)));
return new TaskCreationIdentity(hash(key), hash(mapper.writeValueAsString(request)));
} catch (JsonProcessingException | IllegalArgumentException exception) {
throw new ApiException("INVALID_REQUEST", "任务请求无法编码");
}
}
private static JsonNode canonical(ObjectMapper mapper, JsonNode node) {
if (node.isObject()) {
ObjectNode result = mapper.createObjectNode();
var fields = new TreeSet<String>();
node.fieldNames().forEachRemaining(fields::add);
for (String field : fields) result.set(field, canonical(mapper, node.get(field)));
return result;
}
if (node.isArray()) {
ArrayNode result = mapper.createArrayNode();
node.forEach(value -> result.add(canonical(mapper, value)));
return result;
}
if (node.isNumber()) return DecimalNode.valueOf(node.decimalValue().stripTrailingZeros());
return node;
}
private static String hash(String value) {
try {
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)));
} catch (NoSuchAlgorithmException exception) {
throw new IllegalStateException("SHA-256 is required by the Java runtime", exception);
}
}
}
@@ -0,0 +1,9 @@
-- User-scoped durable identities survive completion; raw Idempotency-Key values are never persisted.
ALTER TABLE model_tasks ADD COLUMN idempotency_hash varchar(64);
ALTER TABLE model_tasks ADD COLUMN request_hash varchar(64);
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_request_identity_check
CHECK ((idempotency_hash IS NULL AND request_hash IS NULL)
OR (idempotency_hash ~ '^[0-9a-f]{64}$' AND request_hash ~ '^[0-9a-f]{64}$'
AND idempotency_hash IS NOT NULL AND request_hash IS NOT NULL));
CREATE UNIQUE INDEX model_tasks_user_idempotency_idx ON model_tasks(user_id, idempotency_hash)
WHERE idempotency_hash IS NOT NULL;
@@ -10,6 +10,11 @@ import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.UUID;
import java.util.ArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -36,6 +41,7 @@ class AsyncTaskPostgresTest {
private UsageAuditService usage;
private AsyncProviderClient provider;
private AsyncTaskService tasks;
private CatalogService catalog;
@BeforeEach void setup() {
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
@@ -48,7 +54,8 @@ class AsyncTaskPostgresTest {
provider = mock(AsyncProviderClient.class);
GatewayGuard guard=mock(GatewayGuard.class);
when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true);
tasks = new AsyncTaskService(jdbc, json, mock(CatalogService.class), 100, 4, 4, 120_000,
catalog = mock(CatalogService.class);
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000,
guard, usage, provider, new DataSourceTransactionManager(source));
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'async-test','username','not-a-password','Test')", userId);
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-prefix',?)", keyId, userId, "b".repeat(64));
@@ -56,6 +63,9 @@ class AsyncTaskPostgresTest {
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
when(catalog.keyOwnerId(keyId)).thenReturn(userId);
when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true);
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
}
@AfterEach void cleanup() {
@@ -81,6 +91,63 @@ class AsyncTaskPostgresTest {
return jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?",BigDecimal.class,taskId);
}
private Map<String,Object> create(String idempotencyKey, Map<String,Object> input) {
return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal());
}
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
var first = create("client-operation-1", input);
var replay = create("client-operation-1", new java.util.LinkedHashMap<>(input));
assertEquals(first.get("id"), replay.get("id"));
String created = String.valueOf(first.get("id"));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, created));
BigDecimal held = wallet("reserved_balance");
ReflectionTestUtils.invokeMethod(tasks, "finishTask", created, "succeeded", 100, Map.of("url", "https://example.test/result.glb"), null, null);
when(catalog.activeModel("model-mock-3d")).thenReturn(null);
var finishedReplay = create("client-operation-1", input);
assertEquals(created, finishedReplay.get("id"));
assertEquals("succeeded", finishedReplay.get("status"));
assertTrue(wallet("reserved_balance").compareTo(held) < 0);
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE request_id IS NOT NULL AND user_id=?", Integer.class, userId));
String stored = jdbc.queryForObject("SELECT idempotency_hash FROM model_tasks WHERE id=?", String.class, created);
assertEquals(64, stored.length());
assertNotEquals("client-operation-1", stored);
}
@Test void sameKeyDifferentRequestConflictsWithoutNewReservation() {
create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "chair"));
BigDecimal held = wallet("reserved_balance");
ApiException conflict = assertThrows(ApiException.class,
() -> create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "table")));
assertEquals("IDEMPOTENCY_CONFLICT", conflict.getCode());
assertEquals(409, conflict.getStatus());
assertEquals(held, wallet("reserved_balance"));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class));
}
@Test void concurrentSameKeyCreatesOneTaskOneAuditAndOneHold() throws Exception {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "concurrent chair");
BigDecimal held = wallet("reserved_balance");
var start = new CountDownLatch(1);
try (var executor = Executors.newFixedThreadPool(8)) {
var results = new ArrayList<java.util.concurrent.Future<Map<String,Object>>>();
for (int i = 0; i < 8; i++) results.add(executor.submit(() -> {
assertTrue(start.await(10, TimeUnit.SECONDS));
return create("concurrent-operation", input);
}));
start.countDown();
var ids = new java.util.HashSet<Object>();
for (var future : results) ids.add(future.get(30, TimeUnit.SECONDS).get("id"));
assertEquals(1, ids.size());
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, ids.iterator().next()));
}
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class));
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance").subtract(held));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds", Integer.class));
verifyNoInteractions(provider);
}
@ParameterizedTest
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
@@ -0,0 +1,36 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class TaskCreationIdentityTest {
private final ObjectMapper mapper = new ObjectMapper();
@Test void canonicalizesObjectOrderAndNumbersButPreservesArrayOrderAndContent() {
var first = new LinkedHashMap<String,Object>();
first.put("model", "route");
first.put("settings", Map.of("quality", 1, "images", List.of("a", "b")));
var reordered = new LinkedHashMap<String,Object>();
reordered.put("settings", Map.of("images", List.of("a", "b"), "quality", 1.0));
reordered.put("model", "route");
var original = TaskCreationIdentity.of(mapper, "same-key", "image", first);
assertEquals(original, TaskCreationIdentity.of(mapper, "same-key", "image", reordered));
assertNotEquals(original.requestHash(), TaskCreationIdentity.of(mapper, "same-key", "video", reordered).requestHash());
assertNotEquals(original.requestHash(), TaskCreationIdentity.of(mapper, "same-key", "image", Map.of("model", "route", "settings", Map.of("quality", 1, "images", List.of("b", "a")))).requestHash());
assertEquals(64, original.keyHash().length());
assertFalse(original.toString().contains("same-key"));
}
@Test void rejectsBlankControlNonAsciiAndOversizeKeys() {
for (String value : List.of("", " ", "has space", "line\nbreak", "中文", "a".repeat(129))) {
assertEquals("INVALID_IDEMPOTENCY_KEY", assertThrows(ApiException.class,
() -> TaskCreationIdentity.of(mapper, value, "image", Map.of())).getCode());
}
assertNull(TaskCreationIdentity.of(mapper, null, "image", Map.of()).keyHash());
}
}
+17
View File
@@ -0,0 +1,17 @@
# 异步任务幂等与执行安全
## 阶段一:创建幂等(2026-10-21)
- 公网 `/v1/{3d|video|image|audio}/tasks` 与 Playground 创建入口支持可选 `Idempotency-Key`。
- 键为 1–128 个可见 ASCII 字符;只保存 SHA-256,不保存原始键。幂等命名空间为用户,不局限于单个 API Key。
- 请求摘要包括任务类型和完整请求体,递归排序 JSON 对象键、规范化数值,保留数组顺序及文本内容;不忽略未知字段或悄悄修剪提示词。
- 相同用户、相同键、相同请求返回原任务的当前状态,包括成功或失败后的重放;相同键和不同请求返回 HTTP 409 / `IDEMPOTENCY_CONFLICT`。
- 有效凭据校验、基础 RPM 检查先执行;幂等重放不再进入生成流程,因此不受后来变更的模型价格、队列容量、余额或路由下线影响。这与同一用户查询历史任务的权限范围一致。
- 同键创建在数据库事务中取得用户/键摘要 advisory lock;部分唯一索引提供最终约束。检查原任务发生在队列计数、预算占用、钱包冻结之前。
- 不提供键失效后自动重用,避免客户端晚到重试再次生成。无幂等键的调用保持原有语义。
验证:Java 21 / Maven,隔离 PostgreSQL 15.18(不连接业务数据库),26 项针对性测试通过,0 失败、0 跳过。包含 8 路并发同键请求、重放、请求冲突、完成后路由下线重放、键摘要规范化与风控顺序。
## 接续项
接续修复提交响应丢失、进程中断、轮询超时以及结算数据库失败的语义。未知结果不得自动重发生成请求或释放冻结余额;只有已证实的拒绝或终态失败才能进入正常释放流程。核账操作必须有证据和审计,不允许用直接修改钱包余额替代结算事务。