feat: make async task creation durably idempotent
This commit is contained in:
1 parent
5533a483d9
commit
76265845dc
8 files changed
+232
-9
No files matched your search
+3
-2
@@ -43,8 +43,9 @@ public class PlaygroundController {
|
||||
|
||||
@PostMapping(value="/{type:3d|video|image|audio}/tasks", consumes="application/json")
|
||||
public ResponseEntity<Map<String, Object>> createTask(@PathVariable String keyId, @PathVariable String type,
|
||||
@RequestBody Map<String, Object> input, HttpServletRequest request) {
|
||||
return ResponseEntity.accepted().body(Map.of("task", tasks.createForKey(requireKey(request, keyId), type, input,
|
||||
@RequestBody Map<String, Object> input,
|
||||
@RequestHeader(value="Idempotency-Key", required=false) String idempotencyKey, HttpServletRequest request) {
|
||||
return ResponseEntity.accepted().body(Map.of("task", tasks.createForKey(requireKey(request, keyId), type, input, idempotencyKey,
|
||||
com.moyi.controlplane.gateway.request.ApiRequestMetadata.playground(request.getRemoteAddr()))));
|
||||
}
|
||||
|
||||
|
||||
@@ -113,7 +113,13 @@ public class AsyncTaskService {
|
||||
|
||||
@Transactional
|
||||
public Map<String, Object> create(String authorization, String taskType, Map<String, Object> input, ApiRequestMetadata metadata) {
|
||||
return createForKey(requireKey(authorization), taskType, input, metadata);
|
||||
return create(authorization, taskType, input, null, metadata);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Map<String, Object> create(String authorization, String taskType, Map<String, Object> input,
|
||||
String idempotencyKey, ApiRequestMetadata metadata) {
|
||||
return createForKey(requireKey(authorization), taskType, input, idempotencyKey, metadata);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -123,8 +129,36 @@ public class AsyncTaskService {
|
||||
|
||||
@Transactional
|
||||
public Map<String, Object> createForKey(Map<String, Object> key, String taskType, Map<String, Object> input, ApiRequestMetadata metadata) {
|
||||
return createForKey(key, taskType, input, null, metadata);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Map<String, Object> createForKey(Map<String, Object> key, String taskType, Map<String, Object> input,
|
||||
String idempotencyKey, ApiRequestMetadata metadata) {
|
||||
// Explicit transaction also protects callers that do not go through the Spring proxy.
|
||||
return inTransaction(() -> createInTransaction(key, taskType, input, idempotencyKey, metadata));
|
||||
}
|
||||
|
||||
private Map<String,Object> createInTransaction(Map<String,Object> key, String taskType, Map<String,Object> input,
|
||||
String idempotencyKey, ApiRequestMetadata metadata) {
|
||||
UUID apiKeyId = UUID.fromString(String.valueOf(key.get("id")));
|
||||
gatewayGuard.checkRate(apiKeyId, false);
|
||||
if (input == null) throw new ApiException("INVALID_REQUEST", "请提供任务请求");
|
||||
TaskCreationIdentity identity = TaskCreationIdentity.of(objectMapper, idempotencyKey, taskType, input);
|
||||
UUID userId = catalogService.keyOwnerId(apiKeyId);
|
||||
if (identity.keyHash() != null) {
|
||||
// Serialize the same user/key before checking capacity, budget or wallet balances.
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))",
|
||||
"moyi:task-idempotency:" + userId + ":" + identity.keyHash());
|
||||
var existing = jdbc.queryForList("SELECT id,request_hash FROM model_tasks WHERE user_id=? AND idempotency_hash=?", userId, identity.keyHash());
|
||||
if (!existing.isEmpty()) {
|
||||
if (!identity.requestHash().equals(existing.getFirst().get("request_hash"))) {
|
||||
throw new ApiException("IDEMPOTENCY_CONFLICT", "此 Idempotency-Key 已用于不同的任务请求", 409);
|
||||
}
|
||||
// Authenticated same-user reads remain valid even if the original route was retired.
|
||||
return taskView(String.valueOf(existing.getFirst().get("id")));
|
||||
}
|
||||
}
|
||||
String requestedModelId = String.valueOf(input.getOrDefault("model", ""));
|
||||
Map<String, Object> model = GatewayModelIds.resolve(catalogService, key, requestedModelId);
|
||||
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线", 404);
|
||||
@@ -137,7 +171,6 @@ public class AsyncTaskService {
|
||||
if (prompt.isBlank() && (mediaInput == null || String.valueOf(mediaInput).isBlank())) {
|
||||
throw new ApiException("PROMPT_REQUIRED", "生成提示词或媒体输入不能为空");
|
||||
}
|
||||
UUID userId = catalogService.keyOwnerId(apiKeyId);
|
||||
requestRisk.enforce(new ApiRequestContext(userId, apiKeyId, ApiRequestProtocol.ASYNC_TASK,
|
||||
requestedModelId, modelId, false, estimateInputUnits(input), 0, metadata, System.nanoTime()));
|
||||
// Serialize the queue-capacity check with task insertion for this
|
||||
@@ -161,8 +194,8 @@ public class AsyncTaskService {
|
||||
OffsetDateTime now = now();
|
||||
try {
|
||||
jdbc.update(
|
||||
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?)",
|
||||
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input == null ? Map.of() : input), reservedAmount, now, now
|
||||
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?)",
|
||||
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash()
|
||||
);
|
||||
} catch (RuntimeException exception) {
|
||||
releaseWallet(userId, reservedAmount);
|
||||
|
||||
@@ -27,11 +27,12 @@ public class TaskController {
|
||||
public ResponseEntity<Map<String, Object>> create3d(
|
||||
@org.springframework.web.bind.annotation.PathVariable(required = false) String type,
|
||||
@RequestHeader(value = "Authorization", required = false) String authorization,
|
||||
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
|
||||
@RequestBody Map<String, Object> input,
|
||||
HttpServletRequest request
|
||||
) {
|
||||
String taskType = type == null ? String.valueOf(input == null ? "3d" : input.getOrDefault("taskType", "3d")) : type;
|
||||
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, taskType, input,
|
||||
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, taskType, input, idempotencyKey,
|
||||
ApiRequestMetadata.fromHttp(request.getRemoteAddr()))));
|
||||
}
|
||||
|
||||
@@ -39,10 +40,11 @@ public class TaskController {
|
||||
@PostMapping("/video/tasks")
|
||||
public ResponseEntity<Map<String, Object>> createVideo(
|
||||
@RequestHeader(value = "Authorization", required = false) String authorization,
|
||||
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
|
||||
@RequestBody Map<String, Object> input,
|
||||
HttpServletRequest request
|
||||
) {
|
||||
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, "video", input,
|
||||
return ResponseEntity.status(HttpStatus.ACCEPTED).body(Map.of("task", tasks.create(authorization, "video", input, idempotencyKey,
|
||||
ApiRequestMetadata.fromHttp(request.getRemoteAddr()))));
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.DecimalNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HexFormat;
|
||||
import java.util.Map;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/** Stable request identity only; never stores or exposes the caller's raw idempotency key. */
|
||||
record TaskCreationIdentity(String keyHash, String requestHash) {
|
||||
static TaskCreationIdentity of(ObjectMapper mapper, String key, String type, Map<String,Object> input) {
|
||||
if (key == null) return new TaskCreationIdentity(null, null);
|
||||
if (!key.matches("[!-~]{1,128}")) {
|
||||
throw new ApiException("INVALID_IDEMPOTENCY_KEY", "Idempotency-Key 须为 1–128 个可见 ASCII 字符");
|
||||
}
|
||||
try {
|
||||
ObjectNode request = mapper.createObjectNode();
|
||||
request.put("taskType", type);
|
||||
request.set("input", canonical(mapper, mapper.valueToTree(input)));
|
||||
return new TaskCreationIdentity(hash(key), hash(mapper.writeValueAsString(request)));
|
||||
} catch (JsonProcessingException | IllegalArgumentException exception) {
|
||||
throw new ApiException("INVALID_REQUEST", "任务请求无法编码");
|
||||
}
|
||||
}
|
||||
|
||||
private static JsonNode canonical(ObjectMapper mapper, JsonNode node) {
|
||||
if (node.isObject()) {
|
||||
ObjectNode result = mapper.createObjectNode();
|
||||
var fields = new TreeSet<String>();
|
||||
node.fieldNames().forEachRemaining(fields::add);
|
||||
for (String field : fields) result.set(field, canonical(mapper, node.get(field)));
|
||||
return result;
|
||||
}
|
||||
if (node.isArray()) {
|
||||
ArrayNode result = mapper.createArrayNode();
|
||||
node.forEach(value -> result.add(canonical(mapper, value)));
|
||||
return result;
|
||||
}
|
||||
if (node.isNumber()) return DecimalNode.valueOf(node.decimalValue().stripTrailingZeros());
|
||||
return node;
|
||||
}
|
||||
|
||||
private static String hash(String value) {
|
||||
try {
|
||||
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)));
|
||||
} catch (NoSuchAlgorithmException exception) {
|
||||
throw new IllegalStateException("SHA-256 is required by the Java runtime", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
-- User-scoped durable identities survive completion; raw Idempotency-Key values are never persisted.
|
||||
ALTER TABLE model_tasks ADD COLUMN idempotency_hash varchar(64);
|
||||
ALTER TABLE model_tasks ADD COLUMN request_hash varchar(64);
|
||||
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_request_identity_check
|
||||
CHECK ((idempotency_hash IS NULL AND request_hash IS NULL)
|
||||
OR (idempotency_hash ~ '^[0-9a-f]{64}$' AND request_hash ~ '^[0-9a-f]{64}$'
|
||||
AND idempotency_hash IS NOT NULL AND request_hash IS NOT NULL));
|
||||
CREATE UNIQUE INDEX model_tasks_user_idempotency_idx ON model_tasks(user_id, idempotency_hash)
|
||||
WHERE idempotency_hash IS NOT NULL;
|
||||
+68
-1
@@ -10,6 +10,11 @@ import java.time.Duration;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -36,6 +41,7 @@ class AsyncTaskPostgresTest {
|
||||
private UsageAuditService usage;
|
||||
private AsyncProviderClient provider;
|
||||
private AsyncTaskService tasks;
|
||||
private CatalogService catalog;
|
||||
|
||||
@BeforeEach void setup() {
|
||||
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
|
||||
@@ -48,7 +54,8 @@ class AsyncTaskPostgresTest {
|
||||
provider = mock(AsyncProviderClient.class);
|
||||
GatewayGuard guard=mock(GatewayGuard.class);
|
||||
when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true);
|
||||
tasks = new AsyncTaskService(jdbc, json, mock(CatalogService.class), 100, 4, 4, 120_000,
|
||||
catalog = mock(CatalogService.class);
|
||||
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000,
|
||||
guard, usage, provider, new DataSourceTransactionManager(source));
|
||||
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'async-test','username','not-a-password','Test')", userId);
|
||||
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-prefix',?)", keyId, userId, "b".repeat(64));
|
||||
@@ -56,6 +63,9 @@ class AsyncTaskPostgresTest {
|
||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
|
||||
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
|
||||
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
|
||||
when(catalog.keyOwnerId(keyId)).thenReturn(userId);
|
||||
when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true);
|
||||
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() {
|
||||
@@ -81,6 +91,63 @@ class AsyncTaskPostgresTest {
|
||||
return jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?",BigDecimal.class,taskId);
|
||||
}
|
||||
|
||||
private Map<String,Object> create(String idempotencyKey, Map<String,Object> input) {
|
||||
return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal());
|
||||
}
|
||||
|
||||
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
|
||||
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
|
||||
var first = create("client-operation-1", input);
|
||||
var replay = create("client-operation-1", new java.util.LinkedHashMap<>(input));
|
||||
assertEquals(first.get("id"), replay.get("id"));
|
||||
String created = String.valueOf(first.get("id"));
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, created));
|
||||
BigDecimal held = wallet("reserved_balance");
|
||||
ReflectionTestUtils.invokeMethod(tasks, "finishTask", created, "succeeded", 100, Map.of("url", "https://example.test/result.glb"), null, null);
|
||||
when(catalog.activeModel("model-mock-3d")).thenReturn(null);
|
||||
var finishedReplay = create("client-operation-1", input);
|
||||
assertEquals(created, finishedReplay.get("id"));
|
||||
assertEquals("succeeded", finishedReplay.get("status"));
|
||||
assertTrue(wallet("reserved_balance").compareTo(held) < 0);
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE request_id IS NOT NULL AND user_id=?", Integer.class, userId));
|
||||
String stored = jdbc.queryForObject("SELECT idempotency_hash FROM model_tasks WHERE id=?", String.class, created);
|
||||
assertEquals(64, stored.length());
|
||||
assertNotEquals("client-operation-1", stored);
|
||||
}
|
||||
|
||||
@Test void sameKeyDifferentRequestConflictsWithoutNewReservation() {
|
||||
create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "chair"));
|
||||
BigDecimal held = wallet("reserved_balance");
|
||||
ApiException conflict = assertThrows(ApiException.class,
|
||||
() -> create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "table")));
|
||||
assertEquals("IDEMPOTENCY_CONFLICT", conflict.getCode());
|
||||
assertEquals(409, conflict.getStatus());
|
||||
assertEquals(held, wallet("reserved_balance"));
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class));
|
||||
}
|
||||
|
||||
@Test void concurrentSameKeyCreatesOneTaskOneAuditAndOneHold() throws Exception {
|
||||
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "concurrent chair");
|
||||
BigDecimal held = wallet("reserved_balance");
|
||||
var start = new CountDownLatch(1);
|
||||
try (var executor = Executors.newFixedThreadPool(8)) {
|
||||
var results = new ArrayList<java.util.concurrent.Future<Map<String,Object>>>();
|
||||
for (int i = 0; i < 8; i++) results.add(executor.submit(() -> {
|
||||
assertTrue(start.await(10, TimeUnit.SECONDS));
|
||||
return create("concurrent-operation", input);
|
||||
}));
|
||||
start.countDown();
|
||||
var ids = new java.util.HashSet<Object>();
|
||||
for (var future : results) ids.add(future.get(30, TimeUnit.SECONDS).get("id"));
|
||||
assertEquals(1, ids.size());
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, ids.iterator().next()));
|
||||
}
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class));
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance").subtract(held));
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds", Integer.class));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
|
||||
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class TaskCreationIdentityTest {
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
@Test void canonicalizesObjectOrderAndNumbersButPreservesArrayOrderAndContent() {
|
||||
var first = new LinkedHashMap<String,Object>();
|
||||
first.put("model", "route");
|
||||
first.put("settings", Map.of("quality", 1, "images", List.of("a", "b")));
|
||||
var reordered = new LinkedHashMap<String,Object>();
|
||||
reordered.put("settings", Map.of("images", List.of("a", "b"), "quality", 1.0));
|
||||
reordered.put("model", "route");
|
||||
var original = TaskCreationIdentity.of(mapper, "same-key", "image", first);
|
||||
assertEquals(original, TaskCreationIdentity.of(mapper, "same-key", "image", reordered));
|
||||
assertNotEquals(original.requestHash(), TaskCreationIdentity.of(mapper, "same-key", "video", reordered).requestHash());
|
||||
assertNotEquals(original.requestHash(), TaskCreationIdentity.of(mapper, "same-key", "image", Map.of("model", "route", "settings", Map.of("quality", 1, "images", List.of("b", "a")))).requestHash());
|
||||
assertEquals(64, original.keyHash().length());
|
||||
assertFalse(original.toString().contains("same-key"));
|
||||
}
|
||||
|
||||
@Test void rejectsBlankControlNonAsciiAndOversizeKeys() {
|
||||
for (String value : List.of("", " ", "has space", "line\nbreak", "中文", "a".repeat(129))) {
|
||||
assertEquals("INVALID_IDEMPOTENCY_KEY", assertThrows(ApiException.class,
|
||||
() -> TaskCreationIdentity.of(mapper, value, "image", Map.of())).getCode());
|
||||
}
|
||||
assertNull(TaskCreationIdentity.of(mapper, null, "image", Map.of()).keyHash());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user