feat: add audited async task reconciliation and frozen bindings

This commit is contained in:
xpp committed 2026-10-21 20:08:38 +08:00
1 parent 5be06a97c1
commit c716ec7478
28 files changed
+1312 -76

No files matched your search

+3 -1
View File
@@ -17,7 +17,9 @@
账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;已确认失败的释放和成功结算沿用原子事务;不完整流或无效用量不得估算扣费。
异步创建支持用户级 `Idempotency-Key`(`V45`),同键同请求返回原任务,同键不同请求返回 409。异步上游受理未知、回执丢失、worker 中断或轮询期限到期时进入 `reconciling`(`V46`),保留冻结与预算占用,不自动重发/退款。已观察的生成结果先持久化再结算,账本失败只重试本地结算。发布时须先停旧 worker,不能混跑旧的未知结果退款逻辑。详见 `docs/22-async-task-safety.md`;受审计的人工核账入口仍属下一阶段。
异步创建支持用户级 `Idempotency-Key`(`V45`),同键同请求返回原任务,同键不同请求返回 409。异步上游受理未知、回执丢失、worker 中断或轮询期限到期时进入 `reconciling`(`V46`),保留冻结与预算占用,不自动重发/退款。已观察的生成结果先持久化再结算,账本失败只重试本地结算。发布时须先停旧 worker,不能混跑旧的未知结果退款逻辑。详见 `../docs/22-async-task-safety.md`。
`V47` 冻结异步 provider/model 绑定并约束已证明的上游任务身份;`V48` 增加仅追加核账收据。管理端“价格版本与成本对账”提供人工核账,可根据证据恢复只读轮询、确认失败释放原冻结、确认成功并按原价和实际用量结算。操作使用管理员会话、版本检查、UUID 幂等收据和与钱包同事务的审计;worker 和人工操作共用 `TaskSettlementService`,不另写扣费逻辑。历史无法证明的已受理绑定不能猜测补齐。详见 `../docs/23-audited-task-reconciliation.md`;尚无双人审批,不等同于完整财务流程。
同步和原生调用在结算前单独提交精简用量快照(Flyway `V44`)。快照已持久化而账本事务失败时,接口返回 `SETTLEMENT_PENDING`,冻结额度保留,由后台恢复任务重试结算。该快照不保存提示词、生成内容或密钥;不能将其当作响应重放存储。若数据库在快照写入前已不可用,则尚不能保证结果的持久恢复,必须结合上游记录核对。
@@ -23,6 +23,7 @@ public class GatewayGuard {
private static final long SLOT_TTL_MS = 180_000;
private static final DefaultRedisScript<Long> RATE = script("redis/rate-limit.lua");
private static final DefaultRedisScript<Long> ACQUIRE = script("redis/acquire-slot.lua");
private static final DefaultRedisScript<Long> ACQUIRE_TASK = script("redis/acquire-task-slot.lua");
private static final DefaultRedisScript<Long> ACQUIRE_PAIR = script("redis/acquire-pair.lua");
private static final DefaultRedisScript<List> CAPACITY = new DefaultRedisScript<>("""
local time = redis.call('TIME')
@@ -163,9 +164,11 @@ public class GatewayGuard {
if(policies==null) return true;
var policy=policies.taskLimit(keyId);
String bucket=redisPrefix+"gateway:slots:user:"+policy.userId();
boolean acquired=execute(ACQUIRE,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId)==1;
long result=execute(ACQUIRE_TASK,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId);
boolean acquired=result>0;
// Redis is outside the database transaction: roll back the lease if claiming the task fails.
if(acquired&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
// Rolling back renewal must not remove a lease that existed before this transaction.
if(result==1&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
org.springframework.transaction.support.TransactionSynchronizationManager.registerSynchronization(new org.springframework.transaction.support.TransactionSynchronization() {@Override public void afterCompletion(int status) {if(status!=STATUS_COMMITTED) releaseQuietly(bucket,taskId);}});
return acquired;
}
@@ -110,6 +110,17 @@ public class AsyncProviderClient {
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
}
/** Validate a proven polling binding without sending any HTTP request or exposing credential values. */
public void requirePollableBinding(String providerId, String routeId, String upstreamModel, String taskType, String fingerprint) {
if (fingerprint == null || !fingerprint.matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "需要有效的上游账号指纹", 422);
Provider provider = provider(providerId, routeId, upstreamModel);
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
boolean pollable = isTripo(provider.code(), provider.baseUrl()) ? "3d".equals(taskType)
: (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com") || "openai-compatible".equalsIgnoreCase(provider.protocol())) && "video".equals(taskType);
if (!pollable) throw new ApiException("TASK_NOT_POLLABLE", "该执行器没有可恢复的只读任务查询接口,请核实终态", 409);
provider.keyForFingerprint(routeId, upstreamModel, environment, keyCursors, fingerprint);
}
public Map<String,Object> poll(String providerId, String providerTaskId) {
return poll(providerId, null, providerTaskId);
}
@@ -19,8 +19,6 @@ import java.sql.SQLException;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -52,6 +50,7 @@ public class AsyncTaskService {
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
private final ApiRequestRiskPolicy requestRisk;
private final TaskOutcomeJournal outcomes;
private final TaskSettlementService settlements;
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
@@ -108,6 +107,7 @@ public class AsyncTaskService {
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
this.settlements = new TaskSettlementService(jdbc, objectMapper, gatewayGuard, usage, transactionManager);
}
@Transactional
@@ -177,6 +177,8 @@ public class AsyncTaskService {
}
requestRisk.enforce(new ApiRequestContext(userId, apiKeyId, ApiRequestProtocol.ASYNC_TASK,
requestedModelId, modelId, false, estimateInputUnits(input), 0, metadata, System.nanoTime()));
String upstreamModel = model.get("upstreamModel") instanceof String value ? value : null;
if (upstreamModel == null || upstreamModel.isBlank()) throw new ApiException("ASYNC_BINDING_MISSING", "模型缺少可冻结的上游执行标识", 503);
// Serialize the queue-capacity check with task insertion for this
// model. Without the transaction advisory lock, concurrent creates
// could all observe the same count and exceed maxQueuedPerModel.
@@ -198,8 +200,8 @@ public class AsyncTaskService {
OffsetDateTime now = now();
try {
jdbc.update(
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash()
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash, provider_id_snapshot, upstream_model_snapshot) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash(), model.get("providerId"), upstreamModel
);
} catch (RuntimeException exception) {
releaseWallet(userId, reservedAmount);
@@ -253,7 +255,7 @@ public class AsyncTaskService {
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
);
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model FROM model_tasks t WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
for (Map<String,Object> row : pending) pollProviderTask(row);
dispatchQueuedTasks();
}
@@ -314,10 +316,10 @@ public class AsyncTaskService {
// workers reliably.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
String sql = """
SELECT t.id, t.user_id, t.api_key_id, t.model_id, m.provider_id, m.upstream_model, t.task_type, t.prompt, t.input
SELECT t.id, t.user_id, t.api_key_id, t.model_id, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model, t.task_type, t.prompt, t.input
FROM model_tasks t
JOIN model_catalog m ON m.id=t.model_id
JOIN model_providers p ON p.id=m.provider_id
JOIN model_providers p ON p.id=t.provider_id_snapshot
WHERE t.status='queued'
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
@@ -373,35 +375,7 @@ public class AsyncTaskService {
}
private void finishTask(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
// The terminal state and wallet change must commit together. Otherwise a
// crash after marking failed/succeeded leaves a hold that polling never
// revisits. The conditional update also prevents duplicate settlement.
inTransaction(() -> {
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
// A late failure must not override a durably observed success whose settlement is pending.
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
AsyncPriceSchedule.Bill bill=null;
String finalCode=code,finalMessage=message;
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
try { bill=AsyncPriceSchedule.bill(readMap(String.valueOf(task.get("pricing_snapshot"))),result); }
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
}
int changed = jdbc.update(
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
status, progress, result == null ? null : writeJson(result),
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
if (changed == 0) return null;
if ("TASK_USAGE_PENDING".equals(finalCode)) {
releaseTaskInTransaction(taskId);
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?",taskId);
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'",writeJson(usageOnly(result)),taskId);
}
else if ("succeeded".equals(status)) settleTaskInTransaction(taskId,bill);
else releaseTaskInTransaction(taskId);
usage.updateTask(taskId, status, result, finalCode, finalMessage);
return null;
});
settlements.finish(taskId, status, progress, result, code, message);
}
private Map<String, Object> requireKey(String authorization) {
@@ -476,6 +450,10 @@ public class AsyncTaskService {
private void pollProviderTask(Map<String,Object> row) {
String taskId = String.valueOf(row.get("id"));
if (row.get("provider_id") == null || row.get("upstream_model") == null) {
outcomes.requireReconciliation(taskId, "ASYNC_BINDING_MISSING", "历史任务缺少可证明的原始执行路由;冻结额度保留,请根据上游记录核账", false);
return;
}
OffsetDateTime deadline = taskTimestamp(row.get("provider_deadline_at"));
if (deadline == null) {
// Defensive fallback for pre-migration tasks. The migration also
@@ -622,42 +600,10 @@ public class AsyncTaskService {
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, userId);
}
private void settleTaskInTransaction(String taskId,AsyncPriceSchedule.Bill bill) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
BigDecimal amount = bill==null ? reserved : bill.amount();
policies.release(taskId);
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
if (amount == null || task.get("cost_amount") != null) return;
UUID userId = (UUID) task.get("user_id");
Map<String,Object> wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
BigDecimal before = (BigDecimal) wallet.get("balance");
BigDecimal after = before.subtract(amount);
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
if(amount.signum()>0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)", UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
if(bill!=null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",bill.input(),bill.output(),bill.input(),bill.output(),bill.imageInput(),taskId);
}
private void releaseTaskInTransaction(String taskId) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
policies.release(taskId);
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) {
releaseWallet((UUID) task.get("user_id"), amount);
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
}
}
private <T> T inTransaction(Supplier<T> work) {
return transactions == null ? work.get() : transactions.execute(status -> work.get());
}
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
private void putIfPresent(Map<String,Object> target, String name, ResultSet resultSet, String column) throws SQLException {
Object value = resultSet.getObject(column);
if (value != null) target.put(name, value);
@@ -0,0 +1,21 @@
package com.moyi.controlplane.task;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.UUID;
/** Administrative evidence and optimistic-concurrency preconditions, never a raw provider key. */
public record TaskReconciliationCommand(
UUID requestId,
String action,
OffsetDateTime expectedUpdatedAt,
String evidenceReference,
String note,
String providerTaskId,
String credentialFingerprint,
BigDecimal expectedCost,
Map<String,Object> result
) {
@Override public String toString() { return "TaskReconciliationCommand[requestId=" + requestId + ", action=" + action + "]"; }
}
@@ -0,0 +1,42 @@
package com.moyi.controlplane.task;
import com.moyi.controlplane.auth.AuthController;
import com.moyi.controlplane.common.ApiException;
import jakarta.servlet.http.HttpServletRequest;
import java.util.Map;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/admin/task-reconciliation")
public class TaskReconciliationController {
private final TaskReconciliationService reconciliations;
public TaskReconciliationController(TaskReconciliationService reconciliations) { this.reconciliations = reconciliations; }
@GetMapping
public Map<String,Object> list(@RequestParam(defaultValue="1") int page, HttpServletRequest request) {
return reconciliations.list(AuthController.sessionToken(request), page);
}
@GetMapping("/{taskId}")
public Map<String,Object> detail(@PathVariable String taskId, HttpServletRequest request) {
return reconciliations.detail(AuthController.sessionToken(request), taskId);
}
@PostMapping(value="/{taskId}", consumes="application/json")
public Map<String,Object> reconcile(@PathVariable String taskId, @RequestBody TaskReconciliationCommand command, HttpServletRequest request) {
// A custom header + JSON requires CORS preflight; the header is a CSRF barrier, not authentication.
if (!"console".equals(request.getHeader("X-Moyi-Source"))) throw new ApiException("CONSOLE_REQUEST_REQUIRED", "请通过管理控制台发起核账", 403);
return reconciliations.reconcile(AuthController.sessionToken(request), taskId, command, AuthController.meta(request));
}
@PostMapping(value="/{taskId}/quote", consumes="application/json")
public Map<String,Object> quote(@PathVariable String taskId, @RequestBody Map<String,Object> result, HttpServletRequest request) {
return reconciliations.quote(AuthController.sessionToken(request), taskId, result);
}
}
@@ -0,0 +1,64 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import java.math.BigDecimal;
import java.net.URI;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
/** Only normalized assets and numeric usage may enter a manual completion, never raw provider payloads. */
final class TaskReconciliationResult {
private static final Set<String> ASSETS = Set.of("url", "modelUrl", "videoUrl", "imageUrl", "audioUrl", "previewUrl");
private static final Set<String> OPENAI = Set.of("input_tokens", "output_tokens", "total_tokens", "cached_tokens", "input_tokens_details");
private static final Set<String> DETAILS = Set.of("text_tokens", "image_tokens", "cached_tokens", "cached_tokens_details");
private static final Set<String> CACHED = Set.of("text_tokens", "image_tokens");
private static final Set<String> GEMINI = Set.of("promptTokenCount", "candidatesTokenCount", "thoughtsTokenCount", "cachedContentTokenCount", "totalTokenCount");
static Map<String,Object> normalize(Map<String,Object> input, ObjectMapper json) {
if (input == null) throw invalid();
try { if (json.writeValueAsBytes(input).length > 32_768) throw invalid(); }
catch (JsonProcessingException exception) { throw invalid(); }
var result = new LinkedHashMap<String,Object>();
for (var entry : input.entrySet()) {
String field = entry.getKey();
Object value = entry.getValue();
if (ASSETS.contains(field)) {
if (!(value instanceof String text) || text.length() > 4096 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
try {
URI uri = URI.create(text);
if (!("https".equalsIgnoreCase(uri.getScheme()) || "http".equalsIgnoreCase(uri.getScheme())) || uri.getHost() == null || uri.getUserInfo() != null) throw invalid();
} catch (IllegalArgumentException exception) { throw invalid(); }
result.put(field, text);
} else if (Set.of("format", "mimeType").contains(field)) {
if (!(value instanceof String text) || text.isBlank() || text.length() > 128 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
result.put(field, text);
} else if ("usage".equals(field)) result.put(field, numericMap(value, OPENAI));
else if ("usageMetadata".equals(field)) result.put(field, numericMap(value, GEMINI));
else throw invalid();
}
return result;
}
private static Map<String,Object> numericMap(Object value, Set<String> allowed) {
if (!(value instanceof Map<?,?> map)) throw invalid();
var result = new LinkedHashMap<String,Object>();
for (var entry : map.entrySet()) {
if (!(entry.getKey() instanceof String name) || !allowed.contains(name)) throw invalid();
if ("input_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), DETAILS));
else if ("cached_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), CACHED));
else {
try {
long count = new BigDecimal(String.valueOf(entry.getValue())).longValueExact();
if (count < 0 || count > 100_000_000) throw invalid();
result.put(name, count);
} catch (NumberFormatException | ArithmeticException exception) { throw invalid(); }
}
}
return result;
}
private static ApiException invalid() { return new ApiException("INVALID_RECONCILIATION_RESULT", "只接受有界的 HTTP(S) 结果地址和规范化数值用量,不接受原始上游响应或密钥", 422); }
}
@@ -0,0 +1,216 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.auth.AuthService;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
/** Evidence-backed administrative decisions; no generation, arbitrary balance adjustment or transparent replay. */
@Service
public class TaskReconciliationService {
private final JdbcTemplate jdbc;
private final ObjectMapper json;
private final AuthService auth;
private final TaskSettlementService settlements;
private final AsyncProviderClient provider;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final TransactionTemplate transactions;
private final long timeoutMs;
// List/detail must not load prompts, original inputs or potentially large generated media.
private static final String TASK_COLUMNS = "SELECT t.id,t.user_id,t.api_key_id,t.model_id,t.task_type,t.status,t.updated_at,t.reserved_amount,t.cost_amount,t.provider_id_snapshot,t.upstream_model_snapshot,t.provider_task_id,t.provider_key_fingerprint,t.completion_status,t.pricing_snapshot::text AS frozen_pricing,t.error::text AS error FROM model_tasks t";
public TaskReconciliationService(JdbcTemplate jdbc, ObjectMapper json, AuthService auth,
TaskSettlementService settlements, AsyncProviderClient provider, GatewayGuard guard,
UsageAuditService usage, PlatformTransactionManager manager,
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long timeoutMs) {
this.jdbc = jdbc; this.json = json; this.auth = auth; this.settlements = settlements;
this.provider = provider; this.guard = guard; this.usage = usage;
this.transactions = new TransactionTemplate(manager);
if (timeoutMs < 1000) throw new IllegalArgumentException("Polling deadline must be positive");
this.timeoutMs = timeoutMs;
}
public Map<String,Object> list(String token, int requestedPage) {
auth.requireAdmin(token);
int page = Math.max(1, Math.min(100000, requestedPage));
return Map.of("records", jdbc.queryForList(TASK_COLUMNS + " WHERE t.status='reconciling' ORDER BY t.reconciliation_required_at NULLS FIRST,t.id LIMIT 50 OFFSET ?", (page - 1) * 50).stream().map(this::view).toList(),
"total", jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE status='reconciling'", Long.class), "page", page);
}
public Map<String,Object> detail(String token, String taskId) {
auth.requireAdmin(token);
Map<String,Object> task = task(taskId, false);
var history = jdbc.queryForList("SELECT request_id,actor_user_id,action,evidence_reference,note,before_state::text AS before_state,after_state::text AS after_state,created_at FROM task_reconciliations WHERE task_id=? ORDER BY created_at DESC LIMIT 50", taskId);
var records = history.stream().map(row -> {
var item = new LinkedHashMap<String,Object>();
for (String field : List.of("request_id", "actor_user_id", "action", "evidence_reference", "note", "created_at")) item.put(field, row.get(field));
item.put("before_state", decode(row.get("before_state")));
item.put("after_state", decode(row.get("after_state")));
return item;
}).toList();
return Map.of("task", view(task), "history", records);
}
public Map<String,Object> quote(String token, String taskId, Map<String,Object> input) {
auth.requireAdmin(token);
var task = task(taskId, false);
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可核账状态,请刷新");
var result = TaskReconciliationResult.normalize(input, json);
return Map.of("expectedCost", confirmedCharge(task, result).toPlainString(),
"expectedUpdatedAt", timestamp(task.get("updated_at")).toInstant().toString());
}
public Map<String,Object> reconcile(String token, String taskId, TaskReconciliationCommand command, AuthService.RequestMeta meta) {
var actor = auth.requireAdmin(token); // Authorization is required even for replay.
validate(command);
Map<String,Object> result = "confirm_succeeded".equals(command.action()) ? TaskReconciliationResult.normalize(command.result(), json) : null;
String requestHash = requestHash(command, taskId, result);
return transactions.execute(status -> {
// Keep role/status revocation ordered with this financial decision, not a stale pre-check.
if (jdbc.queryForList("SELECT id FROM users WHERE id=? AND status='active' AND role IN ('admin','super_admin') FOR SHARE", actor.id()).isEmpty()) throw new ApiException("FORBIDDEN", "管理员权限已变化,请重新登录", 403);
auth.requireAdmin(token); // Recheck session revocation/expiry after entering the transaction.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))", "moyi:task-reconciliation:" + command.requestId());
var prior = jdbc.queryForList("SELECT actor_user_id,request_hash,after_state::text AS after_state FROM task_reconciliations WHERE request_id=?", command.requestId());
if (!prior.isEmpty()) {
var receipt = prior.getFirst();
if (!actor.id().equals(receipt.get("actor_user_id")) || !requestHash.equals(receipt.get("request_hash"))) throw conflict("此核账请求编号已用于不同的操作");
return Map.of("task", decode(receipt.get("after_state")), "requestId", command.requestId(), "replayed", true);
}
var task = task(taskId, true);
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可人工核账状态,或已有可信终态;请刷新");
if (!timestamp(task.get("updated_at")).toInstant().equals(command.expectedUpdatedAt().toInstant())) throw conflict("任务已被其他操作更新,请刷新后重新核对");
var wallets = jdbc.queryForList("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", task.get("user_id"));
if (wallets.isEmpty() || ((BigDecimal) wallets.getFirst().get("reserved_balance")).compareTo((BigDecimal) task.get("reserved_amount")) < 0) throw conflict("任务冻结额度与钱包不一致,请先排查账本,不允许直接覆盖");
Map<String,Object> before = view(task);
if ("resume_polling".equals(command.action())) resume(task, command);
else {
if ("confirm_succeeded".equals(command.action())) {
BigDecimal charge = confirmedCharge(task, result);
if (charge == null || charge.compareTo(command.expectedCost()) != 0) throw conflict("确认金额与任务原价格快照及实际用量不一致");
}
String target = "confirm_succeeded".equals(command.action()) ? "succeeded" : "failed";
String code = "failed".equals(target) ? "TASK_RECONCILED_FAILED" : null;
String message = "failed".equals(target) ? "管理员已根据上游记录确认任务失败,冻结额度已释放" : null;
// The observed outcome, shared wallet transaction and both audit receipts commit together.
jdbc.update("UPDATE model_tasks SET completion_status=? WHERE id=?", target, taskId);
settlements.finish(taskId, target, "succeeded".equals(target) ? 100 : 0, result, code, message);
}
Map<String,Object> after = view(task(taskId, false));
jdbc.update("INSERT INTO task_reconciliations(request_id,task_id,actor_user_id,action,request_hash,evidence_reference,note,before_state,after_state) VALUES (?,?,?,?,?,?,?,CAST(? AS jsonb),CAST(? AS jsonb))",
command.requestId(), taskId, actor.id(), command.action(), requestHash, command.evidenceReference().trim(), command.note().trim(), encode(before), encode(after));
auth.audit("task.reconciled", actor.id(), meta, Map.of("taskId", taskId, "requestId", command.requestId(), "action", command.action(), "evidenceReference", command.evidenceReference().trim()));
return Map.of("task", after, "requestId", command.requestId(), "replayed", false);
});
}
private BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
try { return settlements.confirmedCharge(task, result); }
catch (ApiException exception) {
if ("TASK_USAGE_PENDING".equals(exception.getCode())) throw new ApiException("TASK_USAGE_REQUIRED", "成功核账必须提供完整可核验用量;任务与冻结额度保持不变", 422);
throw exception;
}
}
private void resume(Map<String,Object> task, TaskReconciliationCommand command) {
String originalId = (String) task.get("provider_task_id");
if (originalId != null && !originalId.equals(command.providerTaskId())) throw conflict("不能将已受理任务改绑到另一个上游编号");
String fingerprint = (String) task.get("provider_key_fingerprint");
if (fingerprint != null && !fingerprint.equals(command.credentialFingerprint())) throw conflict("必须继续使用创建任务时的账号指纹,不能切换账号");
if (task.get("provider_id_snapshot") == null || task.get("upstream_model_snapshot") == null) throw conflict("历史任务原执行路由无法证明,禁止猜测后恢复查询;请核实终态");
provider.requirePollableBinding(String.valueOf(task.get("provider_id_snapshot")), String.valueOf(task.get("model_id")),
String.valueOf(task.get("upstream_model_snapshot")), String.valueOf(task.get("task_type")), command.credentialFingerprint());
String id = String.valueOf(task.get("id"));
if (!guard.tryTaskUserSlot((java.util.UUID) task.get("api_key_id"), id, timeoutMs + 180000)) throw new ApiException("TASK_USER_BUSY", "用户当前并发已满,尚未恢复轮询,请稍后重试", 429);
jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=now() WHERE id=?",
command.providerTaskId(), command.credentialFingerprint(), OffsetDateTime.now(ZoneOffset.UTC).plus(java.time.Duration.ofMillis(timeoutMs)), id);
usage.updateTask(id, "running", null, null, null);
}
private void validate(TaskReconciliationCommand command) {
if (command == null || command.requestId() == null || command.expectedUpdatedAt() == null) throw new ApiException("RECONCILIATION_PRECONDITION_REQUIRED", "必须提供操作编号和任务版本", 422);
if (command.action() == null || !List.of("resume_polling", "confirm_failed", "confirm_succeeded").contains(command.action())) throw new ApiException("INVALID_RECONCILIATION_ACTION", "核账操作无效", 422);
text(command.evidenceReference(), 3, 500); text(command.note(), 10, 2000);
if ("resume_polling".equals(command.action())) {
text(command.providerTaskId(), 1, 160);
if (!command.providerTaskId().matches("[A-Za-z0-9][A-Za-z0-9._:-]{0,159}")) throw new ApiException("INVALID_PROVIDER_TASK_ID", "上游任务编号不能包含路径、查询串或空白", 422);
if (command.credentialFingerprint() == null || !command.credentialFingerprint().matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "必须提供 SHA-256 账号指纹,不能填写 API Key", 422);
if (command.result() != null || command.expectedCost() != null) throw conflict("恢复轮询不能同时修改结果或扣费金额");
} else {
if (command.providerTaskId() != null || command.credentialFingerprint() != null) throw conflict("终态核账不能改绑上游编号或账号");
if ("confirm_succeeded".equals(command.action())) {
if (command.expectedCost() == null || command.expectedCost().signum() < 0 || command.expectedCost().precision() > 20) throw new ApiException("EXPECTED_COST_REQUIRED", "请确认按原价格快照核算的非负扣费金额", 422);
try { command.expectedCost().setScale(8, RoundingMode.UNNECESSARY); }
catch (ArithmeticException exception) { throw new ApiException("INVALID_EXPECTED_COST", "确认金额最多 8 位小数", 422); }
} else if (command.result() != null || command.expectedCost() != null) throw conflict("确认失败不能附带成功结果或扣费金额");
}
}
private void text(String value, int min, int max) {
if (value == null || value.trim().length() < min || value.trim().length() > max || value.chars().anyMatch(Character::isISOControl)) throw new ApiException("RECONCILIATION_EVIDENCE_REQUIRED", "必须填写有效的上游证据引用与 10–2000 字核账说明;不得包含控制字符", 422);
}
private String requestHash(TaskReconciliationCommand command, String taskId, Map<String,Object> result) {
var body = new LinkedHashMap<String,Object>();
body.put("taskId", taskId); body.put("action", command.action()); body.put("expectedUpdatedAt", command.expectedUpdatedAt().toInstant().toString());
body.put("evidenceReference", command.evidenceReference().trim()); body.put("note", command.note().trim());
body.put("providerTaskId", command.providerTaskId()); body.put("credentialFingerprint", command.credentialFingerprint());
body.put("expectedCost", command.expectedCost()); body.put("result", result);
return TaskCreationIdentity.of(json, command.requestId().toString(), "task-reconciliation", body).requestHash();
}
private Map<String,Object> task(String taskId, boolean lock) {
if (taskId == null || taskId.isBlank() || taskId.length() > 160) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
var rows = jdbc.queryForList(TASK_COLUMNS + " WHERE t.id=?" + (lock ? " FOR UPDATE" : ""), taskId);
if (rows.isEmpty()) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
var task = rows.getFirst();
task.put("pricing_snapshot", task.get("frozen_pricing"));
return task;
}
private Map<String,Object> view(Map<String,Object> task) {
var view = new LinkedHashMap<String,Object>();
view.put("id", task.get("id")); view.put("userId", task.get("user_id")); view.put("modelId", task.get("model_id"));
view.put("taskType", task.get("task_type")); view.put("status", task.get("status"));
view.put("updatedAt", timestamp(task.get("updated_at")).toInstant().toString());
view.put("reservedAmount", ((BigDecimal) task.get("reserved_amount")).toPlainString());
view.put("costAmount", task.get("cost_amount") == null ? null : ((BigDecimal) task.get("cost_amount")).toPlainString());
view.put("providerId", task.get("provider_id_snapshot")); view.put("upstreamModel", task.get("upstream_model_snapshot"));
view.put("providerTaskId", task.get("provider_task_id")); view.put("credentialFingerprint", task.get("provider_key_fingerprint"));
view.put("completionStatus", task.get("completion_status")); view.put("requiresReconciliation", "reconciling".equals(task.get("status")));
view.put("pricing", task.get("frozen_pricing") == null ? null : decode(task.get("frozen_pricing")));
view.put("error", task.get("error") == null ? null : decode(task.get("error")));
return view;
}
private OffsetDateTime timestamp(Object value) {
if (value instanceof OffsetDateTime time) return time;
if (value instanceof java.sql.Timestamp time) return time.toInstant().atOffset(ZoneOffset.UTC);
throw new IllegalStateException("Persisted task timestamp missing");
}
private Map<String,Object> decode(Object value) {
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Persisted reconciliation record cannot be read"); }
}
private String encode(Object value) {
try { return json.writeValueAsString(value); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Reconciliation record cannot be encoded"); }
}
private ApiException conflict(String message) { return new ApiException("RECONCILIATION_CONFLICT", message, 409); }
}
@@ -0,0 +1,125 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.EnterprisePolicies;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
/** The one task/wallet/usage settlement transaction, shared by workers and audited reconciliation. */
@Service
public class TaskSettlementService {
private final JdbcTemplate jdbc;
private final ObjectMapper json;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final EnterprisePolicies policies;
private final TransactionTemplate transactions;
public TaskSettlementService(JdbcTemplate jdbc, ObjectMapper json, GatewayGuard guard,
UsageAuditService usage, PlatformTransactionManager manager) {
this.jdbc = jdbc;
this.json = json;
this.guard = guard;
this.usage = usage;
this.policies = new EnterprisePolicies(jdbc, json);
this.transactions = manager == null ? null : new TransactionTemplate(manager);
}
void finish(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal task state required");
transaction(() -> {
var task = jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE", taskId);
if (!List.of("queued", "running", "reconciling").contains(task.get("status"))) return null;
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
AsyncPriceSchedule.Bill bill = null;
String finalCode = code, finalMessage = message;
if ("succeeded".equals(status) && task.get("pricing_snapshot") != null) {
try { bill = AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result); }
catch (ApiException exception) {
finalCode = "TASK_USAGE_PENDING";
finalMessage = "生成已完成,但上游计费用量未能核对;暂不扣费,等待核账";
}
}
jdbc.update("UPDATE model_tasks SET status=?,progress=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),reconciliation_required_at=NULL,updated_at=now(),completed_at=now() WHERE id=?",
status, progress, encode(result), finalCode == null ? null : encode(Map.of("code", finalCode, "message", finalMessage)), taskId);
if ("TASK_USAGE_PENDING".equals(finalCode)) {
release(taskId);
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?", taskId);
var observed = new LinkedHashMap<String,Object>();
if (result != null) for (String field : List.of("usage", "usageMetadata")) if (result.get(field) instanceof Map<?,?> value) observed.put(field, value);
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'", encode(observed), taskId);
} else if ("succeeded".equals(status)) settle(taskId, bill);
else release(taskId);
usage.updateTask(taskId, status, result, finalCode, finalMessage);
return null;
});
}
/** Strict calculation for explicit administrator confirmation: missing usage is an error, not a refund. */
BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
if (task.get("pricing_snapshot") == null) return (BigDecimal) task.get("reserved_amount");
return AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result).amount();
}
private void settle(String taskId, AsyncPriceSchedule.Bill bill) {
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
BigDecimal amount = bill == null ? reserved : bill.amount();
releasePolicies(taskId, task);
if (amount == null || task.get("cost_amount") != null) return;
UUID userId = (UUID) task.get("user_id");
var wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
BigDecimal before = (BigDecimal) wallet.get("balance"), after = before.subtract(amount);
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
if (amount.signum() > 0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)",
UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
if (bill != null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",
bill.input(), bill.output(), bill.input(), bill.output(), bill.imageInput(), taskId);
}
private void release(String taskId) {
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
releasePolicies(taskId, task);
if (amount != null && task.get("cost_amount") == null) {
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, task.get("user_id"));
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
}
}
private void releasePolicies(String taskId, Map<String,Object> task) {
policies.release(taskId);
guard.releaseTaskUserSlot((UUID) task.get("user_id"), taskId);
}
private Map<String,Object> pricing(Object value) {
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
catch (JsonProcessingException exception) { throw new ApiException("INVALID_PRICE_SNAPSHOT", "任务价格快照无法核验", 409); }
}
private String encode(Object value) {
if (value == null) return null;
try { return json.writeValueAsString(value); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Task settlement cannot be encoded"); }
}
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
private <T> T transaction(Supplier<T> work) { return transactions == null ? work.get() : transactions.execute(status -> work.get()); }
}
@@ -0,0 +1,23 @@
-- Freeze route identity without copying credentials or mutable provider configuration.
ALTER TABLE model_tasks ADD COLUMN provider_id_snapshot varchar(64) REFERENCES model_providers(id);
ALTER TABLE model_tasks ADD COLUMN upstream_model_snapshot varchar(160);
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_execution_binding_check CHECK (
(provider_id_snapshot IS NULL AND upstream_model_snapshot IS NULL)
OR (provider_id_snapshot IS NOT NULL AND upstream_model_snapshot IS NOT NULL
AND length(trim(provider_id_snapshot)) > 0 AND length(trim(upstream_model_snapshot)) > 0)
);
-- Never guess the original route of an already-submitted legacy task.
-- Unsubmitted queued work can safely freeze its route at migration time.
UPDATE model_tasks t SET provider_id_snapshot=m.provider_id, upstream_model_snapshot=m.upstream_model
FROM model_catalog m
WHERE t.model_id=m.id AND t.status='queued' AND t.provider_task_id IS NULL;
COMMENT ON COLUMN model_tasks.provider_id_snapshot IS 'Provider identity at admission; NULL for ambiguous legacy accepted tasks';
-- New proven bindings cannot associate the same provider/account task with two local wallets.
-- Legacy accepted rows are deliberately not guessed/backfilled, so this index does not rewrite their history.
CREATE UNIQUE INDEX model_tasks_proven_upstream_identity_idx
ON model_tasks(provider_id_snapshot,provider_key_fingerprint,provider_task_id)
WHERE provider_id_snapshot IS NOT NULL AND provider_key_fingerprint IS NOT NULL
AND provider_task_id IS NOT NULL AND provider_task_id <> 'mock';
@@ -0,0 +1,20 @@
-- Append-only operation receipts; no prompts, generated media or raw credentials.
CREATE TABLE task_reconciliations (
request_id uuid PRIMARY KEY,
task_id varchar(160) NOT NULL REFERENCES model_tasks(id),
actor_user_id uuid NOT NULL REFERENCES users(id),
action varchar(24) NOT NULL CHECK (action IN ('resume_polling','confirm_failed','confirm_succeeded')),
request_hash varchar(64) NOT NULL CHECK (request_hash ~ '^[0-9a-f]{64}$'),
evidence_reference varchar(500) NOT NULL CHECK (length(trim(evidence_reference)) BETWEEN 3 AND 500),
note varchar(2000) NOT NULL CHECK (length(trim(note)) BETWEEN 10 AND 2000),
before_state jsonb NOT NULL,
after_state jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX task_reconciliations_task_idx ON task_reconciliations(task_id,created_at DESC);
CREATE FUNCTION protect_task_reconciliation_history() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
RAISE EXCEPTION 'Task reconciliation history is append-only';
END $$;
CREATE TRIGGER task_reconciliation_history_immutable BEFORE UPDATE OR DELETE ON task_reconciliations
FOR EACH ROW EXECUTE FUNCTION protect_task_reconciliation_history();
@@ -0,0 +1,14 @@
-- A resumed task renews its own lease instead of consuming a second slot.
local time = redis.call('TIME')
local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
local existing = redis.call('ZSCORE', KEYS[1], ARGV[3])
if not existing and redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[1]) then
return 0
end
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[2]), ARGV[3])
if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then
redis.call('PEXPIRE', KEYS[1], ARGV[2])
end
if existing then return 2 end
return 1
@@ -0,0 +1,64 @@
package com.moyi.controlplane.gateway;
import java.util.UUID;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
@EnabledIfEnvironmentVariable(named="MOYI_GATEWAY_TEST_REDIS_PORT", matches="[0-9]{2,5}")
class TaskLeaseRedisTest {
private LettuceConnectionFactory factory;
private StringRedisTemplate redis;
private GatewayGuard guard;
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID();
private final String prefix = "moyi:test:" + UUID.randomUUID() + ":";
private String bucket() { return prefix + "gateway:slots:user:" + userId; }
@BeforeEach void setup() {
factory = new LettuceConnectionFactory("127.0.0.1", Integer.parseInt(System.getenv("MOYI_GATEWAY_TEST_REDIS_PORT")));
factory.afterPropertiesSet(); factory.start();
redis = new StringRedisTemplate(factory); redis.afterPropertiesSet();
guard = new GatewayGuard(redis, 60, 300, 4, prefix);
var policies = mock(EnterprisePolicies.class);
when(policies.taskLimit(keyId)).thenReturn(new EnterprisePolicies.UserLimit(userId, 120, 1));
guard.setEnterprisePolicies(policies);
}
@AfterEach void cleanup() {
if (TransactionSynchronizationManager.isSynchronizationActive()) TransactionSynchronizationManager.clearSynchronization();
if (redis != null) redis.delete(bucket());
if (factory != null) factory.destroy();
}
@Test void renewalDoesNotConsumeAnotherSlotAndDifferentTaskIsBlocked() {
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
assertEquals(1L, redis.opsForZSet().zCard(bucket()));
assertFalse(guard.tryTaskUserSlot(keyId, "different-task", 180000));
assertTrue(redis.getExpire(bucket(), java.util.concurrent.TimeUnit.MILLISECONDS) > 250000);
}
@Test void rolledBackRenewalKeepsThePreexistingLease() {
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
TransactionSynchronizationManager.initSynchronization();
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
TransactionSynchronizationManager.clearSynchronization();
assertNotNull(redis.opsForZSet().score(bucket(), "original-task"));
}
@Test void rolledBackNewClaimRemovesOnlyItsOwnLease() {
TransactionSynchronizationManager.initSynchronization();
assertTrue(guard.tryTaskUserSlot(keyId, "new-task", 180000));
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
TransactionSynchronizationManager.clearSynchronization();
assertNull(redis.opsForZSet().score(bucket(), "new-task"));
}
}
@@ -289,6 +289,15 @@ class AsyncProviderClientTest {
} finally { server.stop(0); }
}
@Test void pollingBindingValidationChecksTheExistingCredentialWithoutAnHttpCall() throws Exception {
var client = clientFor("http://127.0.0.1:1");
String fingerprint = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256").digest("test-key".getBytes(StandardCharsets.UTF_8)));
org.junit.jupiter.api.Assertions.assertDoesNotThrow(() -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", fingerprint));
assertEquals("UPSTREAM_KEY_NOT_AVAILABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "0".repeat(64))).getCode());
assertEquals("TASK_NOT_POLLABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "image", fingerprint)).getCode());
assertEquals("INVALID_CREDENTIAL_FINGERPRINT", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "test-key")).getCode());
}
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
var client = clientFor("http://127.0.0.1:1");
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
@@ -65,9 +65,11 @@ class AsyncTaskPostgresTest {
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
// This fixture knows its original binding; a migration must not guess it for real legacy jobs.
jdbc.update("UPDATE model_tasks SET provider_id_snapshot='provider-vast',upstream_model_snapshot='v3.1-20260211' WHERE id=?", taskId);
when(catalog.keyOwnerId(keyId)).thenReturn(userId);
when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true);
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "upstreamModel", "v3.1-20260211", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
}
@AfterEach void cleanup() {
@@ -97,6 +99,34 @@ class AsyncTaskPostgresTest {
return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal());
}
@Test void queuedTaskRetainsAdmittedExecutionBindingWhenRouteIsEdited() {
String created = String.valueOf(create("frozen-queue", Map.of("model", "model-mock-3d", "prompt", "chair")).get("id"));
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
jdbc.update("UPDATE model_tasks SET updated_at=now() WHERE id=?", taskId);
when(provider.submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap())).thenReturn(Map.of("id", "original-route-task"));
tasks.completeMockTasks();
assertEquals("original-route-task", jdbc.queryForObject("SELECT provider_task_id FROM model_tasks WHERE id=?", String.class, created));
verify(provider).submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap());
}
@Test void acceptedTaskPollsFrozenProviderRatherThanEditedCatalog() {
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null)).thenReturn(Map.of("status", "running", "progress", 55));
tasks.completeMockTasks();
assertEquals("running", status());
assertEquals(55, jdbc.queryForObject("SELECT progress FROM model_tasks WHERE id=?", Integer.class, taskId));
verify(provider).poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null);
}
@Test void acceptedLegacyTaskWithoutProofOfBindingIsNotPolledOnAGuessedRoute() {
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL,updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
tasks.completeMockTasks();
assertEquals("reconciling", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
verifyNoInteractions(provider);
}
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
var first = create("client-operation-1", input);
@@ -0,0 +1,295 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.moyi.controlplane.auth.AuthService;
import com.moyi.controlplane.auth.PasswordHasher;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.common.ApiExceptionHandler;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import jakarta.servlet.http.Cookie;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.dao.DataAccessException;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
@EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+")
class TaskReconciliationPostgresTest {
private final ObjectMapper json = new ObjectMapper().findAndRegisterModules().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID(), adminId = UUID.randomUUID(), secondAdminId = UUID.randomUUID();
private final String taskId = "reconcile-" + UUID.randomUUID();
private final String fingerprint = "c".repeat(64);
private final AuthService.RequestMeta meta = new AuthService.RequestMeta("127.0.0.1", "isolated-reconciliation-test");
private DriverManagerDataSource base;
private JdbcTemplate jdbc;
private String schema;
private AuthService auth;
private GatewayGuard guard;
private AsyncProviderClient provider;
private TaskReconciliationService service;
private MockMvc http;
@BeforeEach void setup() {
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
schema = "reconciliation_test_" + UUID.randomUUID().toString().replace("-", "");
base = new DriverManagerDataSource(url, "moyi_role_test", "");
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
var source = new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", "");
jdbc = new JdbcTemplate(source);
var manager = new DataSourceTransactionManager(source);
for (var entry : Map.of(userId, "user", adminId, "admin", secondAdminId, "admin").entrySet()) {
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name,role) VALUES (?,?,'username','not-loginable','Test',?)", entry.getKey(), entry.getKey().toString(), entry.getValue());
}
session("admin-session", adminId); session("second-admin-session", secondAdminId); session("user-session", userId);
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-key',?)", keyId, userId, "b".repeat(64));
jdbc.update("INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)", userId);
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot,pricing_snapshot,reconciliation_required_at) VALUES (?,?,?,'model-mock-3d','3d','reconciling','private prompt',1.44,'original-provider-task',?,'provider-vast','v3.1-20260211','{\"unit\":\"CNY / task\",\"price\":1.44}'::jsonb,now())", taskId, userId, keyId, fingerprint);
jdbc.update("INSERT INTO gateway_budget_holds(request_id,user_id,key_id,amount) VALUES (?,?,?,1.44)", taskId, userId, keyId);
var usage = new UsageAuditService(jdbc, json);
usage.recordTask(userId, keyId, "model-mock-3d", "provider-vast", taskId, "reconciling", Map.of("prompt", "private prompt"), null, "UNKNOWN", "pending");
auth = spy(new AuthService(jdbc, mock(PasswordHasher.class), json, 168));
guard = mock(GatewayGuard.class);
when(guard.tryTaskUserSlot(any(), anyString(), anyLong())).thenReturn(true);
provider = mock(AsyncProviderClient.class);
var settlements = new TaskSettlementService(jdbc, json, guard, usage, manager);
service = new TaskReconciliationService(jdbc, json, auth, settlements, provider, guard, usage, manager, 120_000);
http = MockMvcBuilders.standaloneSetup(new TaskReconciliationController(service))
.setControllerAdvice(new ApiExceptionHandler()).setMessageConverters(new MappingJackson2HttpMessageConverter(json)).build();
}
@AfterEach void cleanup() { if (base != null && schema != null) new JdbcTemplate(base).execute("DROP SCHEMA " + schema + " CASCADE"); }
private void session(String token, UUID user) {
jdbc.update("INSERT INTO auth_sessions(id,token_hash,user_id,expires_at) VALUES (?,?,?,now()+interval '1 hour')", UUID.randomUUID(), AuthService.hashSecret(token), user);
}
private OffsetDateTime version() { return jdbc.queryForObject("SELECT updated_at FROM model_tasks WHERE id=?", OffsetDateTime.class, taskId); }
private String state() { return jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, taskId); }
private BigDecimal wallet(String column) { return jdbc.queryForObject("SELECT " + column + " FROM user_wallets WHERE user_id=?", BigDecimal.class, userId); }
private int count(String table) { return jdbc.queryForObject("SELECT count(*) FROM " + table, Integer.class); }
private TaskReconciliationCommand command(String action) {
return new TaskReconciliationCommand(UUID.randomUUID(), action, version(), "provider-bill/test-001", "Verified against the provider task history and billing record",
"resume_polling".equals(action) ? "original-provider-task" : null,
"resume_polling".equals(action) ? fingerprint : null,
"confirm_succeeded".equals(action) ? new BigDecimal("1.44") : null,
"confirm_succeeded".equals(action) ? Map.of("url", "https://example.test/generated.glb") : null);
}
private Map<String,Object> reconcile(TaskReconciliationCommand command) { return service.reconcile("admin-session", taskId, command, meta); }
private void unchanged() {
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertEquals(0, count("wallet_entries")); assertEquals(0, count("task_reconciliations"));
assertEquals(1, count("gateway_budget_holds"));
}
@Test void realSessionsEnforceAdministratorAndCsrfBarrier() throws Exception {
var command = command("confirm_failed");
http.perform(get("/api/admin/task-reconciliation")).andExpect(status().isUnauthorized());
http.perform(get("/api/admin/task-reconciliation").cookie(new Cookie("moyi_session", "user-session"))).andExpect(status().isForbidden());
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "user-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
unchanged();
verifyNoInteractions(provider, guard);
}
@Test void failureConfirmationReleasesOnceAndReplayReturnsTheOperationReceipt() throws Exception {
var command = command("confirm_failed");
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command)))
.andExpect(status().isOk()).andExpect(jsonPath("$.task.status").value("failed")).andExpect(jsonPath("$.replayed").value(false));
assertEquals(true, reconcile(command).get("replayed"));
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertEquals(0, wallet("reserved_balance").signum());
assertEquals(0, count("wallet_entries")); assertEquals(0, count("gateway_budget_holds"));
assertEquals(1, count("task_reconciliations"));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM auth_audit_log WHERE event_type='task.reconciled'", Integer.class));
verifyNoInteractions(provider);
}
@Test void successUsesFrozenPriceAndDebitsExactlyOnce() {
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":99}'::jsonb WHERE id='model-mock-3d'");
var command = command("confirm_succeeded");
reconcile(command); reconcile(command);
assertEquals("succeeded", state());
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
assertEquals(0, wallet("reserved_balance").signum());
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
assertEquals(new BigDecimal("1.44000000"), jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId));
verifyNoInteractions(provider);
}
@Test void quoteIsReadOnlyAndUsesTheFrozenVersionNotEditableCatalogPrices() throws Exception {
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":999}'::jsonb WHERE id='model-mock-3d'");
http.perform(post("/api/admin/task-reconciliation/" + taskId + "/quote").cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content("{}"))
.andExpect(status().isOk()).andExpect(jsonPath("$.expectedCost").value("1.44"));
unchanged();
assertEquals(0, count("auth_audit_log"));
}
@Test void concurrentIdenticalOperationsShareOneReceiptAndOneDebit() throws Exception {
var command = command("confirm_succeeded");
var start = new CountDownLatch(1);
try (var executor = Executors.newFixedThreadPool(8)) {
var results = new ArrayList<java.util.concurrent.Future<Map<String,Object>>>();
for (int i = 0; i < 8; i++) results.add(executor.submit(() -> { assertTrue(start.await(10, TimeUnit.SECONDS)); return reconcile(command); }));
start.countDown();
int originals = 0;
for (var result : results) if (Boolean.FALSE.equals(result.get(30, TimeUnit.SECONDS).get("replayed"))) originals++;
assertEquals(1, originals);
}
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
}
@Test void concurrentDifferentDecisionsHaveExactlyOneWinner() throws Exception {
var success = command("confirm_succeeded"); var failure = command("confirm_failed");
var start = new CountDownLatch(1);
try (var executor = Executors.newFixedThreadPool(2)) {
var results = new ArrayList<java.util.concurrent.Future<Integer>>();
for (var command : List.of(success, failure)) results.add(executor.submit(() -> {
assertTrue(start.await(10, TimeUnit.SECONDS));
try { reconcile(command); return 200; } catch (ApiException exception) { return exception.getStatus(); }
}));
start.countDown();
assertEquals(List.of(200, 409), results.stream().map(f -> { try { return f.get(30, TimeUnit.SECONDS); } catch (Exception exception) { throw new AssertionError(exception); } }).sorted().toList());
}
assertEquals(1, count("task_reconciliations")); assertEquals(0, wallet("reserved_balance").signum());
assertEquals("succeeded".equals(state()) ? new BigDecimal("8.56000000") : new BigDecimal("10.00000000"), wallet("balance"));
}
@Test void operationIdCannotBeReusedForDifferentActionOrAnotherAdministrator() {
var command = command("confirm_failed"); reconcile(command);
var changed = new TaskReconciliationCommand(command.requestId(), "confirm_succeeded", command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.44"), Map.of());
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(changed)).getStatus());
assertEquals(409, assertThrows(ApiException.class, () -> service.reconcile("second-admin-session", taskId, command, meta)).getStatus());
assertEquals(1, count("task_reconciliations"));
}
@Test void staleVersionAndWrongExpectedAmountLeaveFinancialStateUnchanged() {
var command = command("confirm_succeeded");
var wrong = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.45"), command.result());
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(wrong)).getStatus()); unchanged();
jdbc.update("UPDATE model_tasks SET updated_at=updated_at+interval '1 second' WHERE id=?", taskId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command)).getStatus()); unchanged();
}
@Test void missingEvidenceCannotAuthorizeARefund() {
var command = command("confirm_failed");
var invalid = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), "", "short", null, null, null, null);
assertEquals(422, assertThrows(ApiException.class, () -> reconcile(invalid)).getStatus()); unchanged();
}
@Test void missingTokenUsageCannotReleaseTheReservationUnderASuccessLabel() {
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
assertEquals("TASK_USAGE_REQUIRED", assertThrows(ApiException.class, () -> reconcile(command("confirm_succeeded"))).getCode()); unchanged();
}
@Test void auditFailureRollsBackTaskWalletUsageAndReceiptTogether() {
var command = command("confirm_succeeded");
doThrow(new IllegalStateException("audit database unavailable")).when(auth).audit(eq("task.reconciled"), eq(adminId), any(), anyMap());
assertThrows(IllegalStateException.class, () -> reconcile(command)); unchanged();
assertNull(jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId));
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
}
@Test void resumeOnlyQueriesTheOriginalBindingWithoutChangingTheWallet() {
var command = command("resume_polling"); reconcile(command); reconcile(command);
assertEquals("running", state());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertEquals(1, count("gateway_budget_holds")); assertEquals(0, count("wallet_entries"));
assertEquals(1, count("task_reconciliations"));
verify(provider, times(1)).requirePollableBinding("provider-vast", "model-mock-3d", "v3.1-20260211", "3d", fingerprint);
verifyNoMoreInteractions(provider);
}
@Test void differentProviderTaskCannotBeAttachedToAnAcceptedJob() {
var command = command("resume_polling");
var other = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), "some-other-task", fingerprint, null, null);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(other)).getStatus()); unchanged(); verifyNoInteractions(provider);
}
@Test void sameProvenUpstreamTaskCannotBeAssociatedWithTwoLocalTasks() {
assertThrows(DataAccessException.class, () -> jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot) VALUES ('duplicate-local-task',?,?,'model-mock-3d','3d','running','',0,'original-provider-task',?,'provider-vast','v3.1-20260211')", userId, keyId, fingerprint));
unchanged();
}
@Test void validTokenUsageIsQuotedAndSettledOnceUsingTheFrozenSchedule() {
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
var result = Map.<String,Object>of("usage", Map.of("input_tokens", 100, "output_tokens", 100));
var quote = service.quote("admin-session", taskId, result);
assertEquals("0.00110000", quote.get("expectedCost"));
var command = new TaskReconciliationCommand(UUID.randomUUID(), "confirm_succeeded", version(), "provider-token-bill", "Verified exact input and output token usage from provider record", null, null, new BigDecimal("0.0011"), result);
reconcile(command); reconcile(command);
assertEquals(new BigDecimal("9.99890000"), wallet("balance"));
assertEquals(0, wallet("reserved_balance").signum());
assertEquals(100L, jdbc.queryForObject("SELECT input_units FROM usage_events WHERE provider_task_id=?", Long.class, taskId));
assertEquals(1, count("wallet_entries"));
}
@Test void unavailableCredentialAndAmbiguousLegacyRouteCannotResumePolling() {
doThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "safe message", 503)).when(provider).requirePollableBinding(anyString(), anyString(), anyString(), anyString(), anyString());
assertEquals(503, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL WHERE id=?", taskId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
}
@Test void existingObservedOutcomeCannotBeOverruledManually() {
jdbc.update("UPDATE model_tasks SET completion_status='succeeded' WHERE id=?", taskId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus()); unchanged();
}
@Test void insufficientHoldIsAConflictNotABalancingPlug() {
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus());
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance")); assertEquals(0, count("task_reconciliations"));
}
@Test void receiptsAreAppendOnlyAndNeverContainPromptOrGeneratedContent() {
reconcile(command("confirm_succeeded"));
String stored = jdbc.queryForObject("SELECT row_to_json(r)::text FROM task_reconciliations r", String.class);
assertFalse(stored.contains("private prompt")); assertFalse(stored.contains("generated.glb"));
assertThrows(DataAccessException.class, () -> jdbc.update("DELETE FROM task_reconciliations"));
assertThrows(DataAccessException.class, () -> jdbc.update("UPDATE task_reconciliations SET note='rewritten history'"));
assertEquals(1, count("task_reconciliations"));
}
@Test void listAndHistoryAreBoundedAdministratorViewsWithoutRawPayloads() {
var list = service.list("admin-session", 1);
assertEquals(1L, list.get("total"));
String encoded = assertDoesNotThrow(() -> json.writeValueAsString(list));
assertFalse(encoded.contains("private prompt")); assertFalse(encoded.contains("input_payload"));
reconcile(command("confirm_failed"));
assertEquals(1, ((List<?>) service.detail("admin-session", taskId).get("history")).size());
assertEquals(0L, service.list("admin-session", 1).get("total"));
}
@Test void freeFailureHasAnExplicitZeroCostAndNoLedgerEntry() {
jdbc.update("UPDATE model_tasks SET reserved_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
reconcile(command("confirm_failed"));
assertEquals(0, jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId).signum());
assertEquals(0, count("wallet_entries"));
}
}
@@ -0,0 +1,34 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class TaskReconciliationResultTest {
private final ObjectMapper json = new ObjectMapper();
@Test void acceptsOnlyNormalizedAssetsAndIntegralUsage() {
var result = TaskReconciliationResult.normalize(Map.of("url", "https://example.test/image.png", "usage", Map.of("input_tokens", "10", "output_tokens", 20,
"input_tokens_details", Map.of("text_tokens", 2, "image_tokens", 8))), json);
assertEquals(10L, ((Map<?,?>) result.get("usage")).get("input_tokens"));
}
@Test void rejectsCredentialsRawPayloadsAndUnsafeAssetUrls() {
for (Map<String,Object> value : List.<Map<String,Object>>of(
Map.of("Authorization", "secret"), Map.of("base64", "large media"),
Map.of("url", "javascript:alert(1)"), Map.of("url", "https://user:password@example.test/a"),
Map.of("usage", Map.of("api_key", "secret")), Map.of("usage", Map.of("input_tokens", -1)),
Map.of("usage", Map.of("output_tokens", 1.5)), Map.of("usage", Map.of("input_tokens", 100_000_001)))) {
assertEquals(422, assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(value, json)).getStatus());
}
}
@Test void rejectsOversizeObjectsAndCommandLoggingOmitsEvidenceAndPayload() {
assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(Map.of("url", "https://example.test/" + "x".repeat(40_000)), json));
var command = new TaskReconciliationCommand(java.util.UUID.randomUUID(), "confirm_succeeded", null, "private bill", "private note", null, null, null, Map.of("url", "private result"));
assertFalse(command.toString().contains("private"));
}
}