feat: add audited async task reconciliation and frozen bindings

This commit is contained in:
xpp committed 2026-10-21 20:08:38 +08:00
1 parent 5be06a97c1
commit c716ec7478
28 files changed
+1312 -76

No files matched your search

+3 -1
View File
@@ -17,7 +17,9 @@
账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;已确认失败的释放和成功结算沿用原子事务;不完整流或无效用量不得估算扣费。
异步创建支持用户级 `Idempotency-Key`(`V45`),同键同请求返回原任务,同键不同请求返回 409。异步上游受理未知、回执丢失、worker 中断或轮询期限到期时进入 `reconciling`(`V46`),保留冻结与预算占用,不自动重发/退款。已观察的生成结果先持久化再结算,账本失败只重试本地结算。发布时须先停旧 worker,不能混跑旧的未知结果退款逻辑。详见 `docs/22-async-task-safety.md`;受审计的人工核账入口仍属下一阶段。
异步创建支持用户级 `Idempotency-Key`(`V45`),同键同请求返回原任务,同键不同请求返回 409。异步上游受理未知、回执丢失、worker 中断或轮询期限到期时进入 `reconciling`(`V46`),保留冻结与预算占用,不自动重发/退款。已观察的生成结果先持久化再结算,账本失败只重试本地结算。发布时须先停旧 worker,不能混跑旧的未知结果退款逻辑。详见 `../docs/22-async-task-safety.md`。
`V47` 冻结异步 provider/model 绑定并约束已证明的上游任务身份;`V48` 增加仅追加核账收据。管理端“价格版本与成本对账”提供人工核账,可根据证据恢复只读轮询、确认失败释放原冻结、确认成功并按原价和实际用量结算。操作使用管理员会话、版本检查、UUID 幂等收据和与钱包同事务的审计;worker 和人工操作共用 `TaskSettlementService`,不另写扣费逻辑。历史无法证明的已受理绑定不能猜测补齐。详见 `../docs/23-audited-task-reconciliation.md`;尚无双人审批,不等同于完整财务流程。
同步和原生调用在结算前单独提交精简用量快照(Flyway `V44`)。快照已持久化而账本事务失败时,接口返回 `SETTLEMENT_PENDING`,冻结额度保留,由后台恢复任务重试结算。该快照不保存提示词、生成内容或密钥;不能将其当作响应重放存储。若数据库在快照写入前已不可用,则尚不能保证结果的持久恢复,必须结合上游记录核对。
@@ -23,6 +23,7 @@ public class GatewayGuard {
private static final long SLOT_TTL_MS = 180_000;
private static final DefaultRedisScript<Long> RATE = script("redis/rate-limit.lua");
private static final DefaultRedisScript<Long> ACQUIRE = script("redis/acquire-slot.lua");
private static final DefaultRedisScript<Long> ACQUIRE_TASK = script("redis/acquire-task-slot.lua");
private static final DefaultRedisScript<Long> ACQUIRE_PAIR = script("redis/acquire-pair.lua");
private static final DefaultRedisScript<List> CAPACITY = new DefaultRedisScript<>("""
local time = redis.call('TIME')
@@ -163,9 +164,11 @@ public class GatewayGuard {
if(policies==null) return true;
var policy=policies.taskLimit(keyId);
String bucket=redisPrefix+"gateway:slots:user:"+policy.userId();
boolean acquired=execute(ACQUIRE,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId)==1;
long result=execute(ACQUIRE_TASK,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId);
boolean acquired=result>0;
// Redis is outside the database transaction: roll back the lease if claiming the task fails.
if(acquired&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
// Rolling back renewal must not remove a lease that existed before this transaction.
if(result==1&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
org.springframework.transaction.support.TransactionSynchronizationManager.registerSynchronization(new org.springframework.transaction.support.TransactionSynchronization() {@Override public void afterCompletion(int status) {if(status!=STATUS_COMMITTED) releaseQuietly(bucket,taskId);}});
return acquired;
}
@@ -110,6 +110,17 @@ public class AsyncProviderClient {
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
}
/** Validate a proven polling binding without sending any HTTP request or exposing credential values. */
public void requirePollableBinding(String providerId, String routeId, String upstreamModel, String taskType, String fingerprint) {
if (fingerprint == null || !fingerprint.matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "需要有效的上游账号指纹", 422);
Provider provider = provider(providerId, routeId, upstreamModel);
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
boolean pollable = isTripo(provider.code(), provider.baseUrl()) ? "3d".equals(taskType)
: (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com") || "openai-compatible".equalsIgnoreCase(provider.protocol())) && "video".equals(taskType);
if (!pollable) throw new ApiException("TASK_NOT_POLLABLE", "该执行器没有可恢复的只读任务查询接口,请核实终态", 409);
provider.keyForFingerprint(routeId, upstreamModel, environment, keyCursors, fingerprint);
}
public Map<String,Object> poll(String providerId, String providerTaskId) {
return poll(providerId, null, providerTaskId);
}
@@ -19,8 +19,6 @@ import java.sql.SQLException;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -52,6 +50,7 @@ public class AsyncTaskService {
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
private final ApiRequestRiskPolicy requestRisk;
private final TaskOutcomeJournal outcomes;
private final TaskSettlementService settlements;
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
@@ -108,6 +107,7 @@ public class AsyncTaskService {
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
this.settlements = new TaskSettlementService(jdbc, objectMapper, gatewayGuard, usage, transactionManager);
}
@Transactional
@@ -177,6 +177,8 @@ public class AsyncTaskService {
}
requestRisk.enforce(new ApiRequestContext(userId, apiKeyId, ApiRequestProtocol.ASYNC_TASK,
requestedModelId, modelId, false, estimateInputUnits(input), 0, metadata, System.nanoTime()));
String upstreamModel = model.get("upstreamModel") instanceof String value ? value : null;
if (upstreamModel == null || upstreamModel.isBlank()) throw new ApiException("ASYNC_BINDING_MISSING", "模型缺少可冻结的上游执行标识", 503);
// Serialize the queue-capacity check with task insertion for this
// model. Without the transaction advisory lock, concurrent creates
// could all observe the same count and exceed maxQueuedPerModel.
@@ -198,8 +200,8 @@ public class AsyncTaskService {
OffsetDateTime now = now();
try {
jdbc.update(
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash()
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash, provider_id_snapshot, upstream_model_snapshot) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash(), model.get("providerId"), upstreamModel
);
} catch (RuntimeException exception) {
releaseWallet(userId, reservedAmount);
@@ -253,7 +255,7 @@ public class AsyncTaskService {
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
);
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model FROM model_tasks t WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
for (Map<String,Object> row : pending) pollProviderTask(row);
dispatchQueuedTasks();
}
@@ -314,10 +316,10 @@ public class AsyncTaskService {
// workers reliably.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
String sql = """
SELECT t.id, t.user_id, t.api_key_id, t.model_id, m.provider_id, m.upstream_model, t.task_type, t.prompt, t.input
SELECT t.id, t.user_id, t.api_key_id, t.model_id, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model, t.task_type, t.prompt, t.input
FROM model_tasks t
JOIN model_catalog m ON m.id=t.model_id
JOIN model_providers p ON p.id=m.provider_id
JOIN model_providers p ON p.id=t.provider_id_snapshot
WHERE t.status='queued'
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
@@ -373,35 +375,7 @@ public class AsyncTaskService {
}
private void finishTask(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
// The terminal state and wallet change must commit together. Otherwise a
// crash after marking failed/succeeded leaves a hold that polling never
// revisits. The conditional update also prevents duplicate settlement.
inTransaction(() -> {
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
// A late failure must not override a durably observed success whose settlement is pending.
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
AsyncPriceSchedule.Bill bill=null;
String finalCode=code,finalMessage=message;
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
try { bill=AsyncPriceSchedule.bill(readMap(String.valueOf(task.get("pricing_snapshot"))),result); }
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
}
int changed = jdbc.update(
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
status, progress, result == null ? null : writeJson(result),
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
if (changed == 0) return null;
if ("TASK_USAGE_PENDING".equals(finalCode)) {
releaseTaskInTransaction(taskId);
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?",taskId);
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'",writeJson(usageOnly(result)),taskId);
}
else if ("succeeded".equals(status)) settleTaskInTransaction(taskId,bill);
else releaseTaskInTransaction(taskId);
usage.updateTask(taskId, status, result, finalCode, finalMessage);
return null;
});
settlements.finish(taskId, status, progress, result, code, message);
}
private Map<String, Object> requireKey(String authorization) {
@@ -476,6 +450,10 @@ public class AsyncTaskService {
private void pollProviderTask(Map<String,Object> row) {
String taskId = String.valueOf(row.get("id"));
if (row.get("provider_id") == null || row.get("upstream_model") == null) {
outcomes.requireReconciliation(taskId, "ASYNC_BINDING_MISSING", "历史任务缺少可证明的原始执行路由;冻结额度保留,请根据上游记录核账", false);
return;
}
OffsetDateTime deadline = taskTimestamp(row.get("provider_deadline_at"));
if (deadline == null) {
// Defensive fallback for pre-migration tasks. The migration also
@@ -622,42 +600,10 @@ public class AsyncTaskService {
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, userId);
}
private void settleTaskInTransaction(String taskId,AsyncPriceSchedule.Bill bill) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
BigDecimal amount = bill==null ? reserved : bill.amount();
policies.release(taskId);
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
if (amount == null || task.get("cost_amount") != null) return;
UUID userId = (UUID) task.get("user_id");
Map<String,Object> wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
BigDecimal before = (BigDecimal) wallet.get("balance");
BigDecimal after = before.subtract(amount);
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
if(amount.signum()>0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)", UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
if(bill!=null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",bill.input(),bill.output(),bill.input(),bill.output(),bill.imageInput(),taskId);
}
private void releaseTaskInTransaction(String taskId) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
policies.release(taskId);
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) {
releaseWallet((UUID) task.get("user_id"), amount);
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
}
}
private <T> T inTransaction(Supplier<T> work) {
return transactions == null ? work.get() : transactions.execute(status -> work.get());
}
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
private void putIfPresent(Map<String,Object> target, String name, ResultSet resultSet, String column) throws SQLException {
Object value = resultSet.getObject(column);
if (value != null) target.put(name, value);
@@ -0,0 +1,21 @@
package com.moyi.controlplane.task;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.UUID;
/** Administrative evidence and optimistic-concurrency preconditions, never a raw provider key. */
public record TaskReconciliationCommand(
UUID requestId,
String action,
OffsetDateTime expectedUpdatedAt,
String evidenceReference,
String note,
String providerTaskId,
String credentialFingerprint,
BigDecimal expectedCost,
Map<String,Object> result
) {
@Override public String toString() { return "TaskReconciliationCommand[requestId=" + requestId + ", action=" + action + "]"; }
}
@@ -0,0 +1,42 @@
package com.moyi.controlplane.task;
import com.moyi.controlplane.auth.AuthController;
import com.moyi.controlplane.common.ApiException;
import jakarta.servlet.http.HttpServletRequest;
import java.util.Map;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/admin/task-reconciliation")
public class TaskReconciliationController {
private final TaskReconciliationService reconciliations;
public TaskReconciliationController(TaskReconciliationService reconciliations) { this.reconciliations = reconciliations; }
@GetMapping
public Map<String,Object> list(@RequestParam(defaultValue="1") int page, HttpServletRequest request) {
return reconciliations.list(AuthController.sessionToken(request), page);
}
@GetMapping("/{taskId}")
public Map<String,Object> detail(@PathVariable String taskId, HttpServletRequest request) {
return reconciliations.detail(AuthController.sessionToken(request), taskId);
}
@PostMapping(value="/{taskId}", consumes="application/json")
public Map<String,Object> reconcile(@PathVariable String taskId, @RequestBody TaskReconciliationCommand command, HttpServletRequest request) {
// A custom header + JSON requires CORS preflight; the header is a CSRF barrier, not authentication.
if (!"console".equals(request.getHeader("X-Moyi-Source"))) throw new ApiException("CONSOLE_REQUEST_REQUIRED", "请通过管理控制台发起核账", 403);
return reconciliations.reconcile(AuthController.sessionToken(request), taskId, command, AuthController.meta(request));
}
@PostMapping(value="/{taskId}/quote", consumes="application/json")
public Map<String,Object> quote(@PathVariable String taskId, @RequestBody Map<String,Object> result, HttpServletRequest request) {
return reconciliations.quote(AuthController.sessionToken(request), taskId, result);
}
}
@@ -0,0 +1,64 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import java.math.BigDecimal;
import java.net.URI;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
/** Only normalized assets and numeric usage may enter a manual completion, never raw provider payloads. */
final class TaskReconciliationResult {
private static final Set<String> ASSETS = Set.of("url", "modelUrl", "videoUrl", "imageUrl", "audioUrl", "previewUrl");
private static final Set<String> OPENAI = Set.of("input_tokens", "output_tokens", "total_tokens", "cached_tokens", "input_tokens_details");
private static final Set<String> DETAILS = Set.of("text_tokens", "image_tokens", "cached_tokens", "cached_tokens_details");
private static final Set<String> CACHED = Set.of("text_tokens", "image_tokens");
private static final Set<String> GEMINI = Set.of("promptTokenCount", "candidatesTokenCount", "thoughtsTokenCount", "cachedContentTokenCount", "totalTokenCount");
static Map<String,Object> normalize(Map<String,Object> input, ObjectMapper json) {
if (input == null) throw invalid();
try { if (json.writeValueAsBytes(input).length > 32_768) throw invalid(); }
catch (JsonProcessingException exception) { throw invalid(); }
var result = new LinkedHashMap<String,Object>();
for (var entry : input.entrySet()) {
String field = entry.getKey();
Object value = entry.getValue();
if (ASSETS.contains(field)) {
if (!(value instanceof String text) || text.length() > 4096 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
try {
URI uri = URI.create(text);
if (!("https".equalsIgnoreCase(uri.getScheme()) || "http".equalsIgnoreCase(uri.getScheme())) || uri.getHost() == null || uri.getUserInfo() != null) throw invalid();
} catch (IllegalArgumentException exception) { throw invalid(); }
result.put(field, text);
} else if (Set.of("format", "mimeType").contains(field)) {
if (!(value instanceof String text) || text.isBlank() || text.length() > 128 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
result.put(field, text);
} else if ("usage".equals(field)) result.put(field, numericMap(value, OPENAI));
else if ("usageMetadata".equals(field)) result.put(field, numericMap(value, GEMINI));
else throw invalid();
}
return result;
}
private static Map<String,Object> numericMap(Object value, Set<String> allowed) {
if (!(value instanceof Map<?,?> map)) throw invalid();
var result = new LinkedHashMap<String,Object>();
for (var entry : map.entrySet()) {
if (!(entry.getKey() instanceof String name) || !allowed.contains(name)) throw invalid();
if ("input_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), DETAILS));
else if ("cached_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), CACHED));
else {
try {
long count = new BigDecimal(String.valueOf(entry.getValue())).longValueExact();
if (count < 0 || count > 100_000_000) throw invalid();
result.put(name, count);
} catch (NumberFormatException | ArithmeticException exception) { throw invalid(); }
}
}
return result;
}
private static ApiException invalid() { return new ApiException("INVALID_RECONCILIATION_RESULT", "只接受有界的 HTTP(S) 结果地址和规范化数值用量,不接受原始上游响应或密钥", 422); }
}
@@ -0,0 +1,216 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.auth.AuthService;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
/** Evidence-backed administrative decisions; no generation, arbitrary balance adjustment or transparent replay. */
@Service
public class TaskReconciliationService {
private final JdbcTemplate jdbc;
private final ObjectMapper json;
private final AuthService auth;
private final TaskSettlementService settlements;
private final AsyncProviderClient provider;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final TransactionTemplate transactions;
private final long timeoutMs;
// List/detail must not load prompts, original inputs or potentially large generated media.
private static final String TASK_COLUMNS = "SELECT t.id,t.user_id,t.api_key_id,t.model_id,t.task_type,t.status,t.updated_at,t.reserved_amount,t.cost_amount,t.provider_id_snapshot,t.upstream_model_snapshot,t.provider_task_id,t.provider_key_fingerprint,t.completion_status,t.pricing_snapshot::text AS frozen_pricing,t.error::text AS error FROM model_tasks t";
public TaskReconciliationService(JdbcTemplate jdbc, ObjectMapper json, AuthService auth,
TaskSettlementService settlements, AsyncProviderClient provider, GatewayGuard guard,
UsageAuditService usage, PlatformTransactionManager manager,
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long timeoutMs) {
this.jdbc = jdbc; this.json = json; this.auth = auth; this.settlements = settlements;
this.provider = provider; this.guard = guard; this.usage = usage;
this.transactions = new TransactionTemplate(manager);
if (timeoutMs < 1000) throw new IllegalArgumentException("Polling deadline must be positive");
this.timeoutMs = timeoutMs;
}
public Map<String,Object> list(String token, int requestedPage) {
auth.requireAdmin(token);
int page = Math.max(1, Math.min(100000, requestedPage));
return Map.of("records", jdbc.queryForList(TASK_COLUMNS + " WHERE t.status='reconciling' ORDER BY t.reconciliation_required_at NULLS FIRST,t.id LIMIT 50 OFFSET ?", (page - 1) * 50).stream().map(this::view).toList(),
"total", jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE status='reconciling'", Long.class), "page", page);
}
public Map<String,Object> detail(String token, String taskId) {
auth.requireAdmin(token);
Map<String,Object> task = task(taskId, false);
var history = jdbc.queryForList("SELECT request_id,actor_user_id,action,evidence_reference,note,before_state::text AS before_state,after_state::text AS after_state,created_at FROM task_reconciliations WHERE task_id=? ORDER BY created_at DESC LIMIT 50", taskId);
var records = history.stream().map(row -> {
var item = new LinkedHashMap<String,Object>();
for (String field : List.of("request_id", "actor_user_id", "action", "evidence_reference", "note", "created_at")) item.put(field, row.get(field));
item.put("before_state", decode(row.get("before_state")));
item.put("after_state", decode(row.get("after_state")));
return item;
}).toList();
return Map.of("task", view(task), "history", records);
}
public Map<String,Object> quote(String token, String taskId, Map<String,Object> input) {
auth.requireAdmin(token);
var task = task(taskId, false);
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可核账状态,请刷新");
var result = TaskReconciliationResult.normalize(input, json);
return Map.of("expectedCost", confirmedCharge(task, result).toPlainString(),
"expectedUpdatedAt", timestamp(task.get("updated_at")).toInstant().toString());
}
public Map<String,Object> reconcile(String token, String taskId, TaskReconciliationCommand command, AuthService.RequestMeta meta) {
var actor = auth.requireAdmin(token); // Authorization is required even for replay.
validate(command);
Map<String,Object> result = "confirm_succeeded".equals(command.action()) ? TaskReconciliationResult.normalize(command.result(), json) : null;
String requestHash = requestHash(command, taskId, result);
return transactions.execute(status -> {
// Keep role/status revocation ordered with this financial decision, not a stale pre-check.
if (jdbc.queryForList("SELECT id FROM users WHERE id=? AND status='active' AND role IN ('admin','super_admin') FOR SHARE", actor.id()).isEmpty()) throw new ApiException("FORBIDDEN", "管理员权限已变化,请重新登录", 403);
auth.requireAdmin(token); // Recheck session revocation/expiry after entering the transaction.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))", "moyi:task-reconciliation:" + command.requestId());
var prior = jdbc.queryForList("SELECT actor_user_id,request_hash,after_state::text AS after_state FROM task_reconciliations WHERE request_id=?", command.requestId());
if (!prior.isEmpty()) {
var receipt = prior.getFirst();
if (!actor.id().equals(receipt.get("actor_user_id")) || !requestHash.equals(receipt.get("request_hash"))) throw conflict("此核账请求编号已用于不同的操作");
return Map.of("task", decode(receipt.get("after_state")), "requestId", command.requestId(), "replayed", true);
}
var task = task(taskId, true);
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可人工核账状态,或已有可信终态;请刷新");
if (!timestamp(task.get("updated_at")).toInstant().equals(command.expectedUpdatedAt().toInstant())) throw conflict("任务已被其他操作更新,请刷新后重新核对");
var wallets = jdbc.queryForList("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", task.get("user_id"));
if (wallets.isEmpty() || ((BigDecimal) wallets.getFirst().get("reserved_balance")).compareTo((BigDecimal) task.get("reserved_amount")) < 0) throw conflict("任务冻结额度与钱包不一致,请先排查账本,不允许直接覆盖");
Map<String,Object> before = view(task);
if ("resume_polling".equals(command.action())) resume(task, command);
else {
if ("confirm_succeeded".equals(command.action())) {
BigDecimal charge = confirmedCharge(task, result);
if (charge == null || charge.compareTo(command.expectedCost()) != 0) throw conflict("确认金额与任务原价格快照及实际用量不一致");
}
String target = "confirm_succeeded".equals(command.action()) ? "succeeded" : "failed";
String code = "failed".equals(target) ? "TASK_RECONCILED_FAILED" : null;
String message = "failed".equals(target) ? "管理员已根据上游记录确认任务失败,冻结额度已释放" : null;
// The observed outcome, shared wallet transaction and both audit receipts commit together.
jdbc.update("UPDATE model_tasks SET completion_status=? WHERE id=?", target, taskId);
settlements.finish(taskId, target, "succeeded".equals(target) ? 100 : 0, result, code, message);
}
Map<String,Object> after = view(task(taskId, false));
jdbc.update("INSERT INTO task_reconciliations(request_id,task_id,actor_user_id,action,request_hash,evidence_reference,note,before_state,after_state) VALUES (?,?,?,?,?,?,?,CAST(? AS jsonb),CAST(? AS jsonb))",
command.requestId(), taskId, actor.id(), command.action(), requestHash, command.evidenceReference().trim(), command.note().trim(), encode(before), encode(after));
auth.audit("task.reconciled", actor.id(), meta, Map.of("taskId", taskId, "requestId", command.requestId(), "action", command.action(), "evidenceReference", command.evidenceReference().trim()));
return Map.of("task", after, "requestId", command.requestId(), "replayed", false);
});
}
private BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
try { return settlements.confirmedCharge(task, result); }
catch (ApiException exception) {
if ("TASK_USAGE_PENDING".equals(exception.getCode())) throw new ApiException("TASK_USAGE_REQUIRED", "成功核账必须提供完整可核验用量;任务与冻结额度保持不变", 422);
throw exception;
}
}
private void resume(Map<String,Object> task, TaskReconciliationCommand command) {
String originalId = (String) task.get("provider_task_id");
if (originalId != null && !originalId.equals(command.providerTaskId())) throw conflict("不能将已受理任务改绑到另一个上游编号");
String fingerprint = (String) task.get("provider_key_fingerprint");
if (fingerprint != null && !fingerprint.equals(command.credentialFingerprint())) throw conflict("必须继续使用创建任务时的账号指纹,不能切换账号");
if (task.get("provider_id_snapshot") == null || task.get("upstream_model_snapshot") == null) throw conflict("历史任务原执行路由无法证明,禁止猜测后恢复查询;请核实终态");
provider.requirePollableBinding(String.valueOf(task.get("provider_id_snapshot")), String.valueOf(task.get("model_id")),
String.valueOf(task.get("upstream_model_snapshot")), String.valueOf(task.get("task_type")), command.credentialFingerprint());
String id = String.valueOf(task.get("id"));
if (!guard.tryTaskUserSlot((java.util.UUID) task.get("api_key_id"), id, timeoutMs + 180000)) throw new ApiException("TASK_USER_BUSY", "用户当前并发已满,尚未恢复轮询,请稍后重试", 429);
jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=now() WHERE id=?",
command.providerTaskId(), command.credentialFingerprint(), OffsetDateTime.now(ZoneOffset.UTC).plus(java.time.Duration.ofMillis(timeoutMs)), id);
usage.updateTask(id, "running", null, null, null);
}
private void validate(TaskReconciliationCommand command) {
if (command == null || command.requestId() == null || command.expectedUpdatedAt() == null) throw new ApiException("RECONCILIATION_PRECONDITION_REQUIRED", "必须提供操作编号和任务版本", 422);
if (command.action() == null || !List.of("resume_polling", "confirm_failed", "confirm_succeeded").contains(command.action())) throw new ApiException("INVALID_RECONCILIATION_ACTION", "核账操作无效", 422);
text(command.evidenceReference(), 3, 500); text(command.note(), 10, 2000);
if ("resume_polling".equals(command.action())) {
text(command.providerTaskId(), 1, 160);
if (!command.providerTaskId().matches("[A-Za-z0-9][A-Za-z0-9._:-]{0,159}")) throw new ApiException("INVALID_PROVIDER_TASK_ID", "上游任务编号不能包含路径、查询串或空白", 422);
if (command.credentialFingerprint() == null || !command.credentialFingerprint().matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "必须提供 SHA-256 账号指纹,不能填写 API Key", 422);
if (command.result() != null || command.expectedCost() != null) throw conflict("恢复轮询不能同时修改结果或扣费金额");
} else {
if (command.providerTaskId() != null || command.credentialFingerprint() != null) throw conflict("终态核账不能改绑上游编号或账号");
if ("confirm_succeeded".equals(command.action())) {
if (command.expectedCost() == null || command.expectedCost().signum() < 0 || command.expectedCost().precision() > 20) throw new ApiException("EXPECTED_COST_REQUIRED", "请确认按原价格快照核算的非负扣费金额", 422);
try { command.expectedCost().setScale(8, RoundingMode.UNNECESSARY); }
catch (ArithmeticException exception) { throw new ApiException("INVALID_EXPECTED_COST", "确认金额最多 8 位小数", 422); }
} else if (command.result() != null || command.expectedCost() != null) throw conflict("确认失败不能附带成功结果或扣费金额");
}
}
private void text(String value, int min, int max) {
if (value == null || value.trim().length() < min || value.trim().length() > max || value.chars().anyMatch(Character::isISOControl)) throw new ApiException("RECONCILIATION_EVIDENCE_REQUIRED", "必须填写有效的上游证据引用与 10–2000 字核账说明;不得包含控制字符", 422);
}
private String requestHash(TaskReconciliationCommand command, String taskId, Map<String,Object> result) {
var body = new LinkedHashMap<String,Object>();
body.put("taskId", taskId); body.put("action", command.action()); body.put("expectedUpdatedAt", command.expectedUpdatedAt().toInstant().toString());
body.put("evidenceReference", command.evidenceReference().trim()); body.put("note", command.note().trim());
body.put("providerTaskId", command.providerTaskId()); body.put("credentialFingerprint", command.credentialFingerprint());
body.put("expectedCost", command.expectedCost()); body.put("result", result);
return TaskCreationIdentity.of(json, command.requestId().toString(), "task-reconciliation", body).requestHash();
}
private Map<String,Object> task(String taskId, boolean lock) {
if (taskId == null || taskId.isBlank() || taskId.length() > 160) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
var rows = jdbc.queryForList(TASK_COLUMNS + " WHERE t.id=?" + (lock ? " FOR UPDATE" : ""), taskId);
if (rows.isEmpty()) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
var task = rows.getFirst();
task.put("pricing_snapshot", task.get("frozen_pricing"));
return task;
}
private Map<String,Object> view(Map<String,Object> task) {
var view = new LinkedHashMap<String,Object>();
view.put("id", task.get("id")); view.put("userId", task.get("user_id")); view.put("modelId", task.get("model_id"));
view.put("taskType", task.get("task_type")); view.put("status", task.get("status"));
view.put("updatedAt", timestamp(task.get("updated_at")).toInstant().toString());
view.put("reservedAmount", ((BigDecimal) task.get("reserved_amount")).toPlainString());
view.put("costAmount", task.get("cost_amount") == null ? null : ((BigDecimal) task.get("cost_amount")).toPlainString());
view.put("providerId", task.get("provider_id_snapshot")); view.put("upstreamModel", task.get("upstream_model_snapshot"));
view.put("providerTaskId", task.get("provider_task_id")); view.put("credentialFingerprint", task.get("provider_key_fingerprint"));
view.put("completionStatus", task.get("completion_status")); view.put("requiresReconciliation", "reconciling".equals(task.get("status")));
view.put("pricing", task.get("frozen_pricing") == null ? null : decode(task.get("frozen_pricing")));
view.put("error", task.get("error") == null ? null : decode(task.get("error")));
return view;
}
private OffsetDateTime timestamp(Object value) {
if (value instanceof OffsetDateTime time) return time;
if (value instanceof java.sql.Timestamp time) return time.toInstant().atOffset(ZoneOffset.UTC);
throw new IllegalStateException("Persisted task timestamp missing");
}
private Map<String,Object> decode(Object value) {
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Persisted reconciliation record cannot be read"); }
}
private String encode(Object value) {
try { return json.writeValueAsString(value); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Reconciliation record cannot be encoded"); }
}
private ApiException conflict(String message) { return new ApiException("RECONCILIATION_CONFLICT", message, 409); }
}
@@ -0,0 +1,125 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.EnterprisePolicies;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
/** The one task/wallet/usage settlement transaction, shared by workers and audited reconciliation. */
@Service
public class TaskSettlementService {
private final JdbcTemplate jdbc;
private final ObjectMapper json;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final EnterprisePolicies policies;
private final TransactionTemplate transactions;
public TaskSettlementService(JdbcTemplate jdbc, ObjectMapper json, GatewayGuard guard,
UsageAuditService usage, PlatformTransactionManager manager) {
this.jdbc = jdbc;
this.json = json;
this.guard = guard;
this.usage = usage;
this.policies = new EnterprisePolicies(jdbc, json);
this.transactions = manager == null ? null : new TransactionTemplate(manager);
}
void finish(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal task state required");
transaction(() -> {
var task = jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE", taskId);
if (!List.of("queued", "running", "reconciling").contains(task.get("status"))) return null;
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
AsyncPriceSchedule.Bill bill = null;
String finalCode = code, finalMessage = message;
if ("succeeded".equals(status) && task.get("pricing_snapshot") != null) {
try { bill = AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result); }
catch (ApiException exception) {
finalCode = "TASK_USAGE_PENDING";
finalMessage = "生成已完成,但上游计费用量未能核对;暂不扣费,等待核账";
}
}
jdbc.update("UPDATE model_tasks SET status=?,progress=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),reconciliation_required_at=NULL,updated_at=now(),completed_at=now() WHERE id=?",
status, progress, encode(result), finalCode == null ? null : encode(Map.of("code", finalCode, "message", finalMessage)), taskId);
if ("TASK_USAGE_PENDING".equals(finalCode)) {
release(taskId);
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?", taskId);
var observed = new LinkedHashMap<String,Object>();
if (result != null) for (String field : List.of("usage", "usageMetadata")) if (result.get(field) instanceof Map<?,?> value) observed.put(field, value);
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'", encode(observed), taskId);
} else if ("succeeded".equals(status)) settle(taskId, bill);
else release(taskId);
usage.updateTask(taskId, status, result, finalCode, finalMessage);
return null;
});
}
/** Strict calculation for explicit administrator confirmation: missing usage is an error, not a refund. */
BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
if (task.get("pricing_snapshot") == null) return (BigDecimal) task.get("reserved_amount");
return AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result).amount();
}
private void settle(String taskId, AsyncPriceSchedule.Bill bill) {
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
BigDecimal amount = bill == null ? reserved : bill.amount();
releasePolicies(taskId, task);
if (amount == null || task.get("cost_amount") != null) return;
UUID userId = (UUID) task.get("user_id");
var wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
BigDecimal before = (BigDecimal) wallet.get("balance"), after = before.subtract(amount);
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
if (amount.signum() > 0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)",
UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
if (bill != null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",
bill.input(), bill.output(), bill.input(), bill.output(), bill.imageInput(), taskId);
}
private void release(String taskId) {
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
releasePolicies(taskId, task);
if (amount != null && task.get("cost_amount") == null) {
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, task.get("user_id"));
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
}
}
private void releasePolicies(String taskId, Map<String,Object> task) {
policies.release(taskId);
guard.releaseTaskUserSlot((UUID) task.get("user_id"), taskId);
}
private Map<String,Object> pricing(Object value) {
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
catch (JsonProcessingException exception) { throw new ApiException("INVALID_PRICE_SNAPSHOT", "任务价格快照无法核验", 409); }
}
private String encode(Object value) {
if (value == null) return null;
try { return json.writeValueAsString(value); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Task settlement cannot be encoded"); }
}
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
private <T> T transaction(Supplier<T> work) { return transactions == null ? work.get() : transactions.execute(status -> work.get()); }
}
@@ -0,0 +1,23 @@
-- Freeze route identity without copying credentials or mutable provider configuration.
ALTER TABLE model_tasks ADD COLUMN provider_id_snapshot varchar(64) REFERENCES model_providers(id);
ALTER TABLE model_tasks ADD COLUMN upstream_model_snapshot varchar(160);
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_execution_binding_check CHECK (
(provider_id_snapshot IS NULL AND upstream_model_snapshot IS NULL)
OR (provider_id_snapshot IS NOT NULL AND upstream_model_snapshot IS NOT NULL
AND length(trim(provider_id_snapshot)) > 0 AND length(trim(upstream_model_snapshot)) > 0)
);
-- Never guess the original route of an already-submitted legacy task.
-- Unsubmitted queued work can safely freeze its route at migration time.
UPDATE model_tasks t SET provider_id_snapshot=m.provider_id, upstream_model_snapshot=m.upstream_model
FROM model_catalog m
WHERE t.model_id=m.id AND t.status='queued' AND t.provider_task_id IS NULL;
COMMENT ON COLUMN model_tasks.provider_id_snapshot IS 'Provider identity at admission; NULL for ambiguous legacy accepted tasks';
-- New proven bindings cannot associate the same provider/account task with two local wallets.
-- Legacy accepted rows are deliberately not guessed/backfilled, so this index does not rewrite their history.
CREATE UNIQUE INDEX model_tasks_proven_upstream_identity_idx
ON model_tasks(provider_id_snapshot,provider_key_fingerprint,provider_task_id)
WHERE provider_id_snapshot IS NOT NULL AND provider_key_fingerprint IS NOT NULL
AND provider_task_id IS NOT NULL AND provider_task_id <> 'mock';
@@ -0,0 +1,20 @@
-- Append-only operation receipts; no prompts, generated media or raw credentials.
CREATE TABLE task_reconciliations (
request_id uuid PRIMARY KEY,
task_id varchar(160) NOT NULL REFERENCES model_tasks(id),
actor_user_id uuid NOT NULL REFERENCES users(id),
action varchar(24) NOT NULL CHECK (action IN ('resume_polling','confirm_failed','confirm_succeeded')),
request_hash varchar(64) NOT NULL CHECK (request_hash ~ '^[0-9a-f]{64}$'),
evidence_reference varchar(500) NOT NULL CHECK (length(trim(evidence_reference)) BETWEEN 3 AND 500),
note varchar(2000) NOT NULL CHECK (length(trim(note)) BETWEEN 10 AND 2000),
before_state jsonb NOT NULL,
after_state jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX task_reconciliations_task_idx ON task_reconciliations(task_id,created_at DESC);
CREATE FUNCTION protect_task_reconciliation_history() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
RAISE EXCEPTION 'Task reconciliation history is append-only';
END $$;
CREATE TRIGGER task_reconciliation_history_immutable BEFORE UPDATE OR DELETE ON task_reconciliations
FOR EACH ROW EXECUTE FUNCTION protect_task_reconciliation_history();
@@ -0,0 +1,14 @@
-- A resumed task renews its own lease instead of consuming a second slot.
local time = redis.call('TIME')
local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
local existing = redis.call('ZSCORE', KEYS[1], ARGV[3])
if not existing and redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[1]) then
return 0
end
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[2]), ARGV[3])
if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then
redis.call('PEXPIRE', KEYS[1], ARGV[2])
end
if existing then return 2 end
return 1
@@ -0,0 +1,64 @@
package com.moyi.controlplane.gateway;
import java.util.UUID;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
@EnabledIfEnvironmentVariable(named="MOYI_GATEWAY_TEST_REDIS_PORT", matches="[0-9]{2,5}")
class TaskLeaseRedisTest {
private LettuceConnectionFactory factory;
private StringRedisTemplate redis;
private GatewayGuard guard;
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID();
private final String prefix = "moyi:test:" + UUID.randomUUID() + ":";
private String bucket() { return prefix + "gateway:slots:user:" + userId; }
@BeforeEach void setup() {
factory = new LettuceConnectionFactory("127.0.0.1", Integer.parseInt(System.getenv("MOYI_GATEWAY_TEST_REDIS_PORT")));
factory.afterPropertiesSet(); factory.start();
redis = new StringRedisTemplate(factory); redis.afterPropertiesSet();
guard = new GatewayGuard(redis, 60, 300, 4, prefix);
var policies = mock(EnterprisePolicies.class);
when(policies.taskLimit(keyId)).thenReturn(new EnterprisePolicies.UserLimit(userId, 120, 1));
guard.setEnterprisePolicies(policies);
}
@AfterEach void cleanup() {
if (TransactionSynchronizationManager.isSynchronizationActive()) TransactionSynchronizationManager.clearSynchronization();
if (redis != null) redis.delete(bucket());
if (factory != null) factory.destroy();
}
@Test void renewalDoesNotConsumeAnotherSlotAndDifferentTaskIsBlocked() {
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
assertEquals(1L, redis.opsForZSet().zCard(bucket()));
assertFalse(guard.tryTaskUserSlot(keyId, "different-task", 180000));
assertTrue(redis.getExpire(bucket(), java.util.concurrent.TimeUnit.MILLISECONDS) > 250000);
}
@Test void rolledBackRenewalKeepsThePreexistingLease() {
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
TransactionSynchronizationManager.initSynchronization();
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
TransactionSynchronizationManager.clearSynchronization();
assertNotNull(redis.opsForZSet().score(bucket(), "original-task"));
}
@Test void rolledBackNewClaimRemovesOnlyItsOwnLease() {
TransactionSynchronizationManager.initSynchronization();
assertTrue(guard.tryTaskUserSlot(keyId, "new-task", 180000));
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
TransactionSynchronizationManager.clearSynchronization();
assertNull(redis.opsForZSet().score(bucket(), "new-task"));
}
}
@@ -289,6 +289,15 @@ class AsyncProviderClientTest {
} finally { server.stop(0); }
}
@Test void pollingBindingValidationChecksTheExistingCredentialWithoutAnHttpCall() throws Exception {
var client = clientFor("http://127.0.0.1:1");
String fingerprint = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256").digest("test-key".getBytes(StandardCharsets.UTF_8)));
org.junit.jupiter.api.Assertions.assertDoesNotThrow(() -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", fingerprint));
assertEquals("UPSTREAM_KEY_NOT_AVAILABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "0".repeat(64))).getCode());
assertEquals("TASK_NOT_POLLABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "image", fingerprint)).getCode());
assertEquals("INVALID_CREDENTIAL_FINGERPRINT", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "test-key")).getCode());
}
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
var client = clientFor("http://127.0.0.1:1");
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
@@ -65,9 +65,11 @@ class AsyncTaskPostgresTest {
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
// This fixture knows its original binding; a migration must not guess it for real legacy jobs.
jdbc.update("UPDATE model_tasks SET provider_id_snapshot='provider-vast',upstream_model_snapshot='v3.1-20260211' WHERE id=?", taskId);
when(catalog.keyOwnerId(keyId)).thenReturn(userId);
when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true);
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "upstreamModel", "v3.1-20260211", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
}
@AfterEach void cleanup() {
@@ -97,6 +99,34 @@ class AsyncTaskPostgresTest {
return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal());
}
@Test void queuedTaskRetainsAdmittedExecutionBindingWhenRouteIsEdited() {
String created = String.valueOf(create("frozen-queue", Map.of("model", "model-mock-3d", "prompt", "chair")).get("id"));
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
jdbc.update("UPDATE model_tasks SET updated_at=now() WHERE id=?", taskId);
when(provider.submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap())).thenReturn(Map.of("id", "original-route-task"));
tasks.completeMockTasks();
assertEquals("original-route-task", jdbc.queryForObject("SELECT provider_task_id FROM model_tasks WHERE id=?", String.class, created));
verify(provider).submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap());
}
@Test void acceptedTaskPollsFrozenProviderRatherThanEditedCatalog() {
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null)).thenReturn(Map.of("status", "running", "progress", 55));
tasks.completeMockTasks();
assertEquals("running", status());
assertEquals(55, jdbc.queryForObject("SELECT progress FROM model_tasks WHERE id=?", Integer.class, taskId));
verify(provider).poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null);
}
@Test void acceptedLegacyTaskWithoutProofOfBindingIsNotPolledOnAGuessedRoute() {
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL,updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
tasks.completeMockTasks();
assertEquals("reconciling", status());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
verifyNoInteractions(provider);
}
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
var first = create("client-operation-1", input);
@@ -0,0 +1,295 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.moyi.controlplane.auth.AuthService;
import com.moyi.controlplane.auth.PasswordHasher;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.common.ApiExceptionHandler;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import jakarta.servlet.http.Cookie;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.dao.DataAccessException;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
@EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+")
class TaskReconciliationPostgresTest {
private final ObjectMapper json = new ObjectMapper().findAndRegisterModules().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID(), adminId = UUID.randomUUID(), secondAdminId = UUID.randomUUID();
private final String taskId = "reconcile-" + UUID.randomUUID();
private final String fingerprint = "c".repeat(64);
private final AuthService.RequestMeta meta = new AuthService.RequestMeta("127.0.0.1", "isolated-reconciliation-test");
private DriverManagerDataSource base;
private JdbcTemplate jdbc;
private String schema;
private AuthService auth;
private GatewayGuard guard;
private AsyncProviderClient provider;
private TaskReconciliationService service;
private MockMvc http;
@BeforeEach void setup() {
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
schema = "reconciliation_test_" + UUID.randomUUID().toString().replace("-", "");
base = new DriverManagerDataSource(url, "moyi_role_test", "");
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
var source = new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", "");
jdbc = new JdbcTemplate(source);
var manager = new DataSourceTransactionManager(source);
for (var entry : Map.of(userId, "user", adminId, "admin", secondAdminId, "admin").entrySet()) {
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name,role) VALUES (?,?,'username','not-loginable','Test',?)", entry.getKey(), entry.getKey().toString(), entry.getValue());
}
session("admin-session", adminId); session("second-admin-session", secondAdminId); session("user-session", userId);
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-key',?)", keyId, userId, "b".repeat(64));
jdbc.update("INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)", userId);
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot,pricing_snapshot,reconciliation_required_at) VALUES (?,?,?,'model-mock-3d','3d','reconciling','private prompt',1.44,'original-provider-task',?,'provider-vast','v3.1-20260211','{\"unit\":\"CNY / task\",\"price\":1.44}'::jsonb,now())", taskId, userId, keyId, fingerprint);
jdbc.update("INSERT INTO gateway_budget_holds(request_id,user_id,key_id,amount) VALUES (?,?,?,1.44)", taskId, userId, keyId);
var usage = new UsageAuditService(jdbc, json);
usage.recordTask(userId, keyId, "model-mock-3d", "provider-vast", taskId, "reconciling", Map.of("prompt", "private prompt"), null, "UNKNOWN", "pending");
auth = spy(new AuthService(jdbc, mock(PasswordHasher.class), json, 168));
guard = mock(GatewayGuard.class);
when(guard.tryTaskUserSlot(any(), anyString(), anyLong())).thenReturn(true);
provider = mock(AsyncProviderClient.class);
var settlements = new TaskSettlementService(jdbc, json, guard, usage, manager);
service = new TaskReconciliationService(jdbc, json, auth, settlements, provider, guard, usage, manager, 120_000);
http = MockMvcBuilders.standaloneSetup(new TaskReconciliationController(service))
.setControllerAdvice(new ApiExceptionHandler()).setMessageConverters(new MappingJackson2HttpMessageConverter(json)).build();
}
@AfterEach void cleanup() { if (base != null && schema != null) new JdbcTemplate(base).execute("DROP SCHEMA " + schema + " CASCADE"); }
private void session(String token, UUID user) {
jdbc.update("INSERT INTO auth_sessions(id,token_hash,user_id,expires_at) VALUES (?,?,?,now()+interval '1 hour')", UUID.randomUUID(), AuthService.hashSecret(token), user);
}
private OffsetDateTime version() { return jdbc.queryForObject("SELECT updated_at FROM model_tasks WHERE id=?", OffsetDateTime.class, taskId); }
private String state() { return jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, taskId); }
private BigDecimal wallet(String column) { return jdbc.queryForObject("SELECT " + column + " FROM user_wallets WHERE user_id=?", BigDecimal.class, userId); }
private int count(String table) { return jdbc.queryForObject("SELECT count(*) FROM " + table, Integer.class); }
private TaskReconciliationCommand command(String action) {
return new TaskReconciliationCommand(UUID.randomUUID(), action, version(), "provider-bill/test-001", "Verified against the provider task history and billing record",
"resume_polling".equals(action) ? "original-provider-task" : null,
"resume_polling".equals(action) ? fingerprint : null,
"confirm_succeeded".equals(action) ? new BigDecimal("1.44") : null,
"confirm_succeeded".equals(action) ? Map.of("url", "https://example.test/generated.glb") : null);
}
private Map<String,Object> reconcile(TaskReconciliationCommand command) { return service.reconcile("admin-session", taskId, command, meta); }
private void unchanged() {
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertEquals(0, count("wallet_entries")); assertEquals(0, count("task_reconciliations"));
assertEquals(1, count("gateway_budget_holds"));
}
@Test void realSessionsEnforceAdministratorAndCsrfBarrier() throws Exception {
var command = command("confirm_failed");
http.perform(get("/api/admin/task-reconciliation")).andExpect(status().isUnauthorized());
http.perform(get("/api/admin/task-reconciliation").cookie(new Cookie("moyi_session", "user-session"))).andExpect(status().isForbidden());
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "user-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
unchanged();
verifyNoInteractions(provider, guard);
}
@Test void failureConfirmationReleasesOnceAndReplayReturnsTheOperationReceipt() throws Exception {
var command = command("confirm_failed");
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command)))
.andExpect(status().isOk()).andExpect(jsonPath("$.task.status").value("failed")).andExpect(jsonPath("$.replayed").value(false));
assertEquals(true, reconcile(command).get("replayed"));
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertEquals(0, wallet("reserved_balance").signum());
assertEquals(0, count("wallet_entries")); assertEquals(0, count("gateway_budget_holds"));
assertEquals(1, count("task_reconciliations"));
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM auth_audit_log WHERE event_type='task.reconciled'", Integer.class));
verifyNoInteractions(provider);
}
@Test void successUsesFrozenPriceAndDebitsExactlyOnce() {
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":99}'::jsonb WHERE id='model-mock-3d'");
var command = command("confirm_succeeded");
reconcile(command); reconcile(command);
assertEquals("succeeded", state());
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
assertEquals(0, wallet("reserved_balance").signum());
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
assertEquals(new BigDecimal("1.44000000"), jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId));
verifyNoInteractions(provider);
}
@Test void quoteIsReadOnlyAndUsesTheFrozenVersionNotEditableCatalogPrices() throws Exception {
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":999}'::jsonb WHERE id='model-mock-3d'");
http.perform(post("/api/admin/task-reconciliation/" + taskId + "/quote").cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content("{}"))
.andExpect(status().isOk()).andExpect(jsonPath("$.expectedCost").value("1.44"));
unchanged();
assertEquals(0, count("auth_audit_log"));
}
@Test void concurrentIdenticalOperationsShareOneReceiptAndOneDebit() throws Exception {
var command = command("confirm_succeeded");
var start = new CountDownLatch(1);
try (var executor = Executors.newFixedThreadPool(8)) {
var results = new ArrayList<java.util.concurrent.Future<Map<String,Object>>>();
for (int i = 0; i < 8; i++) results.add(executor.submit(() -> { assertTrue(start.await(10, TimeUnit.SECONDS)); return reconcile(command); }));
start.countDown();
int originals = 0;
for (var result : results) if (Boolean.FALSE.equals(result.get(30, TimeUnit.SECONDS).get("replayed"))) originals++;
assertEquals(1, originals);
}
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
}
@Test void concurrentDifferentDecisionsHaveExactlyOneWinner() throws Exception {
var success = command("confirm_succeeded"); var failure = command("confirm_failed");
var start = new CountDownLatch(1);
try (var executor = Executors.newFixedThreadPool(2)) {
var results = new ArrayList<java.util.concurrent.Future<Integer>>();
for (var command : List.of(success, failure)) results.add(executor.submit(() -> {
assertTrue(start.await(10, TimeUnit.SECONDS));
try { reconcile(command); return 200; } catch (ApiException exception) { return exception.getStatus(); }
}));
start.countDown();
assertEquals(List.of(200, 409), results.stream().map(f -> { try { return f.get(30, TimeUnit.SECONDS); } catch (Exception exception) { throw new AssertionError(exception); } }).sorted().toList());
}
assertEquals(1, count("task_reconciliations")); assertEquals(0, wallet("reserved_balance").signum());
assertEquals("succeeded".equals(state()) ? new BigDecimal("8.56000000") : new BigDecimal("10.00000000"), wallet("balance"));
}
@Test void operationIdCannotBeReusedForDifferentActionOrAnotherAdministrator() {
var command = command("confirm_failed"); reconcile(command);
var changed = new TaskReconciliationCommand(command.requestId(), "confirm_succeeded", command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.44"), Map.of());
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(changed)).getStatus());
assertEquals(409, assertThrows(ApiException.class, () -> service.reconcile("second-admin-session", taskId, command, meta)).getStatus());
assertEquals(1, count("task_reconciliations"));
}
@Test void staleVersionAndWrongExpectedAmountLeaveFinancialStateUnchanged() {
var command = command("confirm_succeeded");
var wrong = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.45"), command.result());
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(wrong)).getStatus()); unchanged();
jdbc.update("UPDATE model_tasks SET updated_at=updated_at+interval '1 second' WHERE id=?", taskId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command)).getStatus()); unchanged();
}
@Test void missingEvidenceCannotAuthorizeARefund() {
var command = command("confirm_failed");
var invalid = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), "", "short", null, null, null, null);
assertEquals(422, assertThrows(ApiException.class, () -> reconcile(invalid)).getStatus()); unchanged();
}
@Test void missingTokenUsageCannotReleaseTheReservationUnderASuccessLabel() {
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
assertEquals("TASK_USAGE_REQUIRED", assertThrows(ApiException.class, () -> reconcile(command("confirm_succeeded"))).getCode()); unchanged();
}
@Test void auditFailureRollsBackTaskWalletUsageAndReceiptTogether() {
var command = command("confirm_succeeded");
doThrow(new IllegalStateException("audit database unavailable")).when(auth).audit(eq("task.reconciled"), eq(adminId), any(), anyMap());
assertThrows(IllegalStateException.class, () -> reconcile(command)); unchanged();
assertNull(jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId));
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
}
@Test void resumeOnlyQueriesTheOriginalBindingWithoutChangingTheWallet() {
var command = command("resume_polling"); reconcile(command); reconcile(command);
assertEquals("running", state());
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
assertEquals(1, count("gateway_budget_holds")); assertEquals(0, count("wallet_entries"));
assertEquals(1, count("task_reconciliations"));
verify(provider, times(1)).requirePollableBinding("provider-vast", "model-mock-3d", "v3.1-20260211", "3d", fingerprint);
verifyNoMoreInteractions(provider);
}
@Test void differentProviderTaskCannotBeAttachedToAnAcceptedJob() {
var command = command("resume_polling");
var other = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), "some-other-task", fingerprint, null, null);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(other)).getStatus()); unchanged(); verifyNoInteractions(provider);
}
@Test void sameProvenUpstreamTaskCannotBeAssociatedWithTwoLocalTasks() {
assertThrows(DataAccessException.class, () -> jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot) VALUES ('duplicate-local-task',?,?,'model-mock-3d','3d','running','',0,'original-provider-task',?,'provider-vast','v3.1-20260211')", userId, keyId, fingerprint));
unchanged();
}
@Test void validTokenUsageIsQuotedAndSettledOnceUsingTheFrozenSchedule() {
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
var result = Map.<String,Object>of("usage", Map.of("input_tokens", 100, "output_tokens", 100));
var quote = service.quote("admin-session", taskId, result);
assertEquals("0.00110000", quote.get("expectedCost"));
var command = new TaskReconciliationCommand(UUID.randomUUID(), "confirm_succeeded", version(), "provider-token-bill", "Verified exact input and output token usage from provider record", null, null, new BigDecimal("0.0011"), result);
reconcile(command); reconcile(command);
assertEquals(new BigDecimal("9.99890000"), wallet("balance"));
assertEquals(0, wallet("reserved_balance").signum());
assertEquals(100L, jdbc.queryForObject("SELECT input_units FROM usage_events WHERE provider_task_id=?", Long.class, taskId));
assertEquals(1, count("wallet_entries"));
}
@Test void unavailableCredentialAndAmbiguousLegacyRouteCannotResumePolling() {
doThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "safe message", 503)).when(provider).requirePollableBinding(anyString(), anyString(), anyString(), anyString(), anyString());
assertEquals(503, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL WHERE id=?", taskId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
}
@Test void existingObservedOutcomeCannotBeOverruledManually() {
jdbc.update("UPDATE model_tasks SET completion_status='succeeded' WHERE id=?", taskId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus()); unchanged();
}
@Test void insufficientHoldIsAConflictNotABalancingPlug() {
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus());
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance")); assertEquals(0, count("task_reconciliations"));
}
@Test void receiptsAreAppendOnlyAndNeverContainPromptOrGeneratedContent() {
reconcile(command("confirm_succeeded"));
String stored = jdbc.queryForObject("SELECT row_to_json(r)::text FROM task_reconciliations r", String.class);
assertFalse(stored.contains("private prompt")); assertFalse(stored.contains("generated.glb"));
assertThrows(DataAccessException.class, () -> jdbc.update("DELETE FROM task_reconciliations"));
assertThrows(DataAccessException.class, () -> jdbc.update("UPDATE task_reconciliations SET note='rewritten history'"));
assertEquals(1, count("task_reconciliations"));
}
@Test void listAndHistoryAreBoundedAdministratorViewsWithoutRawPayloads() {
var list = service.list("admin-session", 1);
assertEquals(1L, list.get("total"));
String encoded = assertDoesNotThrow(() -> json.writeValueAsString(list));
assertFalse(encoded.contains("private prompt")); assertFalse(encoded.contains("input_payload"));
reconcile(command("confirm_failed"));
assertEquals(1, ((List<?>) service.detail("admin-session", taskId).get("history")).size());
assertEquals(0L, service.list("admin-session", 1).get("total"));
}
@Test void freeFailureHasAnExplicitZeroCostAndNoLedgerEntry() {
jdbc.update("UPDATE model_tasks SET reserved_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
reconcile(command("confirm_failed"));
assertEquals(0, jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId).signum());
assertEquals(0, count("wallet_entries"));
}
}
@@ -0,0 +1,34 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class TaskReconciliationResultTest {
private final ObjectMapper json = new ObjectMapper();
@Test void acceptsOnlyNormalizedAssetsAndIntegralUsage() {
var result = TaskReconciliationResult.normalize(Map.of("url", "https://example.test/image.png", "usage", Map.of("input_tokens", "10", "output_tokens", 20,
"input_tokens_details", Map.of("text_tokens", 2, "image_tokens", 8))), json);
assertEquals(10L, ((Map<?,?>) result.get("usage")).get("input_tokens"));
}
@Test void rejectsCredentialsRawPayloadsAndUnsafeAssetUrls() {
for (Map<String,Object> value : List.<Map<String,Object>>of(
Map.of("Authorization", "secret"), Map.of("base64", "large media"),
Map.of("url", "javascript:alert(1)"), Map.of("url", "https://user:password@example.test/a"),
Map.of("usage", Map.of("api_key", "secret")), Map.of("usage", Map.of("input_tokens", -1)),
Map.of("usage", Map.of("output_tokens", 1.5)), Map.of("usage", Map.of("input_tokens", 100_000_001)))) {
assertEquals(422, assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(value, json)).getStatus());
}
}
@Test void rejectsOversizeObjectsAndCommandLoggingOmitsEvidenceAndPayload() {
assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(Map.of("url", "https://example.test/" + "x".repeat(40_000)), json));
var command = new TaskReconciliationCommand(java.util.UUID.randomUUID(), "confirm_succeeded", null, "private bill", "private note", null, null, null, Map.of("url", "private result"));
assertFalse(command.toString().contains("private"));
}
}
+1 -1
View File
@@ -64,7 +64,7 @@ PostgreSQL 集成测试由 `MOYI_AUTH_TEST_JDBC_URL` 启用,当前测试限制
## 下一阶段优先级
1. 已完成 `V45`:异步创建 `Idempotency-Key`、请求哈希和数据库唯一性/锁语义,客户端同键重试不重复排队或预留。
2. 已完成 `V46` 的未知受理持久状态与冻结保护;审计化人工核账入口仍待实现。详情及本阶段 190 项后端验证结果见 `22-async-task-safety.md`。
2. 已完成 `V46` 的未知受理持久状态与冻结保护;V47/V48 后续补人工核账与执行绑定,详见 `23-audited-task-reconciliation.md`。本阶段原始 190 项后端验证结果见 `22-async-task-safety.md`。
3. 已增加异步已观察终态的结算恢复与待核账提示;继续完善执行路由快照、重试退避、人工对账和不可原子提交窗口。
4. HTTP Transport 生命周期继续拆分;真实 Redis 并发、租约回收与队列公平性回归。
5. 再进入用户/角色/会话、管理操作与密钥生命周期的后续安全优化;本轮模块改造不替代完整审计。
+2 -2
View File
@@ -46,11 +46,11 @@
1. 本阶段只验证隔离测试数据库,没有迁移业务数据库或部署线上服务。发布时先停旧 worker,再迁移 `V45` / `V46` 并启新 worker,避免混跑旧的自动退款逻辑。
2. 不回溯改写历史失败任务或重新冻结曾经释放的资金。历史的提交中断、超时、凭据丢失记录须独立审计,不能自动扣款。
3. 待核账状态不会按 TTL 自动退款,不会自动重发。收集本地任务 ID、用户、时间、模型路由、预留金额、上游任务/账单记录;不要把完整凭据放进工单或日志。
4. 暂不提供“无证据一键退款/重新生成”的管理接口。受审计的人工核账入口、关联上游任务编号、人工确认终态以及差额结算是下一阶段产品工作;在该入口完成前,只允许按受控运维流程核对证据,不直接修改钱包余额替代结算事务。
4. 后续 V47/V48 已补受审计的人工核账入口:恢复只读查询、确认失败释放原冻结、按原快照确认成功,详见 `23-audited-task-reconciliation.md`。不提供无证据一键退款、重新生成、任意改余额或追扣已释放资金。
5. 成功的 token 图片任务缺少可核验用量时,仍沿用既有 `TASK_USAGE_PENDING` 规则:保留结果、暂不扣费、释放该次预留并标记待核账。本阶段没有把“已知完成但计量缺失”与“是否受理未知”混为一类。
6. HTTP 受理至观察日志提交之间仍有无法跨外部系统原子提交的窗口。数据库完全不可用时也不能承诺保存回执;恢复后转入待核账,而非声称精确恢复所有生成结果。
后续重点:审计化人工核账入口、异步路由/上游模型的执行快照、共享有界 HTTP Transport 与媒体下载安全、上游有证明的幂等能力适配。
后续人工核账与 provider/model 执行绑定已在 V47/V48 完成,范围及剩余限制见 `23-audited-task-reconciliation.md`。继续重点处理共享有界 HTTP Transport、媒体下载安全、完整配置版本绑定与上游有证明的幂等能力适配。
### 最终验证
+73
View File
@@ -0,0 +1,73 @@
# 人工任务核账与执行绑定
日期:2026-10-21。本阶段只在本地隔离环境实现和验证,未迁移业务数据库、未调用真实供应商生成接口、未发布线上服务。
## 模块边界
| 模块 | 职责 | 明确禁止 |
| --- | --- | --- |
| `TaskReconciliationController` | 管理会话 HTTP 适配,JSON / 自定义头防跨站表单屏障 | 上游生成、直接改账 |
| `TaskReconciliationService` | 管理员校验、证据引用、版本检查、幂等收据、人工决策事务 | 任意指定扣款金额、自动重发生成 |
| `TaskSettlementService` | worker 与人工核账共享的任务/钱包/用量原子结算及预留释放 | 另起一套人工钱包加减逻辑 |
| `TaskOutcomeJournal` | 记录已观察终态和未知结果 | 代替财务决策 |
| `AsyncProviderClient.requirePollableBinding` | 本地验证查询能力及已有凭据指纹 | 验证时发 HTTP、接受原始 API Key |
| `TaskReconciliationResult` | 有界结果地址与规范化数值用量白名单 | 原始上游响应、任意字段、base64、大段内容 |
`AsyncTaskService` 的钱包结算逻辑已移动到共享 `TaskSettlementService`,worker 门面只委派调用;不复制两份扣费逻辑。已知成功但用量缺失的既有 `TASK_USAGE_PENDING` 语义暂不改变;人工成功确认则必须严格校验用量,校验失败不修改状态、不释放冻结。
## 管理端入口
“价格版本与成本对账”页面增加“异步任务人工核账”区块。它与原采购成本核对是不同流程:采购成本核对不改用户消费,任务核账可能将原冻结额转为消费或释放。
- `GET /api/admin/task-reconciliation?page=1`:每页 50 笔待核账任务,返回总数。
- `GET /api/admin/task-reconciliation/{taskId}`:最小任务视图与最近 50 条操作历史。查询不加载任务提示词、输入或生成媒体。
- `POST /api/admin/task-reconciliation/{taskId}/quote`:只读金额预览,请求体为规范化结果/用量对象;使用任务原价格快照,返回金额和版本,不写账。
- `POST /api/admin/task-reconciliation/{taskId}`:执行有证据的核账决定;须有 `Content-Type: application/json`、`X-Moyi-Source: console` 和有效管理员 Cookie 会话。自定义头只是与已有严格 CORS 白名单配合的跨站屏障,不是身份或风控来源证明。
每次操作必须包含 UUID `requestId`、原 `expectedUpdatedAt`(保留服务器完整精度)、`evidenceReference`(3–500 字)、`note`(10–2000 字)。不得将凭据粘入证据/说明。证据为管理员人工核对后填写的引用,服务端不自动证明引用真实,不会访问或下载证据 URL。
### 三类决定
1. **`resume_polling`**:恢复原任务的只读轮询,不生成、不退款、不扣款。需要已证实的上游编号和原账号 SHA-256 指纹;账号须仍在服务端凭据池中,执行器须支持该媒体类型的查询。已知编号/账号不能改绑;原执行绑定无法证明的历史任务禁止猜测恢复。允许对有证据的新式回执丢失任务补原编号。
2. **`confirm_failed`**:管理员根据上游失败或明确未受理的记录,确认失败;共享结算事务释放原冻结和预算占用,不额外给钱包充值。不能附带金额或成功结果。
3. **`confirm_succeeded`**:管理员核实成功与计费用量,先预览并确认 `expectedCost`;金额必须等于原价格快照结合该用量的计算结果。没有快照的历史固定价任务沿用原预留额,不拿当前目录价补算。不能任意填写价格、改变上游绑定,缺少 token 用量时拒绝操作。实际费用可超过原预留,沿用现有真实用量记债语义,界面须显式确认。
结果/用量 JSON 最多 32 KiB,仅允许 HTTP(S) 资产地址、format/mimeType 和数值 usage/usageMetadata。URL 不会被后台下载;禁止 userinfo 和非 HTTP(S) scheme。结果字段仍保存在任务既有 result 列,操作收据不复制媒体或提示词。
## 并发、账务与审计
- 操作前使用真实会话鉴权;事务内锁住仍启用的管理员角色并再次检查会话,防止使用过期的预检查结果。普通用户和无会话请求不可访问,幂等重放也要鉴权。
- 同一操作编号取 PostgreSQL advisory lock,以 `request_id` 主键兜底;请求摘要包含任务、动作、版本、证据、说明、规范化结果及确认金额。只有同一管理员原请求可重放;编号用于不同动作/内容或不同管理员返回 409。
- 锁任务行,验证 `reconciling`、无已观察终态、无既有 cost 和完整 updatedAt 版本;锁钱包验证原冻结额仍存在。不同管理员竞争只有一个成功,另一方刷新后再决定,不覆盖可信终态,不用平衡数填补资金缺口。
- 状态、原价结算/释放、用量审计、`task_reconciliations` 收据及 `auth_audit_log` 在同一数据库事务提交;审计失败时全部回滚。Redis 新租约回滚只释放新租约,不删除事务前已存在的任务租约。
- `V48` 收据保存操作者、操作编号、时间、动作、证据引用、说明和最小前后状态。数据库触发器禁止 UPDATE / DELETE;这不是防数据库所有者、超级用户或禁用触发器的防篡改承诺,生产仍需最小权限和独立审计备份。
- 页面提交后如果响应丢失,原样重试保留同一操作编号;服务端返回原收据,不重复扣款。页面不能在缺少报价/明确确认时提交成功核账;展示超过预留金额的风险。
## 执行绑定 `V47`
- 新创建任务冻结 `provider_id_snapshot` 与 `upstream_model_snapshot`;派发和轮询使用快照,不再从改过的模型目录读取另一个渠道/上游模型。当前目录开关仍可停止新任务派发。
- 升级时只对尚未发往上游的 queued 任务冻结可用的当前绑定。已经受理的历史任务不猜测原渠道,缺少原绑定时保留冻结并进入待核账。
- 已有证明的 provider + 原账号指纹 + 上游任务编号有部分唯一索引,阻止同一上游任务关联两笔本地任务。原始密钥不落库。
- 这不是供应商全部配置的不可变快照:provider 的 base URL、协议、别名与凭据池仍可能被管理员修改。生产应避免修改有在途任务的 provider,后续补配置版本引用、执行适配器版本以及更强关联证据。
## Redis 任务租约
独立 `acquire-task-slot.lua` 用 Redis TIME:同一个任务续期不占第二个槽位,满额时另一任务不能进入。返回值区分“新建租约”和“既有租约续期”;数据库回滚不能误删旧租约。既有租约续期后回滚可能保留延长后的 TTL(保守占位),不是承诺 Redis 与数据库跨系统原子提交。
## 上线要求与剩余范围
1. 先停所有旧 worker,备份数据库并检查在途任务,再按顺序迁移 V47/V48、发布新后端和前端;本轮没有执行这些线上步骤。不能混跑按可变目录轮询的旧 worker。
2. 历史任务没有证明的执行绑定会变为待核账,这是安全失败,不是自动补绑或自动退款。安排运营逐笔核实,不批量猜测。
3. 当前是单管理员明确确认模式,未实现双人审批、强制二次验证、证据真实性校验或自动供应商账单同步;不要将它宣传成已完成完整财务审批。
4. 当前人工入口只覆盖“受理/结果未知且原冻结仍在”的 reconciling 任务;不重开已成功/失败任务、不追扣已释放额度、不解决既有 TASK_USAGE_PENDING 的后补消费。
5. 后续继续共享有界 HTTP Transport、媒体下载 SSRF/DNS 重绑定与响应体读取限制、配置版本绑定、真实供应商端到端联调,以及用户会话/角色/管理操作全面安全复核。本阶段不代表整个仓库已安全。
## 验证记录
先前基础改造 74 项回归和核账首批 21 项专项测试已通过。前端 `npm run build`(含 vue-tsc)与 `npm test` 通过,25 项测试零失败。Browser 技能用于隔离 fixture 的真实交互检查:未报价禁提交、金额确认、模拟提交已落地但响应丢失、原编号重放、历史回执、处理后移出待核账列表;两次模拟 HTTP 提交只产生一条收据。未接触真实钱包、会话或供应商。截图保存在被忽略的 `.tools/reconciliation-ui-proof.png`。
最终验证在 2026-10-21 20:06(Asia/Shanghai)完成:Docker / Java 21 / Maven 3.9.9 执行 `mvn -B -o clean verify`,后端 **221 项测试,0 失败、0 错误、0 跳过**,Spring Boot JAR 构建成功。包含 21 项真实 PostgreSQL 核账集成测试、38 项异步任务事务测试和 3 项真实 Redis 租约测试。独立 PostgreSQL 15.18 与 Redis 7 测试容器和 Maven 共享隔离 network namespace,没有映射到业务数据库。
新增关键覆盖:管理员会话和普通用户越权拒绝、自定义头屏障、8 路同操作编号并发一次扣费、不同决定并发一个胜者、幂等编号跨动作/管理员冲突、任务版本过期、冻结不足不补平、原快照报价与 Token 精确结算、缺少用量不释放、审计失败整体回滚、收据禁止改删、同一上游身份不能关联两笔任务、目录修改不改变已冻结执行绑定、旧绑定未知不猜测轮询、Redis 同任务续期及回滚保护。
本地通过不等于生产发布验收、供应商真实账单正确性或完整财务安全审计。测试日志、构建产物和截图均为忽略文件,不进入 Git 提交。
+1 -1
View File
@@ -4,7 +4,7 @@
"version": "0.1.0",
"type": "module",
"scripts": {
"test": "node --experimental-strip-types --test tests/chat-stream.test.mjs tests/model-catalog.test.mjs tests/pricing.test.mjs tests/media.test.mjs",
"test": "node --experimental-strip-types --test tests/chat-stream.test.mjs tests/model-catalog.test.mjs tests/pricing.test.mjs tests/media.test.mjs tests/task-reconciliation.test.mjs",
"dev": "vite",
"build": "vue-tsc --noEmit && vite build",
"preview": "vite preview"
@@ -2,6 +2,7 @@
import { computed, onMounted, reactive, ref } from 'vue'
import { catalogClient, type CatalogModel } from '../services/catalog'
import { enterprise } from '../services/enterprise'
import TaskReconciliation from './TaskReconciliation.vue'
type Json = Record<string, unknown>
const models = ref<CatalogModel[]>([]), selected = ref(''), loaded = ref(false), busy = ref(false), notice = ref(''), history = ref<Json[]>([]), records = ref<Json[]>([]), summary = ref<Json>({}), page = ref(1)
const retail = ref<Json>({}), procurement = ref<Json | null>(null), retailFields = reactive<Record<string, string>>({}), costFields = reactive<Record<string, string>>({}), costKnown = ref(false), retailTiers = ref('[]'), costTiers = ref('[]'), protocols = ref<string[]>([])
@@ -49,5 +50,6 @@ async function reconcile() { busy.value = true; try { await enterprise(`/api/adm
<form v-if="reconcileId" @submit.prevent="reconcile"><h4>核对 {{ reconcileId }}</h4><label class="form-label">实际采购成本(元)<input v-model="reconcileCost" class="text-input" inputmode="decimal" required /></label><label class="form-label">渠道账单依据<textarea v-model="reconcileNote" class="text-input" minlength="3" maxlength="2000" required /></label><button class="button primary" :disabled="busy">确认成本</button><button type="button" class="button secondary" @click="reconcileId = ''">取消</button></form>
<div class="responsive-table"><table><thead><tr><th>时间/模型</th><th>调用</th><th>用户消费</th><th>采购成本</th><th>核对状态</th><th></th></tr></thead><tbody><tr v-for="r in records" :key="String(r.id)"><td>{{ r.created_at }}<br />{{ r.public_name }}</td><td>{{ r.status }}</td><td>{{ money(r.cost_amount) }}</td><td>{{ money(r.procurement_cost) }}</td><td>{{ state(r.cost_status) }}</td><td><button class="button secondary small" @click="reconcileId = String(r.id); reconcileCost = ''; reconcileNote = ''">核对</button></td></tr></tbody></table></div><div class="protocols"><button class="button secondary" :disabled="page <= 1" @click="page--; refresh().catch(e => notice = String(e))">上一页</button><span>第 {{ page }} 页</span><button class="button secondary" :disabled="records.length < 50" @click="page++; refresh().catch(e => notice = String(e))">下一页</button></div>
</section>
<TaskReconciliation />
</template>
<style scoped>.finance-card{padding:24px;margin-bottom:22px}.finance-card .form-label{flex-direction:column;gap:8px}.finance-card p{font-size:13px;color:#64736d;line-height:1.7}.finance-grid{display:grid;grid-template-columns:1fr 1fr;gap:24px}.finance-card button,.finance-card details{margin-top:16px}.finance-card summary{cursor:pointer}.history{max-height:340px;overflow:auto}.history pre{white-space:pre-wrap;background:#f3f6f4;padding:12px}.protocols{display:flex;align-items:center;gap:18px;flex-wrap:wrap}@media(max-width:700px){.finance-grid{grid-template-columns:1fr}}</style>
@@ -0,0 +1,108 @@
<script setup lang="ts">
import { computed, onMounted, ref, watch } from 'vue'
import UiModal from './UiModal.vue'
import { reconciliationPayload, taskReconciliation, type ReconciliationAction, type ReconciliationCommand, type ReconciliationReceipt, type ReconciliationTask } from '../services/task-reconciliation'
const rows = ref<ReconciliationTask[]>([]), total = ref(0), page = ref(1), loaded = ref(false), busy = ref(false), loading = ref(false), notice = ref('')
const selected = ref<ReconciliationTask | null>(null), history = ref<ReconciliationReceipt[]>([])
const action = ref<ReconciliationAction>('confirm_failed'), evidence = ref(''), note = ref(''), providerTaskId = ref(''), fingerprint = ref(''), resultText = ref('{}'), confirmed = ref(false)
const quote = ref<{ expectedCost: string; expectedUpdatedAt: string } | null>(null)
const pending = ref<{ signature: string; command: ReconciliationCommand } | null>(null)
const actionName = (value: string) => ({ resume_polling: '恢复只读轮询', confirm_failed: '确认失败并释放冻结', confirm_succeeded: '确认成功并结算' })[value] || value
const stateName = (value: string) => ({ reconciling: '待核账', running: '执行中', failed: '失败', succeeded: '成功', cancelled: '已取消' })[value] || value
const actionable = computed(() => selected.value?.status === 'reconciling' && !selected.value.completionStatus && selected.value.costAmount == null)
const bindingKnown = computed(() => !!selected.value?.providerId && !!selected.value?.upstreamModel)
const confirmation = computed(() => action.value === 'confirm_failed'
? `我已核实上游失败或未受理,确认释放该任务冻结 ¥${selected.value?.reservedAmount ?? '—'},不代表给钱包额外加款`
: action.value === 'confirm_succeeded' ? `我已核实任务成功和计费用量,确认实际扣费 ¥${quote.value?.expectedCost ?? '尚未核算'}`
: '我已核实原任务编号和原账号指纹,仅恢复查询,不重新生成、不释放冻结额度')
watch([action, resultText], () => { quote.value = null; confirmed.value = false })
function result() {
if (new TextEncoder().encode(resultText.value).length > 32_768) throw new Error('结果与用量 JSON 最多 32 KiB')
const parsed: unknown = JSON.parse(resultText.value)
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('结果必须是 JSON 对象')
return parsed as Record<string, unknown>
}
async function refresh() {
loading.value = true
try { const data = await taskReconciliation.list(page.value); rows.value = data.records; total.value = data.total; loaded.value = true }
catch (error) { notice.value = error instanceof Error ? error.message : '读取待核账任务失败' }
finally { loading.value = false }
}
onMounted(refresh)
async function open(task: ReconciliationTask) {
if (busy.value) return
busy.value = true; notice.value = ''
try {
const data = await taskReconciliation.detail(task.id)
selected.value = data.task; history.value = data.history
action.value = 'confirm_failed'; evidence.value = ''; note.value = ''; resultText.value = '{}'
providerTaskId.value = data.task.providerTaskId || ''; fingerprint.value = data.task.credentialFingerprint || ''
confirmed.value = false; quote.value = null; pending.value = null
} catch (error) { notice.value = error instanceof Error ? error.message : '读取任务详情失败' }
finally { busy.value = false }
}
async function calculate() {
if (!selected.value) return
busy.value = true; notice.value = ''; confirmed.value = false
try { quote.value = await taskReconciliation.quote(selected.value.id, result()) }
catch (error) { quote.value = null; notice.value = error instanceof Error ? error.message : '核算失败,冻结额度保持不变' }
finally { busy.value = false }
}
async function submit() {
if (!selected.value || !confirmed.value || !actionable.value) return
busy.value = true; notice.value = ''
try {
const body = reconciliationPayload(selected.value, action.value, evidence.value, note.value, providerTaskId.value, fingerprint.value, quote.value,
action.value === 'confirm_succeeded' ? result() : {})
const signature = JSON.stringify(body)
if (!pending.value || pending.value.signature !== signature) pending.value = { signature, command: { requestId: crypto.randomUUID(), ...body } }
const saved = await taskReconciliation.submit(selected.value.id, pending.value.command)
selected.value = saved.task
confirmed.value = false
notice.value = `核账操作已记录${saved.replayed ? '(原操作回执,未重复执行)' : ''},操作编号 ${saved.requestId}`
try { const data = await taskReconciliation.detail(saved.task.id); selected.value = data.task; history.value = data.history; await refresh() }
catch { notice.value += ';操作已成功,但详情刷新失败,请稍后刷新查看' }
} catch (error) {
notice.value = `${error instanceof Error ? error.message : '无法确认操作结果'}${pending.value ? `;原样重试将保留操作编号 ${pending.value.command.requestId},不会重新生成任务` : ''}`
} finally { busy.value = false }
}
</script>
<template>
<section class="panel reconciliation-card">
<header class="panel-header"><div><span>ASYNC TASK RECONCILIATION</span><h3>异步任务人工核账</h3></div><button class="button secondary small" :disabled="loading || busy" @click="refresh">刷新</button></header>
<p>仅处理上游受理或执行结果未知的任务。冻结金额仍占用钱包和预算;先核对渠道任务记录与账单,再决定终态。不会自动重发生成请求。</p>
<p v-if="notice && !selected" role="status" class="reconciliation-notice">{{ notice }}</p>
<div class="responsive-table"><table><thead><tr><th>任务 / 用户</th><th>模型 / 原渠道</th><th>冻结金额</th><th>上游编号</th><th>异常</th><th></th></tr></thead>
<tbody><tr v-for="task in rows" :key="task.id"><td><code>{{ task.id }}</code><small>{{ task.userId }}</small></td><td>{{ task.modelId }}<small>{{ task.providerId || '历史执行绑定待核实' }}</small></td><td>¥ {{ task.reservedAmount }}</td><td>{{ task.providerTaskId || '回执未知' }}</td><td>{{ task.error?.message || '待核实上游记录' }}</td><td><button class="button secondary small" :disabled="busy" @click="open(task)">核账</button></td></tr>
<tr v-if="!rows.length"><td colspan="6">{{ loading ? '正在读取…' : !loaded ? '待核账数据尚未读取' : '暂无待核账任务' }}</td></tr></tbody>
</table></div>
<div class="reconciliation-actions"><button class="button secondary small" :disabled="page <= 1 || loading || busy" @click="page--; refresh()">上一页</button><span>第 {{ page }} 页 · {{ loaded ? total : '—' }} 笔待核账</span><button class="button secondary small" :disabled="page * 50 >= total || loading || busy" @click="page++; refresh()">下一页</button></div>
</section>
<UiModal v-if="selected" title="核实异步任务" eyebrow="EVIDENCE-BACKED DECISION" wide @close="!busy && (selected = null)">
<p v-if="notice" role="status" class="reconciliation-notice">{{ notice }}</p>
<dl class="detail-list"><div><dt>本地任务</dt><dd><code>{{ selected.id }}</code></dd></div><div><dt>状态 / 原预留金额</dt><dd>{{ stateName(selected.status) }} · ¥ {{ selected.reservedAmount }}</dd></div><div><dt>原执行渠道 / 模型</dt><dd>{{ selected.providerId || '不可证明' }} / {{ selected.upstreamModel || '不可证明' }}</dd></div><div><dt>实际扣费</dt><dd>{{ selected.costAmount == null ? '尚未结算' : `¥ ${selected.costAmount}` }}</dd></div></dl>
<details><summary>任务原价格快照(不是当前目录价格)</summary><pre>{{ selected.pricing ? JSON.stringify(selected.pricing, null, 2) : '历史固定价任务:沿用原预留金额;不按当前目录价格补算' }}</pre></details>
<p v-if="!bindingKnown" class="reconciliation-notice">历史任务原执行路由不可证明,不能猜测后恢复查询。核实上游终态后再选择成功或失败确认。</p>
<form v-if="actionable" id="task-reconciliation-form" @submit.prevent="submit">
<fieldset :disabled="busy">
<label class="form-label">核账决定<select v-model="action" class="text-input"><option value="confirm_failed">确认失败并释放冻结</option><option value="confirm_succeeded">确认成功并按原快照结算</option><option value="resume_polling" :disabled="!bindingKnown">关联原任务,恢复只读轮询</option></select></label>
<template v-if="action === 'resume_polling'"><label class="form-label">已核实的上游任务编号<input v-model="providerTaskId" class="text-input" required maxlength="160" :readonly="!!selected.providerTaskId" /></label><label class="form-label">原上游账号 SHA-256 指纹(不是 API Key)<input v-model="fingerprint" class="text-input" required pattern="[0-9a-f]{64}" maxlength="64" :readonly="!!selected.credentialFingerprint" /></label><p>只允许已有服务器凭据池中的账号,不接受粘贴密钥;已知任务编号和账号不可改绑。</p></template>
<template v-if="action === 'confirm_succeeded'"><label class="form-label">核实结果及用量 JSON<textarea v-model="resultText" class="text-input" rows="7" spellcheck="false" required /></label><p>仅接受规范化结果地址、usage 或 usageMetadata。不得粘贴原始响应、密钥或大段生成内容;Token 用量不完整时不能确认成功扣费。</p><button type="button" class="button secondary small" @click="calculate">按原快照核算金额</button><p class="reconciliation-charge">本次实际扣费:{{ quote ? `¥ ${quote.expectedCost}` : '尚未核算' }}。金额可能超过预留,须核对上游用量。</p></template>
<label class="form-label">上游证据引用<input v-model="evidence" class="text-input" minlength="3" maxlength="500" placeholder="渠道账单编号、工单号或任务记录引用;不要包含密钥" required /></label>
<label class="form-label">核账说明<textarea v-model="note" class="text-input" minlength="10" maxlength="2000" rows="3" required placeholder="说明核对了哪些记录及结论,至少 10 字;不要记录凭据" /></label>
<label class="reconciliation-confirm"><input v-model="confirmed" type="checkbox" required :disabled="action === 'confirm_succeeded' && !quote" />{{ confirmation }}</label>
</fieldset>
</form>
<p v-else>此任务已被处理或存在可信终态,不允许再次人工改写。请查看操作历史。</p>
<section v-if="history.length" class="reconciliation-history"><h4>最近核账历史</h4><article v-for="receipt in history" :key="receipt.request_id"><strong>{{ actionName(receipt.action) }} · {{ stateName(receipt.after_state.status) }}</strong><small>{{ receipt.created_at }} · {{ receipt.actor_user_id }}</small><p>{{ receipt.evidence_reference }} · {{ receipt.note }}</p><code>{{ receipt.request_id }}</code></article></section>
<template #footer><button type="button" class="button secondary" :disabled="busy" @click="selected = null">关闭</button><button v-if="actionable" class="button primary" form="task-reconciliation-form" :disabled="busy || !confirmed || (action === 'confirm_succeeded' && !quote)">{{ busy ? '正在提交…' : '确认并记录审计' }}</button></template>
</UiModal>
</template>
<style scoped>
.reconciliation-card{padding:24px;margin-top:22px}.reconciliation-card p,.reconciliation-actions,.reconciliation-history p{font-size:13px;color:#64736d;line-height:1.7}.reconciliation-card td small,.reconciliation-history small{display:block;margin-top:5px;color:#64736d}.reconciliation-card code{word-break:break-all}.reconciliation-actions{display:flex;gap:16px;align-items:center;margin-top:16px}.reconciliation-notice{padding:12px;border:1px solid #f0dcc6;background:#fffaf4;color:#8b5218;word-break:break-word}.form-label{display:flex;flex-direction:column;gap:8px;margin-top:18px}fieldset{border:0;margin:0;padding:0;min-width:0}.reconciliation-confirm{display:flex;gap:10px;align-items:flex-start;margin-top:20px;font-size:13px;line-height:1.7}.reconciliation-charge{font-weight:600}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#f3f6f4;padding:12px;font-size:12px;max-height:220px;overflow:auto}.reconciliation-history{margin-top:24px}.reconciliation-history article{padding:14px 0;border-top:1px solid #e5e9e7}.reconciliation-history code{font-size:11px;overflow-wrap:anywhere}
</style>
@@ -0,0 +1,17 @@
import type { ReconciliationTask, ReconciliationAction, ReconciliationCommand } from './task-reconciliation'
/** Build one explicit decision. Its request ID is retained by the UI until the payload changes. */
export function reconciliationPayload(task: ReconciliationTask, action: ReconciliationAction, evidenceReference: string,
note: string, providerTaskId: string, credentialFingerprint: string, quote: { expectedCost: string; expectedUpdatedAt: string } | null,
result: Record<string, unknown>): Omit<ReconciliationCommand, 'requestId'> {
const payload: Omit<ReconciliationCommand, 'requestId'> = { action, expectedUpdatedAt: task.updatedAt, evidenceReference: evidenceReference.trim(), note: note.trim() }
if (action === 'resume_polling') {
payload.providerTaskId = providerTaskId.trim()
payload.credentialFingerprint = credentialFingerprint.trim()
} else if (action === 'confirm_succeeded') {
if (!quote || quote.expectedUpdatedAt !== task.updatedAt) throw new Error('请先按任务原价格快照核算并确认金额;任务版本变化后需重新打开核对')
payload.expectedCost = quote.expectedCost
payload.result = result
}
return payload
}
@@ -0,0 +1,52 @@
import { enterprise } from './enterprise'
export type ReconciliationAction = 'resume_polling' | 'confirm_failed' | 'confirm_succeeded'
export interface ReconciliationTask {
id: string
userId: string
modelId: string
taskType: string
status: string
updatedAt: string
reservedAmount: string
costAmount: string | null
providerId: string | null
upstreamModel: string | null
providerTaskId: string | null
credentialFingerprint: string | null
completionStatus: string | null
requiresReconciliation: boolean
pricing: Record<string, unknown> | null
error: { code?: string; message?: string } | null
}
export interface ReconciliationReceipt {
request_id: string
actor_user_id: string
action: ReconciliationAction
evidence_reference: string
note: string
created_at: string
before_state: ReconciliationTask
after_state: ReconciliationTask
}
export interface ReconciliationCommand {
requestId: string
action: ReconciliationAction
expectedUpdatedAt: string
evidenceReference: string
note: string
providerTaskId?: string
credentialFingerprint?: string
expectedCost?: string
result?: Record<string, unknown>
}
const root = '/api/admin/task-reconciliation'
export const taskReconciliation = {
list(page: number) { return enterprise<{ records: ReconciliationTask[]; total: number; page: number }>(`${root}?page=${page}`) },
detail(id: string) { return enterprise<{ task: ReconciliationTask; history: ReconciliationReceipt[] }>(`${root}/${encodeURIComponent(id)}`) },
quote(id: string, result: Record<string, unknown>) { return enterprise<{ expectedCost: string; expectedUpdatedAt: string }>(`${root}/${encodeURIComponent(id)}/quote`, result, 'POST') },
submit(id: string, command: ReconciliationCommand) { return enterprise<{ task: ReconciliationTask; requestId: string; replayed: boolean }>(`${root}/${encodeURIComponent(id)}`, command, 'POST') },
}
export { reconciliationPayload } from './task-reconciliation-state'
@@ -0,0 +1,2 @@
<!doctype html>
<html lang="zh-CN"><head><meta charset="UTF-8" /><meta name="viewport" content="width=device-width,initial-scale=1" /><title>Isolated Task Reconciliation Preview</title></head><body><div id="app"></div><pre id="preview-observations" aria-label="测试回执状态"></pre><script type="module" src="/tests/reconciliation-preview.ts"></script></body></html>
+36
View File
@@ -0,0 +1,36 @@
// Dedicated UI fixture. No real backend, credentials, wallet or provider is contacted.
import { createApp } from 'vue'
import TaskReconciliation from '../src/components/TaskReconciliation.vue'
import '../src/styles.css'
import type { ReconciliationTask, ReconciliationCommand, ReconciliationReceipt } from '../src/services/task-reconciliation'
const task: ReconciliationTask = { id: 'ui-reconciliation-task', userId: 'isolated-test-user', modelId: 'isolated-test-model', taskType: '3d', status: 'reconciling', updatedAt: '2026-10-21T11:00:00Z', reservedAmount: '1.44000000', costAmount: null, providerId: 'isolated-test-provider', upstreamModel: 'isolated-test-upstream', providerTaskId: 'isolated-provider-task', credentialFingerprint: 'a'.repeat(64), completionStatus: null, requiresReconciliation: true, pricing: { unit: 'CNY / task', price: 1.44 }, error: { code: 'TEST_TIMEOUT', message: '隔离测试:上游结果待核账,额度仍冻结' } }
const receipts: ReconciliationReceipt[] = []
let accepted: ReconciliationCommand | null = null
let attempts = 0
const respond = (data: unknown, status = 200) => Promise.resolve(new Response(JSON.stringify(data), { status, headers: { 'Content-Type': 'application/json' } }))
function observations() { const output = document.getElementById('preview-observations'); if (output) output.textContent = JSON.stringify({ testOnly: true, wireAttempts: attempts, uniqueReceipts: receipts.length, requestId: accepted?.requestId || null, status: task.status }, null, 2) }
window.fetch = async (input, init) => {
const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url, location.origin)
if (!url.pathname.startsWith('/api/admin/task-reconciliation')) return respond({ error: { message: '隔离测试禁止其他 API 请求' } }, 404)
if (url.pathname.endsWith('/quote')) return respond({ expectedCost: '1.44', expectedUpdatedAt: task.updatedAt })
if (init?.method === 'POST') {
attempts++
const command = JSON.parse(String(init.body)) as ReconciliationCommand
if (!accepted) {
accepted = command
const before = { ...task }
task.status = 'succeeded'; task.costAmount = '1.44000000'; task.requiresReconciliation = false; task.updatedAt = '2026-10-21T11:01:00Z'
receipts.push({ request_id: command.requestId, actor_user_id: 'isolated-test-admin', action: command.action, evidence_reference: command.evidenceReference, note: command.note, created_at: task.updatedAt, before_state: before, after_state: { ...task } })
observations()
throw new TypeError('隔离测试:服务端已提交,但响应丢失,请原样重试')
}
observations()
if (JSON.stringify(accepted) !== JSON.stringify(command)) return respond({ error: { message: '隔离测试发现重复提交没有保留原操作编号或请求内容' } }, 409)
return respond({ task: { ...task }, requestId: command.requestId, replayed: true })
}
if (url.pathname.endsWith('/ui-reconciliation-task')) return respond({ task: { ...task }, history: receipts })
return respond({ records: task.status === 'reconciling' ? [{ ...task }] : [], total: task.status === 'reconciling' ? 1 : 0, page: 1 })
}
observations()
createApp(TaskReconciliation).mount('#app')
@@ -0,0 +1,27 @@
import test from 'node:test'
import assert from 'node:assert/strict'
import { reconciliationPayload } from '../src/services/task-reconciliation-state.ts'
const task = { id: 'test-task', updatedAt: '2026-10-21T11:00:00Z' }
test('success requires a server quote for the same frozen task version', () => {
assert.throws(() => reconciliationPayload(task, 'confirm_succeeded', 'bill', 'verified note', '', '', null, {}))
assert.throws(() => reconciliationPayload(task, 'confirm_succeeded', 'bill', 'verified note', '', '', { expectedCost: '1.44', expectedUpdatedAt: 'old' }, {}))
const value = reconciliationPayload(task, 'confirm_succeeded', ' bill ', ' verified note ', 'unused', 'unused', { expectedCost: '0.00000001', expectedUpdatedAt: task.updatedAt }, { usage: { input_tokens: 1, output_tokens: 1 } })
assert.equal(value.expectedCost, '0.00000001')
assert.equal(value.expectedUpdatedAt, task.updatedAt)
assert.equal(value.evidenceReference, 'bill')
assert.equal('providerTaskId' in value, false)
})
test('a failure confirmation cannot accidentally include a charge or generated result', () => {
const value = reconciliationPayload(task, 'confirm_failed', 'bill', 'verified note', 'unused', 'unused', { expectedCost: '100', expectedUpdatedAt: task.updatedAt }, { url: 'https://example.test/x' })
assert.equal('expectedCost' in value, false)
assert.equal('result' in value, false)
assert.equal('credentialFingerprint' in value, false)
})
test('resume only carries the original task binding and never a financial adjustment', () => {
const value = reconciliationPayload(task, 'resume_polling', 'bill', 'verified note', ' original-task ', ' fingerprint ', null, {})
assert.equal(value.providerTaskId, 'original-task')
assert.equal(value.credentialFingerprint, 'fingerprint')
assert.equal('result' in value, false)
assert.equal('expectedCost' in value, false)
})