feat: add audited async task reconciliation and frozen bindings
This commit is contained in:
1 parent
5be06a97c1
commit
c716ec7478
28 files changed
+1312
-76
No files matched your search
@@ -23,6 +23,7 @@ public class GatewayGuard {
|
||||
private static final long SLOT_TTL_MS = 180_000;
|
||||
private static final DefaultRedisScript<Long> RATE = script("redis/rate-limit.lua");
|
||||
private static final DefaultRedisScript<Long> ACQUIRE = script("redis/acquire-slot.lua");
|
||||
private static final DefaultRedisScript<Long> ACQUIRE_TASK = script("redis/acquire-task-slot.lua");
|
||||
private static final DefaultRedisScript<Long> ACQUIRE_PAIR = script("redis/acquire-pair.lua");
|
||||
private static final DefaultRedisScript<List> CAPACITY = new DefaultRedisScript<>("""
|
||||
local time = redis.call('TIME')
|
||||
@@ -163,9 +164,11 @@ public class GatewayGuard {
|
||||
if(policies==null) return true;
|
||||
var policy=policies.taskLimit(keyId);
|
||||
String bucket=redisPrefix+"gateway:slots:user:"+policy.userId();
|
||||
boolean acquired=execute(ACQUIRE,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId)==1;
|
||||
long result=execute(ACQUIRE_TASK,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId);
|
||||
boolean acquired=result>0;
|
||||
// Redis is outside the database transaction: roll back the lease if claiming the task fails.
|
||||
if(acquired&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
|
||||
// Rolling back renewal must not remove a lease that existed before this transaction.
|
||||
if(result==1&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
|
||||
org.springframework.transaction.support.TransactionSynchronizationManager.registerSynchronization(new org.springframework.transaction.support.TransactionSynchronization() {@Override public void afterCompletion(int status) {if(status!=STATUS_COMMITTED) releaseQuietly(bucket,taskId);}});
|
||||
return acquired;
|
||||
}
|
||||
|
||||
@@ -110,6 +110,17 @@ public class AsyncProviderClient {
|
||||
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
|
||||
}
|
||||
|
||||
/** Validate a proven polling binding without sending any HTTP request or exposing credential values. */
|
||||
public void requirePollableBinding(String providerId, String routeId, String upstreamModel, String taskType, String fingerprint) {
|
||||
if (fingerprint == null || !fingerprint.matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "需要有效的上游账号指纹", 422);
|
||||
Provider provider = provider(providerId, routeId, upstreamModel);
|
||||
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
|
||||
boolean pollable = isTripo(provider.code(), provider.baseUrl()) ? "3d".equals(taskType)
|
||||
: (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com") || "openai-compatible".equalsIgnoreCase(provider.protocol())) && "video".equals(taskType);
|
||||
if (!pollable) throw new ApiException("TASK_NOT_POLLABLE", "该执行器没有可恢复的只读任务查询接口,请核实终态", 409);
|
||||
provider.keyForFingerprint(routeId, upstreamModel, environment, keyCursors, fingerprint);
|
||||
}
|
||||
|
||||
public Map<String,Object> poll(String providerId, String providerTaskId) {
|
||||
return poll(providerId, null, providerTaskId);
|
||||
}
|
||||
|
||||
@@ -19,8 +19,6 @@ import java.sql.SQLException;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -52,6 +50,7 @@ public class AsyncTaskService {
|
||||
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
|
||||
private final ApiRequestRiskPolicy requestRisk;
|
||||
private final TaskOutcomeJournal outcomes;
|
||||
private final TaskSettlementService settlements;
|
||||
|
||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
@@ -108,6 +107,7 @@ public class AsyncTaskService {
|
||||
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
|
||||
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
|
||||
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
|
||||
this.settlements = new TaskSettlementService(jdbc, objectMapper, gatewayGuard, usage, transactionManager);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -177,6 +177,8 @@ public class AsyncTaskService {
|
||||
}
|
||||
requestRisk.enforce(new ApiRequestContext(userId, apiKeyId, ApiRequestProtocol.ASYNC_TASK,
|
||||
requestedModelId, modelId, false, estimateInputUnits(input), 0, metadata, System.nanoTime()));
|
||||
String upstreamModel = model.get("upstreamModel") instanceof String value ? value : null;
|
||||
if (upstreamModel == null || upstreamModel.isBlank()) throw new ApiException("ASYNC_BINDING_MISSING", "模型缺少可冻结的上游执行标识", 503);
|
||||
// Serialize the queue-capacity check with task insertion for this
|
||||
// model. Without the transaction advisory lock, concurrent creates
|
||||
// could all observe the same count and exceed maxQueuedPerModel.
|
||||
@@ -198,8 +200,8 @@ public class AsyncTaskService {
|
||||
OffsetDateTime now = now();
|
||||
try {
|
||||
jdbc.update(
|
||||
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?)",
|
||||
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash()
|
||||
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash, provider_id_snapshot, upstream_model_snapshot) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?, ?, ?)",
|
||||
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash(), model.get("providerId"), upstreamModel
|
||||
);
|
||||
} catch (RuntimeException exception) {
|
||||
releaseWallet(userId, reservedAmount);
|
||||
@@ -253,7 +255,7 @@ public class AsyncTaskService {
|
||||
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
|
||||
);
|
||||
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model FROM model_tasks t WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||
for (Map<String,Object> row : pending) pollProviderTask(row);
|
||||
dispatchQueuedTasks();
|
||||
}
|
||||
@@ -314,10 +316,10 @@ public class AsyncTaskService {
|
||||
// workers reliably.
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
|
||||
String sql = """
|
||||
SELECT t.id, t.user_id, t.api_key_id, t.model_id, m.provider_id, m.upstream_model, t.task_type, t.prompt, t.input
|
||||
SELECT t.id, t.user_id, t.api_key_id, t.model_id, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model, t.task_type, t.prompt, t.input
|
||||
FROM model_tasks t
|
||||
JOIN model_catalog m ON m.id=t.model_id
|
||||
JOIN model_providers p ON p.id=m.provider_id
|
||||
JOIN model_providers p ON p.id=t.provider_id_snapshot
|
||||
WHERE t.status='queued'
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
@@ -373,35 +375,7 @@ public class AsyncTaskService {
|
||||
}
|
||||
|
||||
private void finishTask(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
|
||||
// The terminal state and wallet change must commit together. Otherwise a
|
||||
// crash after marking failed/succeeded leaves a hold that polling never
|
||||
// revisits. The conditional update also prevents duplicate settlement.
|
||||
inTransaction(() -> {
|
||||
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
|
||||
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
|
||||
// A late failure must not override a durably observed success whose settlement is pending.
|
||||
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
|
||||
AsyncPriceSchedule.Bill bill=null;
|
||||
String finalCode=code,finalMessage=message;
|
||||
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
|
||||
try { bill=AsyncPriceSchedule.bill(readMap(String.valueOf(task.get("pricing_snapshot"))),result); }
|
||||
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
|
||||
}
|
||||
int changed = jdbc.update(
|
||||
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
|
||||
status, progress, result == null ? null : writeJson(result),
|
||||
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
|
||||
if (changed == 0) return null;
|
||||
if ("TASK_USAGE_PENDING".equals(finalCode)) {
|
||||
releaseTaskInTransaction(taskId);
|
||||
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?",taskId);
|
||||
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'",writeJson(usageOnly(result)),taskId);
|
||||
}
|
||||
else if ("succeeded".equals(status)) settleTaskInTransaction(taskId,bill);
|
||||
else releaseTaskInTransaction(taskId);
|
||||
usage.updateTask(taskId, status, result, finalCode, finalMessage);
|
||||
return null;
|
||||
});
|
||||
settlements.finish(taskId, status, progress, result, code, message);
|
||||
}
|
||||
|
||||
private Map<String, Object> requireKey(String authorization) {
|
||||
@@ -476,6 +450,10 @@ public class AsyncTaskService {
|
||||
|
||||
private void pollProviderTask(Map<String,Object> row) {
|
||||
String taskId = String.valueOf(row.get("id"));
|
||||
if (row.get("provider_id") == null || row.get("upstream_model") == null) {
|
||||
outcomes.requireReconciliation(taskId, "ASYNC_BINDING_MISSING", "历史任务缺少可证明的原始执行路由;冻结额度保留,请根据上游记录核账", false);
|
||||
return;
|
||||
}
|
||||
OffsetDateTime deadline = taskTimestamp(row.get("provider_deadline_at"));
|
||||
if (deadline == null) {
|
||||
// Defensive fallback for pre-migration tasks. The migration also
|
||||
@@ -622,42 +600,10 @@ public class AsyncTaskService {
|
||||
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, userId);
|
||||
}
|
||||
|
||||
private void settleTaskInTransaction(String taskId,AsyncPriceSchedule.Bill bill) {
|
||||
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
|
||||
BigDecimal amount = bill==null ? reserved : bill.amount();
|
||||
policies.release(taskId);
|
||||
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
|
||||
if (amount == null || task.get("cost_amount") != null) return;
|
||||
UUID userId = (UUID) task.get("user_id");
|
||||
Map<String,Object> wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
|
||||
BigDecimal before = (BigDecimal) wallet.get("balance");
|
||||
BigDecimal after = before.subtract(amount);
|
||||
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
|
||||
if(amount.signum()>0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)", UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
|
||||
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
|
||||
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
|
||||
if(bill!=null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",bill.input(),bill.output(),bill.input(),bill.output(),bill.imageInput(),taskId);
|
||||
}
|
||||
|
||||
private void releaseTaskInTransaction(String taskId) {
|
||||
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
|
||||
policies.release(taskId);
|
||||
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
|
||||
if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) {
|
||||
releaseWallet((UUID) task.get("user_id"), amount);
|
||||
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
|
||||
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
|
||||
}
|
||||
}
|
||||
|
||||
private <T> T inTransaction(Supplier<T> work) {
|
||||
return transactions == null ? work.get() : transactions.execute(status -> work.get());
|
||||
}
|
||||
|
||||
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
|
||||
|
||||
private void putIfPresent(Map<String,Object> target, String name, ResultSet resultSet, String column) throws SQLException {
|
||||
Object value = resultSet.getObject(column);
|
||||
if (value != null) target.put(name, value);
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/** Administrative evidence and optimistic-concurrency preconditions, never a raw provider key. */
|
||||
public record TaskReconciliationCommand(
|
||||
UUID requestId,
|
||||
String action,
|
||||
OffsetDateTime expectedUpdatedAt,
|
||||
String evidenceReference,
|
||||
String note,
|
||||
String providerTaskId,
|
||||
String credentialFingerprint,
|
||||
BigDecimal expectedCost,
|
||||
Map<String,Object> result
|
||||
) {
|
||||
@Override public String toString() { return "TaskReconciliationCommand[requestId=" + requestId + ", action=" + action + "]"; }
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.moyi.controlplane.auth.AuthController;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import java.util.Map;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/admin/task-reconciliation")
|
||||
public class TaskReconciliationController {
|
||||
private final TaskReconciliationService reconciliations;
|
||||
public TaskReconciliationController(TaskReconciliationService reconciliations) { this.reconciliations = reconciliations; }
|
||||
|
||||
@GetMapping
|
||||
public Map<String,Object> list(@RequestParam(defaultValue="1") int page, HttpServletRequest request) {
|
||||
return reconciliations.list(AuthController.sessionToken(request), page);
|
||||
}
|
||||
|
||||
@GetMapping("/{taskId}")
|
||||
public Map<String,Object> detail(@PathVariable String taskId, HttpServletRequest request) {
|
||||
return reconciliations.detail(AuthController.sessionToken(request), taskId);
|
||||
}
|
||||
|
||||
@PostMapping(value="/{taskId}", consumes="application/json")
|
||||
public Map<String,Object> reconcile(@PathVariable String taskId, @RequestBody TaskReconciliationCommand command, HttpServletRequest request) {
|
||||
// A custom header + JSON requires CORS preflight; the header is a CSRF barrier, not authentication.
|
||||
if (!"console".equals(request.getHeader("X-Moyi-Source"))) throw new ApiException("CONSOLE_REQUEST_REQUIRED", "请通过管理控制台发起核账", 403);
|
||||
return reconciliations.reconcile(AuthController.sessionToken(request), taskId, command, AuthController.meta(request));
|
||||
}
|
||||
|
||||
@PostMapping(value="/{taskId}/quote", consumes="application/json")
|
||||
public Map<String,Object> quote(@PathVariable String taskId, @RequestBody Map<String,Object> result, HttpServletRequest request) {
|
||||
return reconciliations.quote(AuthController.sessionToken(request), taskId, result);
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import java.math.BigDecimal;
|
||||
import java.net.URI;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/** Only normalized assets and numeric usage may enter a manual completion, never raw provider payloads. */
|
||||
final class TaskReconciliationResult {
|
||||
private static final Set<String> ASSETS = Set.of("url", "modelUrl", "videoUrl", "imageUrl", "audioUrl", "previewUrl");
|
||||
private static final Set<String> OPENAI = Set.of("input_tokens", "output_tokens", "total_tokens", "cached_tokens", "input_tokens_details");
|
||||
private static final Set<String> DETAILS = Set.of("text_tokens", "image_tokens", "cached_tokens", "cached_tokens_details");
|
||||
private static final Set<String> CACHED = Set.of("text_tokens", "image_tokens");
|
||||
private static final Set<String> GEMINI = Set.of("promptTokenCount", "candidatesTokenCount", "thoughtsTokenCount", "cachedContentTokenCount", "totalTokenCount");
|
||||
|
||||
static Map<String,Object> normalize(Map<String,Object> input, ObjectMapper json) {
|
||||
if (input == null) throw invalid();
|
||||
try { if (json.writeValueAsBytes(input).length > 32_768) throw invalid(); }
|
||||
catch (JsonProcessingException exception) { throw invalid(); }
|
||||
var result = new LinkedHashMap<String,Object>();
|
||||
for (var entry : input.entrySet()) {
|
||||
String field = entry.getKey();
|
||||
Object value = entry.getValue();
|
||||
if (ASSETS.contains(field)) {
|
||||
if (!(value instanceof String text) || text.length() > 4096 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
|
||||
try {
|
||||
URI uri = URI.create(text);
|
||||
if (!("https".equalsIgnoreCase(uri.getScheme()) || "http".equalsIgnoreCase(uri.getScheme())) || uri.getHost() == null || uri.getUserInfo() != null) throw invalid();
|
||||
} catch (IllegalArgumentException exception) { throw invalid(); }
|
||||
result.put(field, text);
|
||||
} else if (Set.of("format", "mimeType").contains(field)) {
|
||||
if (!(value instanceof String text) || text.isBlank() || text.length() > 128 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
|
||||
result.put(field, text);
|
||||
} else if ("usage".equals(field)) result.put(field, numericMap(value, OPENAI));
|
||||
else if ("usageMetadata".equals(field)) result.put(field, numericMap(value, GEMINI));
|
||||
else throw invalid();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Map<String,Object> numericMap(Object value, Set<String> allowed) {
|
||||
if (!(value instanceof Map<?,?> map)) throw invalid();
|
||||
var result = new LinkedHashMap<String,Object>();
|
||||
for (var entry : map.entrySet()) {
|
||||
if (!(entry.getKey() instanceof String name) || !allowed.contains(name)) throw invalid();
|
||||
if ("input_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), DETAILS));
|
||||
else if ("cached_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), CACHED));
|
||||
else {
|
||||
try {
|
||||
long count = new BigDecimal(String.valueOf(entry.getValue())).longValueExact();
|
||||
if (count < 0 || count > 100_000_000) throw invalid();
|
||||
result.put(name, count);
|
||||
} catch (NumberFormatException | ArithmeticException exception) { throw invalid(); }
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static ApiException invalid() { return new ApiException("INVALID_RECONCILIATION_RESULT", "只接受有界的 HTTP(S) 结果地址和规范化数值用量,不接受原始上游响应或密钥", 422); }
|
||||
}
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.auth.AuthService;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.gateway.GatewayGuard;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
/** Evidence-backed administrative decisions; no generation, arbitrary balance adjustment or transparent replay. */
|
||||
@Service
|
||||
public class TaskReconciliationService {
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper json;
|
||||
private final AuthService auth;
|
||||
private final TaskSettlementService settlements;
|
||||
private final AsyncProviderClient provider;
|
||||
private final GatewayGuard guard;
|
||||
private final UsageAuditService usage;
|
||||
private final TransactionTemplate transactions;
|
||||
private final long timeoutMs;
|
||||
// List/detail must not load prompts, original inputs or potentially large generated media.
|
||||
private static final String TASK_COLUMNS = "SELECT t.id,t.user_id,t.api_key_id,t.model_id,t.task_type,t.status,t.updated_at,t.reserved_amount,t.cost_amount,t.provider_id_snapshot,t.upstream_model_snapshot,t.provider_task_id,t.provider_key_fingerprint,t.completion_status,t.pricing_snapshot::text AS frozen_pricing,t.error::text AS error FROM model_tasks t";
|
||||
|
||||
public TaskReconciliationService(JdbcTemplate jdbc, ObjectMapper json, AuthService auth,
|
||||
TaskSettlementService settlements, AsyncProviderClient provider, GatewayGuard guard,
|
||||
UsageAuditService usage, PlatformTransactionManager manager,
|
||||
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long timeoutMs) {
|
||||
this.jdbc = jdbc; this.json = json; this.auth = auth; this.settlements = settlements;
|
||||
this.provider = provider; this.guard = guard; this.usage = usage;
|
||||
this.transactions = new TransactionTemplate(manager);
|
||||
if (timeoutMs < 1000) throw new IllegalArgumentException("Polling deadline must be positive");
|
||||
this.timeoutMs = timeoutMs;
|
||||
}
|
||||
|
||||
public Map<String,Object> list(String token, int requestedPage) {
|
||||
auth.requireAdmin(token);
|
||||
int page = Math.max(1, Math.min(100000, requestedPage));
|
||||
return Map.of("records", jdbc.queryForList(TASK_COLUMNS + " WHERE t.status='reconciling' ORDER BY t.reconciliation_required_at NULLS FIRST,t.id LIMIT 50 OFFSET ?", (page - 1) * 50).stream().map(this::view).toList(),
|
||||
"total", jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE status='reconciling'", Long.class), "page", page);
|
||||
}
|
||||
|
||||
public Map<String,Object> detail(String token, String taskId) {
|
||||
auth.requireAdmin(token);
|
||||
Map<String,Object> task = task(taskId, false);
|
||||
var history = jdbc.queryForList("SELECT request_id,actor_user_id,action,evidence_reference,note,before_state::text AS before_state,after_state::text AS after_state,created_at FROM task_reconciliations WHERE task_id=? ORDER BY created_at DESC LIMIT 50", taskId);
|
||||
var records = history.stream().map(row -> {
|
||||
var item = new LinkedHashMap<String,Object>();
|
||||
for (String field : List.of("request_id", "actor_user_id", "action", "evidence_reference", "note", "created_at")) item.put(field, row.get(field));
|
||||
item.put("before_state", decode(row.get("before_state")));
|
||||
item.put("after_state", decode(row.get("after_state")));
|
||||
return item;
|
||||
}).toList();
|
||||
return Map.of("task", view(task), "history", records);
|
||||
}
|
||||
|
||||
public Map<String,Object> quote(String token, String taskId, Map<String,Object> input) {
|
||||
auth.requireAdmin(token);
|
||||
var task = task(taskId, false);
|
||||
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可核账状态,请刷新");
|
||||
var result = TaskReconciliationResult.normalize(input, json);
|
||||
return Map.of("expectedCost", confirmedCharge(task, result).toPlainString(),
|
||||
"expectedUpdatedAt", timestamp(task.get("updated_at")).toInstant().toString());
|
||||
}
|
||||
|
||||
public Map<String,Object> reconcile(String token, String taskId, TaskReconciliationCommand command, AuthService.RequestMeta meta) {
|
||||
var actor = auth.requireAdmin(token); // Authorization is required even for replay.
|
||||
validate(command);
|
||||
Map<String,Object> result = "confirm_succeeded".equals(command.action()) ? TaskReconciliationResult.normalize(command.result(), json) : null;
|
||||
String requestHash = requestHash(command, taskId, result);
|
||||
return transactions.execute(status -> {
|
||||
// Keep role/status revocation ordered with this financial decision, not a stale pre-check.
|
||||
if (jdbc.queryForList("SELECT id FROM users WHERE id=? AND status='active' AND role IN ('admin','super_admin') FOR SHARE", actor.id()).isEmpty()) throw new ApiException("FORBIDDEN", "管理员权限已变化,请重新登录", 403);
|
||||
auth.requireAdmin(token); // Recheck session revocation/expiry after entering the transaction.
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))", "moyi:task-reconciliation:" + command.requestId());
|
||||
var prior = jdbc.queryForList("SELECT actor_user_id,request_hash,after_state::text AS after_state FROM task_reconciliations WHERE request_id=?", command.requestId());
|
||||
if (!prior.isEmpty()) {
|
||||
var receipt = prior.getFirst();
|
||||
if (!actor.id().equals(receipt.get("actor_user_id")) || !requestHash.equals(receipt.get("request_hash"))) throw conflict("此核账请求编号已用于不同的操作");
|
||||
return Map.of("task", decode(receipt.get("after_state")), "requestId", command.requestId(), "replayed", true);
|
||||
}
|
||||
var task = task(taskId, true);
|
||||
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可人工核账状态,或已有可信终态;请刷新");
|
||||
if (!timestamp(task.get("updated_at")).toInstant().equals(command.expectedUpdatedAt().toInstant())) throw conflict("任务已被其他操作更新,请刷新后重新核对");
|
||||
var wallets = jdbc.queryForList("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", task.get("user_id"));
|
||||
if (wallets.isEmpty() || ((BigDecimal) wallets.getFirst().get("reserved_balance")).compareTo((BigDecimal) task.get("reserved_amount")) < 0) throw conflict("任务冻结额度与钱包不一致,请先排查账本,不允许直接覆盖");
|
||||
Map<String,Object> before = view(task);
|
||||
if ("resume_polling".equals(command.action())) resume(task, command);
|
||||
else {
|
||||
if ("confirm_succeeded".equals(command.action())) {
|
||||
BigDecimal charge = confirmedCharge(task, result);
|
||||
if (charge == null || charge.compareTo(command.expectedCost()) != 0) throw conflict("确认金额与任务原价格快照及实际用量不一致");
|
||||
}
|
||||
String target = "confirm_succeeded".equals(command.action()) ? "succeeded" : "failed";
|
||||
String code = "failed".equals(target) ? "TASK_RECONCILED_FAILED" : null;
|
||||
String message = "failed".equals(target) ? "管理员已根据上游记录确认任务失败,冻结额度已释放" : null;
|
||||
// The observed outcome, shared wallet transaction and both audit receipts commit together.
|
||||
jdbc.update("UPDATE model_tasks SET completion_status=? WHERE id=?", target, taskId);
|
||||
settlements.finish(taskId, target, "succeeded".equals(target) ? 100 : 0, result, code, message);
|
||||
}
|
||||
Map<String,Object> after = view(task(taskId, false));
|
||||
jdbc.update("INSERT INTO task_reconciliations(request_id,task_id,actor_user_id,action,request_hash,evidence_reference,note,before_state,after_state) VALUES (?,?,?,?,?,?,?,CAST(? AS jsonb),CAST(? AS jsonb))",
|
||||
command.requestId(), taskId, actor.id(), command.action(), requestHash, command.evidenceReference().trim(), command.note().trim(), encode(before), encode(after));
|
||||
auth.audit("task.reconciled", actor.id(), meta, Map.of("taskId", taskId, "requestId", command.requestId(), "action", command.action(), "evidenceReference", command.evidenceReference().trim()));
|
||||
return Map.of("task", after, "requestId", command.requestId(), "replayed", false);
|
||||
});
|
||||
}
|
||||
|
||||
private BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
|
||||
try { return settlements.confirmedCharge(task, result); }
|
||||
catch (ApiException exception) {
|
||||
if ("TASK_USAGE_PENDING".equals(exception.getCode())) throw new ApiException("TASK_USAGE_REQUIRED", "成功核账必须提供完整可核验用量;任务与冻结额度保持不变", 422);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
private void resume(Map<String,Object> task, TaskReconciliationCommand command) {
|
||||
String originalId = (String) task.get("provider_task_id");
|
||||
if (originalId != null && !originalId.equals(command.providerTaskId())) throw conflict("不能将已受理任务改绑到另一个上游编号");
|
||||
String fingerprint = (String) task.get("provider_key_fingerprint");
|
||||
if (fingerprint != null && !fingerprint.equals(command.credentialFingerprint())) throw conflict("必须继续使用创建任务时的账号指纹,不能切换账号");
|
||||
if (task.get("provider_id_snapshot") == null || task.get("upstream_model_snapshot") == null) throw conflict("历史任务原执行路由无法证明,禁止猜测后恢复查询;请核实终态");
|
||||
provider.requirePollableBinding(String.valueOf(task.get("provider_id_snapshot")), String.valueOf(task.get("model_id")),
|
||||
String.valueOf(task.get("upstream_model_snapshot")), String.valueOf(task.get("task_type")), command.credentialFingerprint());
|
||||
String id = String.valueOf(task.get("id"));
|
||||
if (!guard.tryTaskUserSlot((java.util.UUID) task.get("api_key_id"), id, timeoutMs + 180000)) throw new ApiException("TASK_USER_BUSY", "用户当前并发已满,尚未恢复轮询,请稍后重试", 429);
|
||||
jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=now() WHERE id=?",
|
||||
command.providerTaskId(), command.credentialFingerprint(), OffsetDateTime.now(ZoneOffset.UTC).plus(java.time.Duration.ofMillis(timeoutMs)), id);
|
||||
usage.updateTask(id, "running", null, null, null);
|
||||
}
|
||||
|
||||
private void validate(TaskReconciliationCommand command) {
|
||||
if (command == null || command.requestId() == null || command.expectedUpdatedAt() == null) throw new ApiException("RECONCILIATION_PRECONDITION_REQUIRED", "必须提供操作编号和任务版本", 422);
|
||||
if (command.action() == null || !List.of("resume_polling", "confirm_failed", "confirm_succeeded").contains(command.action())) throw new ApiException("INVALID_RECONCILIATION_ACTION", "核账操作无效", 422);
|
||||
text(command.evidenceReference(), 3, 500); text(command.note(), 10, 2000);
|
||||
if ("resume_polling".equals(command.action())) {
|
||||
text(command.providerTaskId(), 1, 160);
|
||||
if (!command.providerTaskId().matches("[A-Za-z0-9][A-Za-z0-9._:-]{0,159}")) throw new ApiException("INVALID_PROVIDER_TASK_ID", "上游任务编号不能包含路径、查询串或空白", 422);
|
||||
if (command.credentialFingerprint() == null || !command.credentialFingerprint().matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "必须提供 SHA-256 账号指纹,不能填写 API Key", 422);
|
||||
if (command.result() != null || command.expectedCost() != null) throw conflict("恢复轮询不能同时修改结果或扣费金额");
|
||||
} else {
|
||||
if (command.providerTaskId() != null || command.credentialFingerprint() != null) throw conflict("终态核账不能改绑上游编号或账号");
|
||||
if ("confirm_succeeded".equals(command.action())) {
|
||||
if (command.expectedCost() == null || command.expectedCost().signum() < 0 || command.expectedCost().precision() > 20) throw new ApiException("EXPECTED_COST_REQUIRED", "请确认按原价格快照核算的非负扣费金额", 422);
|
||||
try { command.expectedCost().setScale(8, RoundingMode.UNNECESSARY); }
|
||||
catch (ArithmeticException exception) { throw new ApiException("INVALID_EXPECTED_COST", "确认金额最多 8 位小数", 422); }
|
||||
} else if (command.result() != null || command.expectedCost() != null) throw conflict("确认失败不能附带成功结果或扣费金额");
|
||||
}
|
||||
}
|
||||
|
||||
private void text(String value, int min, int max) {
|
||||
if (value == null || value.trim().length() < min || value.trim().length() > max || value.chars().anyMatch(Character::isISOControl)) throw new ApiException("RECONCILIATION_EVIDENCE_REQUIRED", "必须填写有效的上游证据引用与 10–2000 字核账说明;不得包含控制字符", 422);
|
||||
}
|
||||
|
||||
private String requestHash(TaskReconciliationCommand command, String taskId, Map<String,Object> result) {
|
||||
var body = new LinkedHashMap<String,Object>();
|
||||
body.put("taskId", taskId); body.put("action", command.action()); body.put("expectedUpdatedAt", command.expectedUpdatedAt().toInstant().toString());
|
||||
body.put("evidenceReference", command.evidenceReference().trim()); body.put("note", command.note().trim());
|
||||
body.put("providerTaskId", command.providerTaskId()); body.put("credentialFingerprint", command.credentialFingerprint());
|
||||
body.put("expectedCost", command.expectedCost()); body.put("result", result);
|
||||
return TaskCreationIdentity.of(json, command.requestId().toString(), "task-reconciliation", body).requestHash();
|
||||
}
|
||||
|
||||
private Map<String,Object> task(String taskId, boolean lock) {
|
||||
if (taskId == null || taskId.isBlank() || taskId.length() > 160) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
|
||||
var rows = jdbc.queryForList(TASK_COLUMNS + " WHERE t.id=?" + (lock ? " FOR UPDATE" : ""), taskId);
|
||||
if (rows.isEmpty()) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
|
||||
var task = rows.getFirst();
|
||||
task.put("pricing_snapshot", task.get("frozen_pricing"));
|
||||
return task;
|
||||
}
|
||||
|
||||
private Map<String,Object> view(Map<String,Object> task) {
|
||||
var view = new LinkedHashMap<String,Object>();
|
||||
view.put("id", task.get("id")); view.put("userId", task.get("user_id")); view.put("modelId", task.get("model_id"));
|
||||
view.put("taskType", task.get("task_type")); view.put("status", task.get("status"));
|
||||
view.put("updatedAt", timestamp(task.get("updated_at")).toInstant().toString());
|
||||
view.put("reservedAmount", ((BigDecimal) task.get("reserved_amount")).toPlainString());
|
||||
view.put("costAmount", task.get("cost_amount") == null ? null : ((BigDecimal) task.get("cost_amount")).toPlainString());
|
||||
view.put("providerId", task.get("provider_id_snapshot")); view.put("upstreamModel", task.get("upstream_model_snapshot"));
|
||||
view.put("providerTaskId", task.get("provider_task_id")); view.put("credentialFingerprint", task.get("provider_key_fingerprint"));
|
||||
view.put("completionStatus", task.get("completion_status")); view.put("requiresReconciliation", "reconciling".equals(task.get("status")));
|
||||
view.put("pricing", task.get("frozen_pricing") == null ? null : decode(task.get("frozen_pricing")));
|
||||
view.put("error", task.get("error") == null ? null : decode(task.get("error")));
|
||||
return view;
|
||||
}
|
||||
|
||||
private OffsetDateTime timestamp(Object value) {
|
||||
if (value instanceof OffsetDateTime time) return time;
|
||||
if (value instanceof java.sql.Timestamp time) return time.toInstant().atOffset(ZoneOffset.UTC);
|
||||
throw new IllegalStateException("Persisted task timestamp missing");
|
||||
}
|
||||
|
||||
private Map<String,Object> decode(Object value) {
|
||||
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
|
||||
catch (JsonProcessingException exception) { throw new IllegalStateException("Persisted reconciliation record cannot be read"); }
|
||||
}
|
||||
private String encode(Object value) {
|
||||
try { return json.writeValueAsString(value); }
|
||||
catch (JsonProcessingException exception) { throw new IllegalStateException("Reconciliation record cannot be encoded"); }
|
||||
}
|
||||
private ApiException conflict(String message) { return new ApiException("RECONCILIATION_CONFLICT", message, 409); }
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.gateway.EnterprisePolicies;
|
||||
import com.moyi.controlplane.gateway.GatewayGuard;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import java.math.BigDecimal;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Supplier;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
/** The one task/wallet/usage settlement transaction, shared by workers and audited reconciliation. */
|
||||
@Service
|
||||
public class TaskSettlementService {
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper json;
|
||||
private final GatewayGuard guard;
|
||||
private final UsageAuditService usage;
|
||||
private final EnterprisePolicies policies;
|
||||
private final TransactionTemplate transactions;
|
||||
|
||||
public TaskSettlementService(JdbcTemplate jdbc, ObjectMapper json, GatewayGuard guard,
|
||||
UsageAuditService usage, PlatformTransactionManager manager) {
|
||||
this.jdbc = jdbc;
|
||||
this.json = json;
|
||||
this.guard = guard;
|
||||
this.usage = usage;
|
||||
this.policies = new EnterprisePolicies(jdbc, json);
|
||||
this.transactions = manager == null ? null : new TransactionTemplate(manager);
|
||||
}
|
||||
|
||||
void finish(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
|
||||
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal task state required");
|
||||
transaction(() -> {
|
||||
var task = jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
if (!List.of("queued", "running", "reconciling").contains(task.get("status"))) return null;
|
||||
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
|
||||
AsyncPriceSchedule.Bill bill = null;
|
||||
String finalCode = code, finalMessage = message;
|
||||
if ("succeeded".equals(status) && task.get("pricing_snapshot") != null) {
|
||||
try { bill = AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result); }
|
||||
catch (ApiException exception) {
|
||||
finalCode = "TASK_USAGE_PENDING";
|
||||
finalMessage = "生成已完成,但上游计费用量未能核对;暂不扣费,等待核账";
|
||||
}
|
||||
}
|
||||
jdbc.update("UPDATE model_tasks SET status=?,progress=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),reconciliation_required_at=NULL,updated_at=now(),completed_at=now() WHERE id=?",
|
||||
status, progress, encode(result), finalCode == null ? null : encode(Map.of("code", finalCode, "message", finalMessage)), taskId);
|
||||
if ("TASK_USAGE_PENDING".equals(finalCode)) {
|
||||
release(taskId);
|
||||
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?", taskId);
|
||||
var observed = new LinkedHashMap<String,Object>();
|
||||
if (result != null) for (String field : List.of("usage", "usageMetadata")) if (result.get(field) instanceof Map<?,?> value) observed.put(field, value);
|
||||
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'", encode(observed), taskId);
|
||||
} else if ("succeeded".equals(status)) settle(taskId, bill);
|
||||
else release(taskId);
|
||||
usage.updateTask(taskId, status, result, finalCode, finalMessage);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/** Strict calculation for explicit administrator confirmation: missing usage is an error, not a refund. */
|
||||
BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
|
||||
if (task.get("pricing_snapshot") == null) return (BigDecimal) task.get("reserved_amount");
|
||||
return AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result).amount();
|
||||
}
|
||||
|
||||
private void settle(String taskId, AsyncPriceSchedule.Bill bill) {
|
||||
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
|
||||
BigDecimal amount = bill == null ? reserved : bill.amount();
|
||||
releasePolicies(taskId, task);
|
||||
if (amount == null || task.get("cost_amount") != null) return;
|
||||
UUID userId = (UUID) task.get("user_id");
|
||||
var wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
|
||||
BigDecimal before = (BigDecimal) wallet.get("balance"), after = before.subtract(amount);
|
||||
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
|
||||
if (amount.signum() > 0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)",
|
||||
UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
|
||||
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
|
||||
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
|
||||
if (bill != null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",
|
||||
bill.input(), bill.output(), bill.input(), bill.output(), bill.imageInput(), taskId);
|
||||
}
|
||||
|
||||
private void release(String taskId) {
|
||||
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
|
||||
releasePolicies(taskId, task);
|
||||
if (amount != null && task.get("cost_amount") == null) {
|
||||
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, task.get("user_id"));
|
||||
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
|
||||
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
|
||||
}
|
||||
}
|
||||
|
||||
private void releasePolicies(String taskId, Map<String,Object> task) {
|
||||
policies.release(taskId);
|
||||
guard.releaseTaskUserSlot((UUID) task.get("user_id"), taskId);
|
||||
}
|
||||
|
||||
private Map<String,Object> pricing(Object value) {
|
||||
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
|
||||
catch (JsonProcessingException exception) { throw new ApiException("INVALID_PRICE_SNAPSHOT", "任务价格快照无法核验", 409); }
|
||||
}
|
||||
|
||||
private String encode(Object value) {
|
||||
if (value == null) return null;
|
||||
try { return json.writeValueAsString(value); }
|
||||
catch (JsonProcessingException exception) { throw new IllegalStateException("Task settlement cannot be encoded"); }
|
||||
}
|
||||
|
||||
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
|
||||
private <T> T transaction(Supplier<T> work) { return transactions == null ? work.get() : transactions.execute(status -> work.get()); }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Freeze route identity without copying credentials or mutable provider configuration.
|
||||
ALTER TABLE model_tasks ADD COLUMN provider_id_snapshot varchar(64) REFERENCES model_providers(id);
|
||||
ALTER TABLE model_tasks ADD COLUMN upstream_model_snapshot varchar(160);
|
||||
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_execution_binding_check CHECK (
|
||||
(provider_id_snapshot IS NULL AND upstream_model_snapshot IS NULL)
|
||||
OR (provider_id_snapshot IS NOT NULL AND upstream_model_snapshot IS NOT NULL
|
||||
AND length(trim(provider_id_snapshot)) > 0 AND length(trim(upstream_model_snapshot)) > 0)
|
||||
);
|
||||
|
||||
-- Never guess the original route of an already-submitted legacy task.
|
||||
-- Unsubmitted queued work can safely freeze its route at migration time.
|
||||
UPDATE model_tasks t SET provider_id_snapshot=m.provider_id, upstream_model_snapshot=m.upstream_model
|
||||
FROM model_catalog m
|
||||
WHERE t.model_id=m.id AND t.status='queued' AND t.provider_task_id IS NULL;
|
||||
|
||||
COMMENT ON COLUMN model_tasks.provider_id_snapshot IS 'Provider identity at admission; NULL for ambiguous legacy accepted tasks';
|
||||
|
||||
-- New proven bindings cannot associate the same provider/account task with two local wallets.
|
||||
-- Legacy accepted rows are deliberately not guessed/backfilled, so this index does not rewrite their history.
|
||||
CREATE UNIQUE INDEX model_tasks_proven_upstream_identity_idx
|
||||
ON model_tasks(provider_id_snapshot,provider_key_fingerprint,provider_task_id)
|
||||
WHERE provider_id_snapshot IS NOT NULL AND provider_key_fingerprint IS NOT NULL
|
||||
AND provider_task_id IS NOT NULL AND provider_task_id <> 'mock';
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
-- Append-only operation receipts; no prompts, generated media or raw credentials.
|
||||
CREATE TABLE task_reconciliations (
|
||||
request_id uuid PRIMARY KEY,
|
||||
task_id varchar(160) NOT NULL REFERENCES model_tasks(id),
|
||||
actor_user_id uuid NOT NULL REFERENCES users(id),
|
||||
action varchar(24) NOT NULL CHECK (action IN ('resume_polling','confirm_failed','confirm_succeeded')),
|
||||
request_hash varchar(64) NOT NULL CHECK (request_hash ~ '^[0-9a-f]{64}$'),
|
||||
evidence_reference varchar(500) NOT NULL CHECK (length(trim(evidence_reference)) BETWEEN 3 AND 500),
|
||||
note varchar(2000) NOT NULL CHECK (length(trim(note)) BETWEEN 10 AND 2000),
|
||||
before_state jsonb NOT NULL,
|
||||
after_state jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX task_reconciliations_task_idx ON task_reconciliations(task_id,created_at DESC);
|
||||
CREATE FUNCTION protect_task_reconciliation_history() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'Task reconciliation history is append-only';
|
||||
END $$;
|
||||
CREATE TRIGGER task_reconciliation_history_immutable BEFORE UPDATE OR DELETE ON task_reconciliations
|
||||
FOR EACH ROW EXECUTE FUNCTION protect_task_reconciliation_history();
|
||||
@@ -0,0 +1,14 @@
|
||||
-- A resumed task renews its own lease instead of consuming a second slot.
|
||||
local time = redis.call('TIME')
|
||||
local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
|
||||
local existing = redis.call('ZSCORE', KEYS[1], ARGV[3])
|
||||
if not existing and redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[1]) then
|
||||
return 0
|
||||
end
|
||||
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[2]), ARGV[3])
|
||||
if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
||||
end
|
||||
if existing then return 2 end
|
||||
return 1
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.moyi.controlplane.gateway;
|
||||
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@EnabledIfEnvironmentVariable(named="MOYI_GATEWAY_TEST_REDIS_PORT", matches="[0-9]{2,5}")
|
||||
class TaskLeaseRedisTest {
|
||||
private LettuceConnectionFactory factory;
|
||||
private StringRedisTemplate redis;
|
||||
private GatewayGuard guard;
|
||||
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID();
|
||||
private final String prefix = "moyi:test:" + UUID.randomUUID() + ":";
|
||||
private String bucket() { return prefix + "gateway:slots:user:" + userId; }
|
||||
|
||||
@BeforeEach void setup() {
|
||||
factory = new LettuceConnectionFactory("127.0.0.1", Integer.parseInt(System.getenv("MOYI_GATEWAY_TEST_REDIS_PORT")));
|
||||
factory.afterPropertiesSet(); factory.start();
|
||||
redis = new StringRedisTemplate(factory); redis.afterPropertiesSet();
|
||||
guard = new GatewayGuard(redis, 60, 300, 4, prefix);
|
||||
var policies = mock(EnterprisePolicies.class);
|
||||
when(policies.taskLimit(keyId)).thenReturn(new EnterprisePolicies.UserLimit(userId, 120, 1));
|
||||
guard.setEnterprisePolicies(policies);
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() {
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) TransactionSynchronizationManager.clearSynchronization();
|
||||
if (redis != null) redis.delete(bucket());
|
||||
if (factory != null) factory.destroy();
|
||||
}
|
||||
|
||||
@Test void renewalDoesNotConsumeAnotherSlotAndDifferentTaskIsBlocked() {
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
|
||||
assertEquals(1L, redis.opsForZSet().zCard(bucket()));
|
||||
assertFalse(guard.tryTaskUserSlot(keyId, "different-task", 180000));
|
||||
assertTrue(redis.getExpire(bucket(), java.util.concurrent.TimeUnit.MILLISECONDS) > 250000);
|
||||
}
|
||||
|
||||
@Test void rolledBackRenewalKeepsThePreexistingLease() {
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
|
||||
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
assertNotNull(redis.opsForZSet().score(bucket(), "original-task"));
|
||||
}
|
||||
|
||||
@Test void rolledBackNewClaimRemovesOnlyItsOwnLease() {
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "new-task", 180000));
|
||||
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
assertNull(redis.opsForZSet().score(bucket(), "new-task"));
|
||||
}
|
||||
}
|
||||
+9
@@ -289,6 +289,15 @@ class AsyncProviderClientTest {
|
||||
} finally { server.stop(0); }
|
||||
}
|
||||
|
||||
@Test void pollingBindingValidationChecksTheExistingCredentialWithoutAnHttpCall() throws Exception {
|
||||
var client = clientFor("http://127.0.0.1:1");
|
||||
String fingerprint = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256").digest("test-key".getBytes(StandardCharsets.UTF_8)));
|
||||
org.junit.jupiter.api.Assertions.assertDoesNotThrow(() -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", fingerprint));
|
||||
assertEquals("UPSTREAM_KEY_NOT_AVAILABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "0".repeat(64))).getCode());
|
||||
assertEquals("TASK_NOT_POLLABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "image", fingerprint)).getCode());
|
||||
assertEquals("INVALID_CREDENTIAL_FINGERPRINT", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "test-key")).getCode());
|
||||
}
|
||||
|
||||
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
|
||||
var client = clientFor("http://127.0.0.1:1");
|
||||
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
|
||||
|
||||
+31
-1
@@ -65,9 +65,11 @@ class AsyncTaskPostgresTest {
|
||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
|
||||
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
|
||||
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
|
||||
// This fixture knows its original binding; a migration must not guess it for real legacy jobs.
|
||||
jdbc.update("UPDATE model_tasks SET provider_id_snapshot='provider-vast',upstream_model_snapshot='v3.1-20260211' WHERE id=?", taskId);
|
||||
when(catalog.keyOwnerId(keyId)).thenReturn(userId);
|
||||
when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true);
|
||||
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
|
||||
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "upstreamModel", "v3.1-20260211", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() {
|
||||
@@ -97,6 +99,34 @@ class AsyncTaskPostgresTest {
|
||||
return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal());
|
||||
}
|
||||
|
||||
@Test void queuedTaskRetainsAdmittedExecutionBindingWhenRouteIsEdited() {
|
||||
String created = String.valueOf(create("frozen-queue", Map.of("model", "model-mock-3d", "prompt", "chair")).get("id"));
|
||||
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=now() WHERE id=?", taskId);
|
||||
when(provider.submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap())).thenReturn(Map.of("id", "original-route-task"));
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("original-route-task", jdbc.queryForObject("SELECT provider_task_id FROM model_tasks WHERE id=?", String.class, created));
|
||||
verify(provider).submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap());
|
||||
}
|
||||
|
||||
@Test void acceptedTaskPollsFrozenProviderRatherThanEditedCatalog() {
|
||||
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
|
||||
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null)).thenReturn(Map.of("status", "running", "progress", 55));
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("running", status());
|
||||
assertEquals(55, jdbc.queryForObject("SELECT progress FROM model_tasks WHERE id=?", Integer.class, taskId));
|
||||
verify(provider).poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null);
|
||||
}
|
||||
|
||||
@Test void acceptedLegacyTaskWithoutProofOfBindingIsNotPolledOnAGuessedRoute() {
|
||||
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL,updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("reconciling", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
|
||||
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
|
||||
var first = create("client-operation-1", input);
|
||||
|
||||
+295
@@ -0,0 +1,295 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.moyi.controlplane.auth.AuthService;
|
||||
import com.moyi.controlplane.auth.PasswordHasher;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.common.ApiExceptionHandler;
|
||||
import com.moyi.controlplane.gateway.GatewayGuard;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import jakarta.servlet.http.Cookie;
|
||||
import java.math.BigDecimal;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
|
||||
|
||||
@EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+")
|
||||
class TaskReconciliationPostgresTest {
|
||||
private final ObjectMapper json = new ObjectMapper().findAndRegisterModules().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
|
||||
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID(), adminId = UUID.randomUUID(), secondAdminId = UUID.randomUUID();
|
||||
private final String taskId = "reconcile-" + UUID.randomUUID();
|
||||
private final String fingerprint = "c".repeat(64);
|
||||
private final AuthService.RequestMeta meta = new AuthService.RequestMeta("127.0.0.1", "isolated-reconciliation-test");
|
||||
private DriverManagerDataSource base;
|
||||
private JdbcTemplate jdbc;
|
||||
private String schema;
|
||||
private AuthService auth;
|
||||
private GatewayGuard guard;
|
||||
private AsyncProviderClient provider;
|
||||
private TaskReconciliationService service;
|
||||
private MockMvc http;
|
||||
|
||||
@BeforeEach void setup() {
|
||||
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
|
||||
schema = "reconciliation_test_" + UUID.randomUUID().toString().replace("-", "");
|
||||
base = new DriverManagerDataSource(url, "moyi_role_test", "");
|
||||
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
|
||||
var source = new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", "");
|
||||
jdbc = new JdbcTemplate(source);
|
||||
var manager = new DataSourceTransactionManager(source);
|
||||
for (var entry : Map.of(userId, "user", adminId, "admin", secondAdminId, "admin").entrySet()) {
|
||||
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name,role) VALUES (?,?,'username','not-loginable','Test',?)", entry.getKey(), entry.getKey().toString(), entry.getValue());
|
||||
}
|
||||
session("admin-session", adminId); session("second-admin-session", secondAdminId); session("user-session", userId);
|
||||
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-key',?)", keyId, userId, "b".repeat(64));
|
||||
jdbc.update("INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)", userId);
|
||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot,pricing_snapshot,reconciliation_required_at) VALUES (?,?,?,'model-mock-3d','3d','reconciling','private prompt',1.44,'original-provider-task',?,'provider-vast','v3.1-20260211','{\"unit\":\"CNY / task\",\"price\":1.44}'::jsonb,now())", taskId, userId, keyId, fingerprint);
|
||||
jdbc.update("INSERT INTO gateway_budget_holds(request_id,user_id,key_id,amount) VALUES (?,?,?,1.44)", taskId, userId, keyId);
|
||||
var usage = new UsageAuditService(jdbc, json);
|
||||
usage.recordTask(userId, keyId, "model-mock-3d", "provider-vast", taskId, "reconciling", Map.of("prompt", "private prompt"), null, "UNKNOWN", "pending");
|
||||
auth = spy(new AuthService(jdbc, mock(PasswordHasher.class), json, 168));
|
||||
guard = mock(GatewayGuard.class);
|
||||
when(guard.tryTaskUserSlot(any(), anyString(), anyLong())).thenReturn(true);
|
||||
provider = mock(AsyncProviderClient.class);
|
||||
var settlements = new TaskSettlementService(jdbc, json, guard, usage, manager);
|
||||
service = new TaskReconciliationService(jdbc, json, auth, settlements, provider, guard, usage, manager, 120_000);
|
||||
http = MockMvcBuilders.standaloneSetup(new TaskReconciliationController(service))
|
||||
.setControllerAdvice(new ApiExceptionHandler()).setMessageConverters(new MappingJackson2HttpMessageConverter(json)).build();
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() { if (base != null && schema != null) new JdbcTemplate(base).execute("DROP SCHEMA " + schema + " CASCADE"); }
|
||||
|
||||
private void session(String token, UUID user) {
|
||||
jdbc.update("INSERT INTO auth_sessions(id,token_hash,user_id,expires_at) VALUES (?,?,?,now()+interval '1 hour')", UUID.randomUUID(), AuthService.hashSecret(token), user);
|
||||
}
|
||||
private OffsetDateTime version() { return jdbc.queryForObject("SELECT updated_at FROM model_tasks WHERE id=?", OffsetDateTime.class, taskId); }
|
||||
private String state() { return jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, taskId); }
|
||||
private BigDecimal wallet(String column) { return jdbc.queryForObject("SELECT " + column + " FROM user_wallets WHERE user_id=?", BigDecimal.class, userId); }
|
||||
private int count(String table) { return jdbc.queryForObject("SELECT count(*) FROM " + table, Integer.class); }
|
||||
private TaskReconciliationCommand command(String action) {
|
||||
return new TaskReconciliationCommand(UUID.randomUUID(), action, version(), "provider-bill/test-001", "Verified against the provider task history and billing record",
|
||||
"resume_polling".equals(action) ? "original-provider-task" : null,
|
||||
"resume_polling".equals(action) ? fingerprint : null,
|
||||
"confirm_succeeded".equals(action) ? new BigDecimal("1.44") : null,
|
||||
"confirm_succeeded".equals(action) ? Map.of("url", "https://example.test/generated.glb") : null);
|
||||
}
|
||||
private Map<String,Object> reconcile(TaskReconciliationCommand command) { return service.reconcile("admin-session", taskId, command, meta); }
|
||||
private void unchanged() {
|
||||
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertEquals(0, count("wallet_entries")); assertEquals(0, count("task_reconciliations"));
|
||||
assertEquals(1, count("gateway_budget_holds"));
|
||||
}
|
||||
|
||||
@Test void realSessionsEnforceAdministratorAndCsrfBarrier() throws Exception {
|
||||
var command = command("confirm_failed");
|
||||
http.perform(get("/api/admin/task-reconciliation")).andExpect(status().isUnauthorized());
|
||||
http.perform(get("/api/admin/task-reconciliation").cookie(new Cookie("moyi_session", "user-session"))).andExpect(status().isForbidden());
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "user-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
|
||||
unchanged();
|
||||
verifyNoInteractions(provider, guard);
|
||||
}
|
||||
|
||||
@Test void failureConfirmationReleasesOnceAndReplayReturnsTheOperationReceipt() throws Exception {
|
||||
var command = command("confirm_failed");
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command)))
|
||||
.andExpect(status().isOk()).andExpect(jsonPath("$.task.status").value("failed")).andExpect(jsonPath("$.replayed").value(false));
|
||||
assertEquals(true, reconcile(command).get("replayed"));
|
||||
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals(0, count("wallet_entries")); assertEquals(0, count("gateway_budget_holds"));
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM auth_audit_log WHERE event_type='task.reconciled'", Integer.class));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void successUsesFrozenPriceAndDebitsExactlyOnce() {
|
||||
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":99}'::jsonb WHERE id='model-mock-3d'");
|
||||
var command = command("confirm_succeeded");
|
||||
reconcile(command); reconcile(command);
|
||||
assertEquals("succeeded", state());
|
||||
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||
assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
|
||||
assertEquals(new BigDecimal("1.44000000"), jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void quoteIsReadOnlyAndUsesTheFrozenVersionNotEditableCatalogPrices() throws Exception {
|
||||
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":999}'::jsonb WHERE id='model-mock-3d'");
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId + "/quote").cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content("{}"))
|
||||
.andExpect(status().isOk()).andExpect(jsonPath("$.expectedCost").value("1.44"));
|
||||
unchanged();
|
||||
assertEquals(0, count("auth_audit_log"));
|
||||
}
|
||||
|
||||
@Test void concurrentIdenticalOperationsShareOneReceiptAndOneDebit() throws Exception {
|
||||
var command = command("confirm_succeeded");
|
||||
var start = new CountDownLatch(1);
|
||||
try (var executor = Executors.newFixedThreadPool(8)) {
|
||||
var results = new ArrayList<java.util.concurrent.Future<Map<String,Object>>>();
|
||||
for (int i = 0; i < 8; i++) results.add(executor.submit(() -> { assertTrue(start.await(10, TimeUnit.SECONDS)); return reconcile(command); }));
|
||||
start.countDown();
|
||||
int originals = 0;
|
||||
for (var result : results) if (Boolean.FALSE.equals(result.get(30, TimeUnit.SECONDS).get("replayed"))) originals++;
|
||||
assertEquals(1, originals);
|
||||
}
|
||||
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
|
||||
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||
}
|
||||
|
||||
@Test void concurrentDifferentDecisionsHaveExactlyOneWinner() throws Exception {
|
||||
var success = command("confirm_succeeded"); var failure = command("confirm_failed");
|
||||
var start = new CountDownLatch(1);
|
||||
try (var executor = Executors.newFixedThreadPool(2)) {
|
||||
var results = new ArrayList<java.util.concurrent.Future<Integer>>();
|
||||
for (var command : List.of(success, failure)) results.add(executor.submit(() -> {
|
||||
assertTrue(start.await(10, TimeUnit.SECONDS));
|
||||
try { reconcile(command); return 200; } catch (ApiException exception) { return exception.getStatus(); }
|
||||
}));
|
||||
start.countDown();
|
||||
assertEquals(List.of(200, 409), results.stream().map(f -> { try { return f.get(30, TimeUnit.SECONDS); } catch (Exception exception) { throw new AssertionError(exception); } }).sorted().toList());
|
||||
}
|
||||
assertEquals(1, count("task_reconciliations")); assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals("succeeded".equals(state()) ? new BigDecimal("8.56000000") : new BigDecimal("10.00000000"), wallet("balance"));
|
||||
}
|
||||
|
||||
@Test void operationIdCannotBeReusedForDifferentActionOrAnotherAdministrator() {
|
||||
var command = command("confirm_failed"); reconcile(command);
|
||||
var changed = new TaskReconciliationCommand(command.requestId(), "confirm_succeeded", command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.44"), Map.of());
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(changed)).getStatus());
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> service.reconcile("second-admin-session", taskId, command, meta)).getStatus());
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
}
|
||||
|
||||
@Test void staleVersionAndWrongExpectedAmountLeaveFinancialStateUnchanged() {
|
||||
var command = command("confirm_succeeded");
|
||||
var wrong = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.45"), command.result());
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(wrong)).getStatus()); unchanged();
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=updated_at+interval '1 second' WHERE id=?", taskId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command)).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void missingEvidenceCannotAuthorizeARefund() {
|
||||
var command = command("confirm_failed");
|
||||
var invalid = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), "", "short", null, null, null, null);
|
||||
assertEquals(422, assertThrows(ApiException.class, () -> reconcile(invalid)).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void missingTokenUsageCannotReleaseTheReservationUnderASuccessLabel() {
|
||||
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
|
||||
assertEquals("TASK_USAGE_REQUIRED", assertThrows(ApiException.class, () -> reconcile(command("confirm_succeeded"))).getCode()); unchanged();
|
||||
}
|
||||
|
||||
@Test void auditFailureRollsBackTaskWalletUsageAndReceiptTogether() {
|
||||
var command = command("confirm_succeeded");
|
||||
doThrow(new IllegalStateException("audit database unavailable")).when(auth).audit(eq("task.reconciled"), eq(adminId), any(), anyMap());
|
||||
assertThrows(IllegalStateException.class, () -> reconcile(command)); unchanged();
|
||||
assertNull(jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId));
|
||||
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||
}
|
||||
|
||||
@Test void resumeOnlyQueriesTheOriginalBindingWithoutChangingTheWallet() {
|
||||
var command = command("resume_polling"); reconcile(command); reconcile(command);
|
||||
assertEquals("running", state());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertEquals(1, count("gateway_budget_holds")); assertEquals(0, count("wallet_entries"));
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
verify(provider, times(1)).requirePollableBinding("provider-vast", "model-mock-3d", "v3.1-20260211", "3d", fingerprint);
|
||||
verifyNoMoreInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void differentProviderTaskCannotBeAttachedToAnAcceptedJob() {
|
||||
var command = command("resume_polling");
|
||||
var other = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), "some-other-task", fingerprint, null, null);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(other)).getStatus()); unchanged(); verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void sameProvenUpstreamTaskCannotBeAssociatedWithTwoLocalTasks() {
|
||||
assertThrows(DataAccessException.class, () -> jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot) VALUES ('duplicate-local-task',?,?,'model-mock-3d','3d','running','',0,'original-provider-task',?,'provider-vast','v3.1-20260211')", userId, keyId, fingerprint));
|
||||
unchanged();
|
||||
}
|
||||
|
||||
@Test void validTokenUsageIsQuotedAndSettledOnceUsingTheFrozenSchedule() {
|
||||
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
|
||||
var result = Map.<String,Object>of("usage", Map.of("input_tokens", 100, "output_tokens", 100));
|
||||
var quote = service.quote("admin-session", taskId, result);
|
||||
assertEquals("0.00110000", quote.get("expectedCost"));
|
||||
var command = new TaskReconciliationCommand(UUID.randomUUID(), "confirm_succeeded", version(), "provider-token-bill", "Verified exact input and output token usage from provider record", null, null, new BigDecimal("0.0011"), result);
|
||||
reconcile(command); reconcile(command);
|
||||
assertEquals(new BigDecimal("9.99890000"), wallet("balance"));
|
||||
assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals(100L, jdbc.queryForObject("SELECT input_units FROM usage_events WHERE provider_task_id=?", Long.class, taskId));
|
||||
assertEquals(1, count("wallet_entries"));
|
||||
}
|
||||
|
||||
@Test void unavailableCredentialAndAmbiguousLegacyRouteCannotResumePolling() {
|
||||
doThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "safe message", 503)).when(provider).requirePollableBinding(anyString(), anyString(), anyString(), anyString(), anyString());
|
||||
assertEquals(503, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
|
||||
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL WHERE id=?", taskId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void existingObservedOutcomeCannotBeOverruledManually() {
|
||||
jdbc.update("UPDATE model_tasks SET completion_status='succeeded' WHERE id=?", taskId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void insufficientHoldIsAConflictNotABalancingPlug() {
|
||||
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus());
|
||||
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance")); assertEquals(0, count("task_reconciliations"));
|
||||
}
|
||||
|
||||
@Test void receiptsAreAppendOnlyAndNeverContainPromptOrGeneratedContent() {
|
||||
reconcile(command("confirm_succeeded"));
|
||||
String stored = jdbc.queryForObject("SELECT row_to_json(r)::text FROM task_reconciliations r", String.class);
|
||||
assertFalse(stored.contains("private prompt")); assertFalse(stored.contains("generated.glb"));
|
||||
assertThrows(DataAccessException.class, () -> jdbc.update("DELETE FROM task_reconciliations"));
|
||||
assertThrows(DataAccessException.class, () -> jdbc.update("UPDATE task_reconciliations SET note='rewritten history'"));
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
}
|
||||
|
||||
@Test void listAndHistoryAreBoundedAdministratorViewsWithoutRawPayloads() {
|
||||
var list = service.list("admin-session", 1);
|
||||
assertEquals(1L, list.get("total"));
|
||||
String encoded = assertDoesNotThrow(() -> json.writeValueAsString(list));
|
||||
assertFalse(encoded.contains("private prompt")); assertFalse(encoded.contains("input_payload"));
|
||||
reconcile(command("confirm_failed"));
|
||||
assertEquals(1, ((List<?>) service.detail("admin-session", taskId).get("history")).size());
|
||||
assertEquals(0L, service.list("admin-session", 1).get("total"));
|
||||
}
|
||||
|
||||
@Test void freeFailureHasAnExplicitZeroCostAndNoLedgerEntry() {
|
||||
jdbc.update("UPDATE model_tasks SET reserved_amount=0 WHERE id=?", taskId);
|
||||
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
|
||||
reconcile(command("confirm_failed"));
|
||||
assertEquals(0, jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId).signum());
|
||||
assertEquals(0, count("wallet_entries"));
|
||||
}
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class TaskReconciliationResultTest {
|
||||
private final ObjectMapper json = new ObjectMapper();
|
||||
|
||||
@Test void acceptsOnlyNormalizedAssetsAndIntegralUsage() {
|
||||
var result = TaskReconciliationResult.normalize(Map.of("url", "https://example.test/image.png", "usage", Map.of("input_tokens", "10", "output_tokens", 20,
|
||||
"input_tokens_details", Map.of("text_tokens", 2, "image_tokens", 8))), json);
|
||||
assertEquals(10L, ((Map<?,?>) result.get("usage")).get("input_tokens"));
|
||||
}
|
||||
|
||||
@Test void rejectsCredentialsRawPayloadsAndUnsafeAssetUrls() {
|
||||
for (Map<String,Object> value : List.<Map<String,Object>>of(
|
||||
Map.of("Authorization", "secret"), Map.of("base64", "large media"),
|
||||
Map.of("url", "javascript:alert(1)"), Map.of("url", "https://user:password@example.test/a"),
|
||||
Map.of("usage", Map.of("api_key", "secret")), Map.of("usage", Map.of("input_tokens", -1)),
|
||||
Map.of("usage", Map.of("output_tokens", 1.5)), Map.of("usage", Map.of("input_tokens", 100_000_001)))) {
|
||||
assertEquals(422, assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(value, json)).getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Test void rejectsOversizeObjectsAndCommandLoggingOmitsEvidenceAndPayload() {
|
||||
assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(Map.of("url", "https://example.test/" + "x".repeat(40_000)), json));
|
||||
var command = new TaskReconciliationCommand(java.util.UUID.randomUUID(), "confirm_succeeded", null, "private bill", "private note", null, null, null, Map.of("url", "private result"));
|
||||
assertFalse(command.toString().contains("private"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user