feat: add audited async task reconciliation and frozen bindings

This commit is contained in:
xpp committed 2026-10-21 20:08:38 +08:00
1 parent 5be06a97c1
commit c716ec7478
28 files changed
+1312 -76

No files matched your search

@@ -23,6 +23,7 @@ public class GatewayGuard {
private static final long SLOT_TTL_MS = 180_000;
private static final DefaultRedisScript<Long> RATE = script("redis/rate-limit.lua");
private static final DefaultRedisScript<Long> ACQUIRE = script("redis/acquire-slot.lua");
private static final DefaultRedisScript<Long> ACQUIRE_TASK = script("redis/acquire-task-slot.lua");
private static final DefaultRedisScript<Long> ACQUIRE_PAIR = script("redis/acquire-pair.lua");
private static final DefaultRedisScript<List> CAPACITY = new DefaultRedisScript<>("""
local time = redis.call('TIME')
@@ -163,9 +164,11 @@ public class GatewayGuard {
if(policies==null) return true;
var policy=policies.taskLimit(keyId);
String bucket=redisPrefix+"gateway:slots:user:"+policy.userId();
boolean acquired=execute(ACQUIRE,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId)==1;
long result=execute(ACQUIRE_TASK,bucket,Integer.toString(policy.concurrency()),Long.toString(Math.max(180000,ttlMs)),taskId);
boolean acquired=result>0;
// Redis is outside the database transaction: roll back the lease if claiming the task fails.
if(acquired&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
// Rolling back renewal must not remove a lease that existed before this transaction.
if(result==1&&org.springframework.transaction.support.TransactionSynchronizationManager.isSynchronizationActive())
org.springframework.transaction.support.TransactionSynchronizationManager.registerSynchronization(new org.springframework.transaction.support.TransactionSynchronization() {@Override public void afterCompletion(int status) {if(status!=STATUS_COMMITTED) releaseQuietly(bucket,taskId);}});
return acquired;
}
@@ -110,6 +110,17 @@ public class AsyncProviderClient {
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
}
/** Validate a proven polling binding without sending any HTTP request or exposing credential values. */
public void requirePollableBinding(String providerId, String routeId, String upstreamModel, String taskType, String fingerprint) {
if (fingerprint == null || !fingerprint.matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "需要有效的上游账号指纹", 422);
Provider provider = provider(providerId, routeId, upstreamModel);
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
boolean pollable = isTripo(provider.code(), provider.baseUrl()) ? "3d".equals(taskType)
: (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com") || "openai-compatible".equalsIgnoreCase(provider.protocol())) && "video".equals(taskType);
if (!pollable) throw new ApiException("TASK_NOT_POLLABLE", "该执行器没有可恢复的只读任务查询接口,请核实终态", 409);
provider.keyForFingerprint(routeId, upstreamModel, environment, keyCursors, fingerprint);
}
public Map<String,Object> poll(String providerId, String providerTaskId) {
return poll(providerId, null, providerTaskId);
}
@@ -19,8 +19,6 @@ import java.sql.SQLException;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -52,6 +50,7 @@ public class AsyncTaskService {
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
private final ApiRequestRiskPolicy requestRisk;
private final TaskOutcomeJournal outcomes;
private final TaskSettlementService settlements;
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
@@ -108,6 +107,7 @@ public class AsyncTaskService {
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
this.settlements = new TaskSettlementService(jdbc, objectMapper, gatewayGuard, usage, transactionManager);
}
@Transactional
@@ -177,6 +177,8 @@ public class AsyncTaskService {
}
requestRisk.enforce(new ApiRequestContext(userId, apiKeyId, ApiRequestProtocol.ASYNC_TASK,
requestedModelId, modelId, false, estimateInputUnits(input), 0, metadata, System.nanoTime()));
String upstreamModel = model.get("upstreamModel") instanceof String value ? value : null;
if (upstreamModel == null || upstreamModel.isBlank()) throw new ApiException("ASYNC_BINDING_MISSING", "模型缺少可冻结的上游执行标识", 503);
// Serialize the queue-capacity check with task insertion for this
// model. Without the transaction advisory lock, concurrent creates
// could all observe the same count and exceed maxQueuedPerModel.
@@ -198,8 +200,8 @@ public class AsyncTaskService {
OffsetDateTime now = now();
try {
jdbc.update(
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash()
"INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, input, reserved_amount, created_at, updated_at, idempotency_hash, request_hash, provider_id_snapshot, upstream_model_snapshot) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, CAST(? AS jsonb), ?, ?, ?, ?, ?, ?, ?)",
id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, writeJson(input), reservedAmount, now, now, identity.keyHash(), identity.requestHash(), model.get("providerId"), upstreamModel
);
} catch (RuntimeException exception) {
releaseWallet(userId, reservedAmount);
@@ -253,7 +255,7 @@ public class AsyncTaskService {
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
);
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model FROM model_tasks t WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
for (Map<String,Object> row : pending) pollProviderTask(row);
dispatchQueuedTasks();
}
@@ -314,10 +316,10 @@ public class AsyncTaskService {
// workers reliably.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
String sql = """
SELECT t.id, t.user_id, t.api_key_id, t.model_id, m.provider_id, m.upstream_model, t.task_type, t.prompt, t.input
SELECT t.id, t.user_id, t.api_key_id, t.model_id, t.provider_id_snapshot AS provider_id, t.upstream_model_snapshot AS upstream_model, t.task_type, t.prompt, t.input
FROM model_tasks t
JOIN model_catalog m ON m.id=t.model_id
JOIN model_providers p ON p.id=m.provider_id
JOIN model_providers p ON p.id=t.provider_id_snapshot
WHERE t.status='queued'
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
@@ -373,35 +375,7 @@ public class AsyncTaskService {
}
private void finishTask(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
// The terminal state and wallet change must commit together. Otherwise a
// crash after marking failed/succeeded leaves a hold that polling never
// revisits. The conditional update also prevents duplicate settlement.
inTransaction(() -> {
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
// A late failure must not override a durably observed success whose settlement is pending.
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
AsyncPriceSchedule.Bill bill=null;
String finalCode=code,finalMessage=message;
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
try { bill=AsyncPriceSchedule.bill(readMap(String.valueOf(task.get("pricing_snapshot"))),result); }
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
}
int changed = jdbc.update(
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
status, progress, result == null ? null : writeJson(result),
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
if (changed == 0) return null;
if ("TASK_USAGE_PENDING".equals(finalCode)) {
releaseTaskInTransaction(taskId);
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?",taskId);
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'",writeJson(usageOnly(result)),taskId);
}
else if ("succeeded".equals(status)) settleTaskInTransaction(taskId,bill);
else releaseTaskInTransaction(taskId);
usage.updateTask(taskId, status, result, finalCode, finalMessage);
return null;
});
settlements.finish(taskId, status, progress, result, code, message);
}
private Map<String, Object> requireKey(String authorization) {
@@ -476,6 +450,10 @@ public class AsyncTaskService {
private void pollProviderTask(Map<String,Object> row) {
String taskId = String.valueOf(row.get("id"));
if (row.get("provider_id") == null || row.get("upstream_model") == null) {
outcomes.requireReconciliation(taskId, "ASYNC_BINDING_MISSING", "历史任务缺少可证明的原始执行路由;冻结额度保留,请根据上游记录核账", false);
return;
}
OffsetDateTime deadline = taskTimestamp(row.get("provider_deadline_at"));
if (deadline == null) {
// Defensive fallback for pre-migration tasks. The migration also
@@ -622,42 +600,10 @@ public class AsyncTaskService {
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, userId);
}
private void settleTaskInTransaction(String taskId,AsyncPriceSchedule.Bill bill) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
BigDecimal amount = bill==null ? reserved : bill.amount();
policies.release(taskId);
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
if (amount == null || task.get("cost_amount") != null) return;
UUID userId = (UUID) task.get("user_id");
Map<String,Object> wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
BigDecimal before = (BigDecimal) wallet.get("balance");
BigDecimal after = before.subtract(amount);
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
if(amount.signum()>0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)", UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
if(bill!=null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",bill.input(),bill.output(),bill.input(),bill.output(),bill.imageInput(),taskId);
}
private void releaseTaskInTransaction(String taskId) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
policies.release(taskId);
gatewayGuard.releaseTaskUserSlot((UUID)task.get("user_id"),taskId);
if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) {
releaseWallet((UUID) task.get("user_id"), amount);
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
}
}
private <T> T inTransaction(Supplier<T> work) {
return transactions == null ? work.get() : transactions.execute(status -> work.get());
}
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
private void putIfPresent(Map<String,Object> target, String name, ResultSet resultSet, String column) throws SQLException {
Object value = resultSet.getObject(column);
if (value != null) target.put(name, value);
@@ -0,0 +1,21 @@
package com.moyi.controlplane.task;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.UUID;
/** Administrative evidence and optimistic-concurrency preconditions, never a raw provider key. */
public record TaskReconciliationCommand(
UUID requestId,
String action,
OffsetDateTime expectedUpdatedAt,
String evidenceReference,
String note,
String providerTaskId,
String credentialFingerprint,
BigDecimal expectedCost,
Map<String,Object> result
) {
@Override public String toString() { return "TaskReconciliationCommand[requestId=" + requestId + ", action=" + action + "]"; }
}
@@ -0,0 +1,42 @@
package com.moyi.controlplane.task;
import com.moyi.controlplane.auth.AuthController;
import com.moyi.controlplane.common.ApiException;
import jakarta.servlet.http.HttpServletRequest;
import java.util.Map;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/admin/task-reconciliation")
public class TaskReconciliationController {
private final TaskReconciliationService reconciliations;
public TaskReconciliationController(TaskReconciliationService reconciliations) { this.reconciliations = reconciliations; }
@GetMapping
public Map<String,Object> list(@RequestParam(defaultValue="1") int page, HttpServletRequest request) {
return reconciliations.list(AuthController.sessionToken(request), page);
}
@GetMapping("/{taskId}")
public Map<String,Object> detail(@PathVariable String taskId, HttpServletRequest request) {
return reconciliations.detail(AuthController.sessionToken(request), taskId);
}
@PostMapping(value="/{taskId}", consumes="application/json")
public Map<String,Object> reconcile(@PathVariable String taskId, @RequestBody TaskReconciliationCommand command, HttpServletRequest request) {
// A custom header + JSON requires CORS preflight; the header is a CSRF barrier, not authentication.
if (!"console".equals(request.getHeader("X-Moyi-Source"))) throw new ApiException("CONSOLE_REQUEST_REQUIRED", "请通过管理控制台发起核账", 403);
return reconciliations.reconcile(AuthController.sessionToken(request), taskId, command, AuthController.meta(request));
}
@PostMapping(value="/{taskId}/quote", consumes="application/json")
public Map<String,Object> quote(@PathVariable String taskId, @RequestBody Map<String,Object> result, HttpServletRequest request) {
return reconciliations.quote(AuthController.sessionToken(request), taskId, result);
}
}
@@ -0,0 +1,64 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import java.math.BigDecimal;
import java.net.URI;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
/** Only normalized assets and numeric usage may enter a manual completion, never raw provider payloads. */
final class TaskReconciliationResult {
private static final Set<String> ASSETS = Set.of("url", "modelUrl", "videoUrl", "imageUrl", "audioUrl", "previewUrl");
private static final Set<String> OPENAI = Set.of("input_tokens", "output_tokens", "total_tokens", "cached_tokens", "input_tokens_details");
private static final Set<String> DETAILS = Set.of("text_tokens", "image_tokens", "cached_tokens", "cached_tokens_details");
private static final Set<String> CACHED = Set.of("text_tokens", "image_tokens");
private static final Set<String> GEMINI = Set.of("promptTokenCount", "candidatesTokenCount", "thoughtsTokenCount", "cachedContentTokenCount", "totalTokenCount");
static Map<String,Object> normalize(Map<String,Object> input, ObjectMapper json) {
if (input == null) throw invalid();
try { if (json.writeValueAsBytes(input).length > 32_768) throw invalid(); }
catch (JsonProcessingException exception) { throw invalid(); }
var result = new LinkedHashMap<String,Object>();
for (var entry : input.entrySet()) {
String field = entry.getKey();
Object value = entry.getValue();
if (ASSETS.contains(field)) {
if (!(value instanceof String text) || text.length() > 4096 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
try {
URI uri = URI.create(text);
if (!("https".equalsIgnoreCase(uri.getScheme()) || "http".equalsIgnoreCase(uri.getScheme())) || uri.getHost() == null || uri.getUserInfo() != null) throw invalid();
} catch (IllegalArgumentException exception) { throw invalid(); }
result.put(field, text);
} else if (Set.of("format", "mimeType").contains(field)) {
if (!(value instanceof String text) || text.isBlank() || text.length() > 128 || text.chars().anyMatch(Character::isISOControl)) throw invalid();
result.put(field, text);
} else if ("usage".equals(field)) result.put(field, numericMap(value, OPENAI));
else if ("usageMetadata".equals(field)) result.put(field, numericMap(value, GEMINI));
else throw invalid();
}
return result;
}
private static Map<String,Object> numericMap(Object value, Set<String> allowed) {
if (!(value instanceof Map<?,?> map)) throw invalid();
var result = new LinkedHashMap<String,Object>();
for (var entry : map.entrySet()) {
if (!(entry.getKey() instanceof String name) || !allowed.contains(name)) throw invalid();
if ("input_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), DETAILS));
else if ("cached_tokens_details".equals(name)) result.put(name, numericMap(entry.getValue(), CACHED));
else {
try {
long count = new BigDecimal(String.valueOf(entry.getValue())).longValueExact();
if (count < 0 || count > 100_000_000) throw invalid();
result.put(name, count);
} catch (NumberFormatException | ArithmeticException exception) { throw invalid(); }
}
}
return result;
}
private static ApiException invalid() { return new ApiException("INVALID_RECONCILIATION_RESULT", "只接受有界的 HTTP(S) 结果地址和规范化数值用量,不接受原始上游响应或密钥", 422); }
}
@@ -0,0 +1,216 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.auth.AuthService;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
/** Evidence-backed administrative decisions; no generation, arbitrary balance adjustment or transparent replay. */
@Service
public class TaskReconciliationService {
private final JdbcTemplate jdbc;
private final ObjectMapper json;
private final AuthService auth;
private final TaskSettlementService settlements;
private final AsyncProviderClient provider;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final TransactionTemplate transactions;
private final long timeoutMs;
// List/detail must not load prompts, original inputs or potentially large generated media.
private static final String TASK_COLUMNS = "SELECT t.id,t.user_id,t.api_key_id,t.model_id,t.task_type,t.status,t.updated_at,t.reserved_amount,t.cost_amount,t.provider_id_snapshot,t.upstream_model_snapshot,t.provider_task_id,t.provider_key_fingerprint,t.completion_status,t.pricing_snapshot::text AS frozen_pricing,t.error::text AS error FROM model_tasks t";
public TaskReconciliationService(JdbcTemplate jdbc, ObjectMapper json, AuthService auth,
TaskSettlementService settlements, AsyncProviderClient provider, GatewayGuard guard,
UsageAuditService usage, PlatformTransactionManager manager,
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long timeoutMs) {
this.jdbc = jdbc; this.json = json; this.auth = auth; this.settlements = settlements;
this.provider = provider; this.guard = guard; this.usage = usage;
this.transactions = new TransactionTemplate(manager);
if (timeoutMs < 1000) throw new IllegalArgumentException("Polling deadline must be positive");
this.timeoutMs = timeoutMs;
}
public Map<String,Object> list(String token, int requestedPage) {
auth.requireAdmin(token);
int page = Math.max(1, Math.min(100000, requestedPage));
return Map.of("records", jdbc.queryForList(TASK_COLUMNS + " WHERE t.status='reconciling' ORDER BY t.reconciliation_required_at NULLS FIRST,t.id LIMIT 50 OFFSET ?", (page - 1) * 50).stream().map(this::view).toList(),
"total", jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE status='reconciling'", Long.class), "page", page);
}
public Map<String,Object> detail(String token, String taskId) {
auth.requireAdmin(token);
Map<String,Object> task = task(taskId, false);
var history = jdbc.queryForList("SELECT request_id,actor_user_id,action,evidence_reference,note,before_state::text AS before_state,after_state::text AS after_state,created_at FROM task_reconciliations WHERE task_id=? ORDER BY created_at DESC LIMIT 50", taskId);
var records = history.stream().map(row -> {
var item = new LinkedHashMap<String,Object>();
for (String field : List.of("request_id", "actor_user_id", "action", "evidence_reference", "note", "created_at")) item.put(field, row.get(field));
item.put("before_state", decode(row.get("before_state")));
item.put("after_state", decode(row.get("after_state")));
return item;
}).toList();
return Map.of("task", view(task), "history", records);
}
public Map<String,Object> quote(String token, String taskId, Map<String,Object> input) {
auth.requireAdmin(token);
var task = task(taskId, false);
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可核账状态,请刷新");
var result = TaskReconciliationResult.normalize(input, json);
return Map.of("expectedCost", confirmedCharge(task, result).toPlainString(),
"expectedUpdatedAt", timestamp(task.get("updated_at")).toInstant().toString());
}
public Map<String,Object> reconcile(String token, String taskId, TaskReconciliationCommand command, AuthService.RequestMeta meta) {
var actor = auth.requireAdmin(token); // Authorization is required even for replay.
validate(command);
Map<String,Object> result = "confirm_succeeded".equals(command.action()) ? TaskReconciliationResult.normalize(command.result(), json) : null;
String requestHash = requestHash(command, taskId, result);
return transactions.execute(status -> {
// Keep role/status revocation ordered with this financial decision, not a stale pre-check.
if (jdbc.queryForList("SELECT id FROM users WHERE id=? AND status='active' AND role IN ('admin','super_admin') FOR SHARE", actor.id()).isEmpty()) throw new ApiException("FORBIDDEN", "管理员权限已变化,请重新登录", 403);
auth.requireAdmin(token); // Recheck session revocation/expiry after entering the transaction.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))", "moyi:task-reconciliation:" + command.requestId());
var prior = jdbc.queryForList("SELECT actor_user_id,request_hash,after_state::text AS after_state FROM task_reconciliations WHERE request_id=?", command.requestId());
if (!prior.isEmpty()) {
var receipt = prior.getFirst();
if (!actor.id().equals(receipt.get("actor_user_id")) || !requestHash.equals(receipt.get("request_hash"))) throw conflict("此核账请求编号已用于不同的操作");
return Map.of("task", decode(receipt.get("after_state")), "requestId", command.requestId(), "replayed", true);
}
var task = task(taskId, true);
if (!"reconciling".equals(task.get("status")) || task.get("completion_status") != null || task.get("cost_amount") != null) throw conflict("任务不处于可人工核账状态,或已有可信终态;请刷新");
if (!timestamp(task.get("updated_at")).toInstant().equals(command.expectedUpdatedAt().toInstant())) throw conflict("任务已被其他操作更新,请刷新后重新核对");
var wallets = jdbc.queryForList("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", task.get("user_id"));
if (wallets.isEmpty() || ((BigDecimal) wallets.getFirst().get("reserved_balance")).compareTo((BigDecimal) task.get("reserved_amount")) < 0) throw conflict("任务冻结额度与钱包不一致,请先排查账本,不允许直接覆盖");
Map<String,Object> before = view(task);
if ("resume_polling".equals(command.action())) resume(task, command);
else {
if ("confirm_succeeded".equals(command.action())) {
BigDecimal charge = confirmedCharge(task, result);
if (charge == null || charge.compareTo(command.expectedCost()) != 0) throw conflict("确认金额与任务原价格快照及实际用量不一致");
}
String target = "confirm_succeeded".equals(command.action()) ? "succeeded" : "failed";
String code = "failed".equals(target) ? "TASK_RECONCILED_FAILED" : null;
String message = "failed".equals(target) ? "管理员已根据上游记录确认任务失败,冻结额度已释放" : null;
// The observed outcome, shared wallet transaction and both audit receipts commit together.
jdbc.update("UPDATE model_tasks SET completion_status=? WHERE id=?", target, taskId);
settlements.finish(taskId, target, "succeeded".equals(target) ? 100 : 0, result, code, message);
}
Map<String,Object> after = view(task(taskId, false));
jdbc.update("INSERT INTO task_reconciliations(request_id,task_id,actor_user_id,action,request_hash,evidence_reference,note,before_state,after_state) VALUES (?,?,?,?,?,?,?,CAST(? AS jsonb),CAST(? AS jsonb))",
command.requestId(), taskId, actor.id(), command.action(), requestHash, command.evidenceReference().trim(), command.note().trim(), encode(before), encode(after));
auth.audit("task.reconciled", actor.id(), meta, Map.of("taskId", taskId, "requestId", command.requestId(), "action", command.action(), "evidenceReference", command.evidenceReference().trim()));
return Map.of("task", after, "requestId", command.requestId(), "replayed", false);
});
}
private BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
try { return settlements.confirmedCharge(task, result); }
catch (ApiException exception) {
if ("TASK_USAGE_PENDING".equals(exception.getCode())) throw new ApiException("TASK_USAGE_REQUIRED", "成功核账必须提供完整可核验用量;任务与冻结额度保持不变", 422);
throw exception;
}
}
private void resume(Map<String,Object> task, TaskReconciliationCommand command) {
String originalId = (String) task.get("provider_task_id");
if (originalId != null && !originalId.equals(command.providerTaskId())) throw conflict("不能将已受理任务改绑到另一个上游编号");
String fingerprint = (String) task.get("provider_key_fingerprint");
if (fingerprint != null && !fingerprint.equals(command.credentialFingerprint())) throw conflict("必须继续使用创建任务时的账号指纹,不能切换账号");
if (task.get("provider_id_snapshot") == null || task.get("upstream_model_snapshot") == null) throw conflict("历史任务原执行路由无法证明,禁止猜测后恢复查询;请核实终态");
provider.requirePollableBinding(String.valueOf(task.get("provider_id_snapshot")), String.valueOf(task.get("model_id")),
String.valueOf(task.get("upstream_model_snapshot")), String.valueOf(task.get("task_type")), command.credentialFingerprint());
String id = String.valueOf(task.get("id"));
if (!guard.tryTaskUserSlot((java.util.UUID) task.get("api_key_id"), id, timeoutMs + 180000)) throw new ApiException("TASK_USER_BUSY", "用户当前并发已满,尚未恢复轮询,请稍后重试", 429);
jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=now() WHERE id=?",
command.providerTaskId(), command.credentialFingerprint(), OffsetDateTime.now(ZoneOffset.UTC).plus(java.time.Duration.ofMillis(timeoutMs)), id);
usage.updateTask(id, "running", null, null, null);
}
private void validate(TaskReconciliationCommand command) {
if (command == null || command.requestId() == null || command.expectedUpdatedAt() == null) throw new ApiException("RECONCILIATION_PRECONDITION_REQUIRED", "必须提供操作编号和任务版本", 422);
if (command.action() == null || !List.of("resume_polling", "confirm_failed", "confirm_succeeded").contains(command.action())) throw new ApiException("INVALID_RECONCILIATION_ACTION", "核账操作无效", 422);
text(command.evidenceReference(), 3, 500); text(command.note(), 10, 2000);
if ("resume_polling".equals(command.action())) {
text(command.providerTaskId(), 1, 160);
if (!command.providerTaskId().matches("[A-Za-z0-9][A-Za-z0-9._:-]{0,159}")) throw new ApiException("INVALID_PROVIDER_TASK_ID", "上游任务编号不能包含路径、查询串或空白", 422);
if (command.credentialFingerprint() == null || !command.credentialFingerprint().matches("[0-9a-f]{64}")) throw new ApiException("INVALID_CREDENTIAL_FINGERPRINT", "必须提供 SHA-256 账号指纹,不能填写 API Key", 422);
if (command.result() != null || command.expectedCost() != null) throw conflict("恢复轮询不能同时修改结果或扣费金额");
} else {
if (command.providerTaskId() != null || command.credentialFingerprint() != null) throw conflict("终态核账不能改绑上游编号或账号");
if ("confirm_succeeded".equals(command.action())) {
if (command.expectedCost() == null || command.expectedCost().signum() < 0 || command.expectedCost().precision() > 20) throw new ApiException("EXPECTED_COST_REQUIRED", "请确认按原价格快照核算的非负扣费金额", 422);
try { command.expectedCost().setScale(8, RoundingMode.UNNECESSARY); }
catch (ArithmeticException exception) { throw new ApiException("INVALID_EXPECTED_COST", "确认金额最多 8 位小数", 422); }
} else if (command.result() != null || command.expectedCost() != null) throw conflict("确认失败不能附带成功结果或扣费金额");
}
}
private void text(String value, int min, int max) {
if (value == null || value.trim().length() < min || value.trim().length() > max || value.chars().anyMatch(Character::isISOControl)) throw new ApiException("RECONCILIATION_EVIDENCE_REQUIRED", "必须填写有效的上游证据引用与 10–2000 字核账说明;不得包含控制字符", 422);
}
private String requestHash(TaskReconciliationCommand command, String taskId, Map<String,Object> result) {
var body = new LinkedHashMap<String,Object>();
body.put("taskId", taskId); body.put("action", command.action()); body.put("expectedUpdatedAt", command.expectedUpdatedAt().toInstant().toString());
body.put("evidenceReference", command.evidenceReference().trim()); body.put("note", command.note().trim());
body.put("providerTaskId", command.providerTaskId()); body.put("credentialFingerprint", command.credentialFingerprint());
body.put("expectedCost", command.expectedCost()); body.put("result", result);
return TaskCreationIdentity.of(json, command.requestId().toString(), "task-reconciliation", body).requestHash();
}
private Map<String,Object> task(String taskId, boolean lock) {
if (taskId == null || taskId.isBlank() || taskId.length() > 160) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
var rows = jdbc.queryForList(TASK_COLUMNS + " WHERE t.id=?" + (lock ? " FOR UPDATE" : ""), taskId);
if (rows.isEmpty()) throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404);
var task = rows.getFirst();
task.put("pricing_snapshot", task.get("frozen_pricing"));
return task;
}
private Map<String,Object> view(Map<String,Object> task) {
var view = new LinkedHashMap<String,Object>();
view.put("id", task.get("id")); view.put("userId", task.get("user_id")); view.put("modelId", task.get("model_id"));
view.put("taskType", task.get("task_type")); view.put("status", task.get("status"));
view.put("updatedAt", timestamp(task.get("updated_at")).toInstant().toString());
view.put("reservedAmount", ((BigDecimal) task.get("reserved_amount")).toPlainString());
view.put("costAmount", task.get("cost_amount") == null ? null : ((BigDecimal) task.get("cost_amount")).toPlainString());
view.put("providerId", task.get("provider_id_snapshot")); view.put("upstreamModel", task.get("upstream_model_snapshot"));
view.put("providerTaskId", task.get("provider_task_id")); view.put("credentialFingerprint", task.get("provider_key_fingerprint"));
view.put("completionStatus", task.get("completion_status")); view.put("requiresReconciliation", "reconciling".equals(task.get("status")));
view.put("pricing", task.get("frozen_pricing") == null ? null : decode(task.get("frozen_pricing")));
view.put("error", task.get("error") == null ? null : decode(task.get("error")));
return view;
}
private OffsetDateTime timestamp(Object value) {
if (value instanceof OffsetDateTime time) return time;
if (value instanceof java.sql.Timestamp time) return time.toInstant().atOffset(ZoneOffset.UTC);
throw new IllegalStateException("Persisted task timestamp missing");
}
private Map<String,Object> decode(Object value) {
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Persisted reconciliation record cannot be read"); }
}
private String encode(Object value) {
try { return json.writeValueAsString(value); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Reconciliation record cannot be encoded"); }
}
private ApiException conflict(String message) { return new ApiException("RECONCILIATION_CONFLICT", message, 409); }
}
@@ -0,0 +1,125 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.EnterprisePolicies;
import com.moyi.controlplane.gateway.GatewayGuard;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
/** The one task/wallet/usage settlement transaction, shared by workers and audited reconciliation. */
@Service
public class TaskSettlementService {
private final JdbcTemplate jdbc;
private final ObjectMapper json;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final EnterprisePolicies policies;
private final TransactionTemplate transactions;
public TaskSettlementService(JdbcTemplate jdbc, ObjectMapper json, GatewayGuard guard,
UsageAuditService usage, PlatformTransactionManager manager) {
this.jdbc = jdbc;
this.json = json;
this.guard = guard;
this.usage = usage;
this.policies = new EnterprisePolicies(jdbc, json);
this.transactions = manager == null ? null : new TransactionTemplate(manager);
}
void finish(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal task state required");
transaction(() -> {
var task = jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE", taskId);
if (!List.of("queued", "running", "reconciling").contains(task.get("status"))) return null;
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
AsyncPriceSchedule.Bill bill = null;
String finalCode = code, finalMessage = message;
if ("succeeded".equals(status) && task.get("pricing_snapshot") != null) {
try { bill = AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result); }
catch (ApiException exception) {
finalCode = "TASK_USAGE_PENDING";
finalMessage = "生成已完成,但上游计费用量未能核对;暂不扣费,等待核账";
}
}
jdbc.update("UPDATE model_tasks SET status=?,progress=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),reconciliation_required_at=NULL,updated_at=now(),completed_at=now() WHERE id=?",
status, progress, encode(result), finalCode == null ? null : encode(Map.of("code", finalCode, "message", finalMessage)), taskId);
if ("TASK_USAGE_PENDING".equals(finalCode)) {
release(taskId);
jdbc.update("UPDATE model_tasks SET cost_amount=NULL WHERE id=?", taskId);
var observed = new LinkedHashMap<String,Object>();
if (result != null) for (String field : List.of("usage", "usageMetadata")) if (result.get(field) instanceof Map<?,?> value) observed.put(field, value);
jdbc.update("UPDATE usage_events SET cost_amount=NULL,cost_status='pending',observed_usage=CAST(? AS jsonb) WHERE provider_task_id=? AND event_kind='task'", encode(observed), taskId);
} else if ("succeeded".equals(status)) settle(taskId, bill);
else release(taskId);
usage.updateTask(taskId, status, result, finalCode, finalMessage);
return null;
});
}
/** Strict calculation for explicit administrator confirmation: missing usage is an error, not a refund. */
BigDecimal confirmedCharge(Map<String,Object> task, Map<String,Object> result) {
if (task.get("pricing_snapshot") == null) return (BigDecimal) task.get("reserved_amount");
return AsyncPriceSchedule.bill(pricing(task.get("pricing_snapshot")), result).amount();
}
private void settle(String taskId, AsyncPriceSchedule.Bill bill) {
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal reserved = (BigDecimal) task.get("reserved_amount");
BigDecimal amount = bill == null ? reserved : bill.amount();
releasePolicies(taskId, task);
if (amount == null || task.get("cost_amount") != null) return;
UUID userId = (UUID) task.get("user_id");
var wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId);
BigDecimal before = (BigDecimal) wallet.get("balance"), after = before.subtract(amount);
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, reserved, userId);
if (amount.signum() > 0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)",
UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before);
jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId);
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
if (bill != null) jdbc.update("UPDATE usage_events SET input_units=?,output_units=?,observed_usage=jsonb_build_object('input_tokens',?::bigint,'output_tokens',?::bigint,'image_input_tokens',?::bigint) WHERE provider_task_id=? AND event_kind='task'",
bill.input(), bill.output(), bill.input(), bill.output(), bill.imageInput(), taskId);
}
private void release(String taskId) {
var task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
releasePolicies(taskId, task);
if (amount != null && task.get("cost_amount") == null) {
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, task.get("user_id"));
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
}
}
private void releasePolicies(String taskId, Map<String,Object> task) {
policies.release(taskId);
guard.releaseTaskUserSlot((UUID) task.get("user_id"), taskId);
}
private Map<String,Object> pricing(Object value) {
try { return json.readValue(String.valueOf(value), new TypeReference<>() {}); }
catch (JsonProcessingException exception) { throw new ApiException("INVALID_PRICE_SNAPSHOT", "任务价格快照无法核验", 409); }
}
private String encode(Object value) {
if (value == null) return null;
try { return json.writeValueAsString(value); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Task settlement cannot be encoded"); }
}
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
private <T> T transaction(Supplier<T> work) { return transactions == null ? work.get() : transactions.execute(status -> work.get()); }
}
@@ -0,0 +1,23 @@
-- Freeze route identity without copying credentials or mutable provider configuration.
ALTER TABLE model_tasks ADD COLUMN provider_id_snapshot varchar(64) REFERENCES model_providers(id);
ALTER TABLE model_tasks ADD COLUMN upstream_model_snapshot varchar(160);
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_execution_binding_check CHECK (
(provider_id_snapshot IS NULL AND upstream_model_snapshot IS NULL)
OR (provider_id_snapshot IS NOT NULL AND upstream_model_snapshot IS NOT NULL
AND length(trim(provider_id_snapshot)) > 0 AND length(trim(upstream_model_snapshot)) > 0)
);
-- Never guess the original route of an already-submitted legacy task.
-- Unsubmitted queued work can safely freeze its route at migration time.
UPDATE model_tasks t SET provider_id_snapshot=m.provider_id, upstream_model_snapshot=m.upstream_model
FROM model_catalog m
WHERE t.model_id=m.id AND t.status='queued' AND t.provider_task_id IS NULL;
COMMENT ON COLUMN model_tasks.provider_id_snapshot IS 'Provider identity at admission; NULL for ambiguous legacy accepted tasks';
-- New proven bindings cannot associate the same provider/account task with two local wallets.
-- Legacy accepted rows are deliberately not guessed/backfilled, so this index does not rewrite their history.
CREATE UNIQUE INDEX model_tasks_proven_upstream_identity_idx
ON model_tasks(provider_id_snapshot,provider_key_fingerprint,provider_task_id)
WHERE provider_id_snapshot IS NOT NULL AND provider_key_fingerprint IS NOT NULL
AND provider_task_id IS NOT NULL AND provider_task_id <> 'mock';
@@ -0,0 +1,20 @@
-- Append-only operation receipts; no prompts, generated media or raw credentials.
CREATE TABLE task_reconciliations (
request_id uuid PRIMARY KEY,
task_id varchar(160) NOT NULL REFERENCES model_tasks(id),
actor_user_id uuid NOT NULL REFERENCES users(id),
action varchar(24) NOT NULL CHECK (action IN ('resume_polling','confirm_failed','confirm_succeeded')),
request_hash varchar(64) NOT NULL CHECK (request_hash ~ '^[0-9a-f]{64}$'),
evidence_reference varchar(500) NOT NULL CHECK (length(trim(evidence_reference)) BETWEEN 3 AND 500),
note varchar(2000) NOT NULL CHECK (length(trim(note)) BETWEEN 10 AND 2000),
before_state jsonb NOT NULL,
after_state jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX task_reconciliations_task_idx ON task_reconciliations(task_id,created_at DESC);
CREATE FUNCTION protect_task_reconciliation_history() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
RAISE EXCEPTION 'Task reconciliation history is append-only';
END $$;
CREATE TRIGGER task_reconciliation_history_immutable BEFORE UPDATE OR DELETE ON task_reconciliations
FOR EACH ROW EXECUTE FUNCTION protect_task_reconciliation_history();
@@ -0,0 +1,14 @@
-- A resumed task renews its own lease instead of consuming a second slot.
local time = redis.call('TIME')
local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
local existing = redis.call('ZSCORE', KEYS[1], ARGV[3])
if not existing and redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[1]) then
return 0
end
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[2]), ARGV[3])
if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then
redis.call('PEXPIRE', KEYS[1], ARGV[2])
end
if existing then return 2 end
return 1