package com.moyi.controlplane.task; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.moyi.controlplane.catalog.CatalogService; import com.moyi.controlplane.common.ApiException; import com.moyi.controlplane.gateway.GatewayGuard; import com.moyi.controlplane.usage.UsageAuditService; import java.sql.ResultSet; import java.sql.SQLException; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.annotation.Transactional; @Service public class AsyncTaskService { private final JdbcTemplate jdbc; private final ObjectMapper objectMapper; private final CatalogService catalogService; private final int maxQueuedPerModel; private final int defaultModelConcurrency; private final GatewayGuard gatewayGuard; private final UsageAuditService usage; private final AsyncProviderClient providerClient; private final TransactionTemplate transactions; /** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */ public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService, int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) { this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, gatewayGuard, usage, null, null); } @org.springframework.beans.factory.annotation.Autowired public AsyncTaskService( JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService, @Value("${moyi.tasks.max-queued-per-model:100}") int maxQueuedPerModel, @Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency, GatewayGuard gatewayGuard, UsageAuditService usage, AsyncProviderClient providerClient, PlatformTransactionManager transactionManager ) { if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive"); this.jdbc = jdbc; this.objectMapper = objectMapper; this.catalogService = catalogService; this.maxQueuedPerModel = maxQueuedPerModel; this.defaultModelConcurrency = defaultModelConcurrency; this.gatewayGuard = gatewayGuard; this.usage = usage; this.providerClient = providerClient; this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager); } @Transactional public Map create(String authorization, String taskType, Map input) { return createForKey(requireKey(authorization), taskType, input); } @Transactional public Map createForKey(Map key, String taskType, Map input) { gatewayGuard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false); String modelId = String.valueOf(input.getOrDefault("model", "")); Map model = catalogService.activeModel(modelId); if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线", 404); if (!catalogService.keyCanInvoke(key, modelId)) throw new ApiException("MODEL_FORBIDDEN", "当前 Key 无权调用该模型", 403); if (!taskType.equals(model.get("type")) || !"async".equals(model.get("mode"))) throw new ApiException("NOT_ASYNC_MEDIA_MODEL", "该模型不是对应的异步媒体模型", 400); if (!"provider-mock".equals(model.get("providerId")) && providerClient == null) throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "异步执行器未配置", 501); // Serialize the queue-capacity check with task insertion for this // model. Without the transaction advisory lock concurrent creates // could all observe the same count and exceed maxQueuedPerModel. jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId); Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId); if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429); String prompt = String.valueOf(input.getOrDefault("prompt", "")).trim(); if (prompt.isBlank()) throw new ApiException("PROMPT_REQUIRED", "生成提示词不能为空"); UUID apiKeyId = UUID.fromString(String.valueOf(key.get("id"))); UUID userId = catalogService.keyOwnerId(apiKeyId); BigDecimal reservedAmount = taskCharge(model); reserveWallet(userId, reservedAmount); // Real async providers are submitted by the worker below. Creating a // task must only reserve the wallet and enqueue the work; submitting // here bypassed the model concurrency limit and made the queue merely // cosmetic. Mock tasks keep their sentinel so existing deterministic // tests can complete them without an upstream call. String providerTaskId = "provider-mock".equals(model.get("providerId")) ? "mock" : null; String id = "task_" + UUID.randomUUID().toString().replace("-", "").substring(0, 24); OffsetDateTime now = now(); try { jdbc.update( "INSERT INTO model_tasks (id, user_id, api_key_id, model_id, task_type, provider_task_id, status, progress, prompt, reserved_amount, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?, ?)", id, userId, apiKeyId, modelId, taskType, providerTaskId, prompt, reservedAmount, now, now ); } catch (RuntimeException exception) { releaseWallet(userId, reservedAmount); throw exception; } usage.recordTask(userId, apiKeyId, modelId, String.valueOf(model.get("providerId")), id, "queued", input, null, null, null); catalogService.touchKey(key); return taskView(id); } public Map get(String authorization, String taskId) { return getForKey(requireKey(authorization), taskId); } public Map getForKey(Map key, String taskId) { UUID userId = catalogService.keyOwnerId(UUID.fromString(String.valueOf(key.get("id")))); try { return jdbc.queryForObject("SELECT * FROM model_tasks WHERE id = ? AND user_id = ?", this::mapTask, taskId, userId); } catch (org.springframework.dao.EmptyResultDataAccessException exception) { throw new ApiException("TASK_NOT_FOUND", "任务不存在", 404); } } public List> list(String authorization, int limit) { Map key = requireKey(authorization); UUID userId = catalogService.keyOwnerId(UUID.fromString(String.valueOf(key.get("id")))); int boundedLimit = Math.max(1, Math.min(limit, 100)); return jdbc.query("SELECT * FROM model_tasks WHERE user_id = ? ORDER BY created_at DESC LIMIT ?", this::mapTask, userId, boundedLimit); } public List> listForUser(String sessionToken, com.moyi.controlplane.auth.AuthService auth, int limit) { UUID userId = auth.requireUser(sessionToken).id(); int boundedLimit = Math.max(1, Math.min(limit, 100)); return jdbc.query("SELECT * FROM model_tasks WHERE user_id = ? ORDER BY created_at DESC LIMIT ?", this::mapTask, userId, boundedLimit); } @Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}") public void completeMockTasks() { List taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id")); jdbc.update( "UPDATE model_tasks SET status = 'succeeded', progress = 100, result = CAST(CASE WHEN task_type = '3d' THEN ? ELSE ? END AS jsonb), updated_at = ?, completed_at = ? WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", "{\"format\":\"glb\",\"url\":\"https://mock.moyi.ai/tasks/result.glb\"}", "{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now() ); taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null)); List> pending = jdbc.queryForList("SELECT t.id, t.provider_task_id, m.provider_id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' ORDER BY t.created_at LIMIT 50"); for (Map row : pending) pollProviderTask(row); dispatchQueuedTasks(); } /** * Claim and submit queued real tasks up to each model's configured * concurrency. The claim is a single PostgreSQL statement: a per-model * advisory lock plus a compare-and-set update prevents two API replicas * from starting more than the configured number of upstream tasks. */ private void dispatchQueuedTasks() { if (providerClient == null) return; // A process can die after claiming a task but before receiving an // upstream task id. Return those claims to the queue after a short // grace period so they neither consume a concurrency slot forever nor // leave the user's reservation stuck. jdbc.update("UPDATE model_tasks SET status='queued', updated_at=now() WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes'"); for (int i = 0; i < 50; i++) { Map task = claimQueuedTask(); if (task == null) return; String taskId = String.valueOf(task.get("id")); usage.updateTask(taskId, "running", null, null, null); try { Map upstream = providerClient.submit( String.valueOf(task.get("provider_id")), String.valueOf(task.get("upstream_model")), String.valueOf(task.get("task_type")), String.valueOf(task.get("prompt")) ); jdbc.update("UPDATE model_tasks SET provider_task_id=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), now(), taskId); } catch (RuntimeException exception) { jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status='running'", writeJson(Map.of("message", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage())), now(), now(), taskId); releaseTask(taskId); usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_SUBMIT_FAILED", "上游异步任务创建失败"); } } } private Map claimQueuedTask() { String sql = """ WITH candidate AS ( SELECT t.id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status='queued' AND t.provider_task_id IS NULL AND m.status='active' AND m.deleted_at IS NULL AND pg_try_advisory_xact_lock(hashtextextended('moyi:async:' || t.model_id, 0)) AND (SELECT COUNT(*) FROM model_tasks running WHERE running.model_id=t.model_id AND running.status='running') < COALESCE(m.max_concurrency, ?) ORDER BY t.created_at LIMIT 1 ) UPDATE model_tasks t SET status='running', updated_at=now() FROM candidate c WHERE t.id=c.id AND t.status='queued' RETURNING t.id, t.provider_id, t.upstream_model, t.task_type, t.prompt """; List> rows = jdbc.queryForList(sql, defaultModelConcurrency); return rows.isEmpty() ? null : rows.getFirst(); } private Map requireKey(String authorization) { if (authorization == null || !authorization.startsWith("Bearer ")) throw new ApiException("UNAUTHENTICATED", "请提供有效的 API Key", 401); Map key = catalogService.authenticateKey(authorization.substring(7)); if (key == null) throw new ApiException("INVALID_API_KEY", "API Key 无效或已停用", 401); return key; } private Map taskView(String id) { return jdbc.queryForObject("SELECT * FROM model_tasks WHERE id = ?", this::mapTask, id); } private Map mapTask(ResultSet resultSet, int rowNumber) throws SQLException { Map task = new LinkedHashMap<>(); task.put("id", resultSet.getString("id")); task.put("provider", "upstream"); task.put("model", resultSet.getString("model_id")); task.put("status", resultSet.getString("status")); task.put("progress", resultSet.getInt("progress")); task.put("createdAt", resultSet.getObject("created_at", OffsetDateTime.class)); task.put("updatedAt", resultSet.getObject("updated_at", OffsetDateTime.class)); putIfPresent(task, "costAmount", resultSet, "cost_amount"); String result = resultSet.getString("result"); if (result != null) task.put("result", readMap(result)); String error = resultSet.getString("error"); if (error != null) task.put("error", readMap(error)); return task; } private Map readMap(String value) { try { return objectMapper.readValue(value, new TypeReference<>() {}); } catch (JsonProcessingException exception) { return Map.of(); } } private OffsetDateTime now() { return OffsetDateTime.now(ZoneOffset.UTC); } private String providerTaskId(Map response) { Object direct = response.get("id"); if (direct == null && response.get("data") instanceof Map data) direct = data.get("task_id"); if (direct == null) throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回异步任务编号", 502); return String.valueOf(direct); } private void pollProviderTask(Map row) { String taskId = String.valueOf(row.get("id")); try { Map response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("provider_task_id"))); String status = normalizedStatus(response); int progress = progress(response, status); Map result = result(response); if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) { jdbc.update("UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", status, progress, result == null ? null : writeJson(result), "failed".equals(status) ? writeJson(Map.of("message", "上游异步任务失败")) : null, now(), now(), taskId); if ("succeeded".equals(status)) settleTask(taskId); else releaseTask(taskId); usage.updateTask(taskId, status, result, "failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null, "failed".equals(status) ? "上游异步任务失败" : null); } else { jdbc.update("UPDATE model_tasks SET status=?, progress=?, updated_at=? WHERE id=? AND status IN ('queued','running')", status, progress, now(), taskId); } } catch (Exception exception) { jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", writeJson(Map.of("message", "上游异步任务查询失败")), now(), now(), taskId); // A polling failure is terminal for this task. Release the // reserved wallet amount as part of the same failure path so a // transient/provider error cannot permanently lock user credit. releaseTask(taskId); usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_POLL_FAILED", "上游异步任务查询失败"); } } private String normalizedStatus(Map response) { Object raw = response.get("status"); if (raw == null && response.get("data") instanceof Map data) raw = data.get("status"); String value = String.valueOf(raw == null ? "running" : raw).toLowerCase(java.util.Locale.ROOT); if (value.contains("success") || value.equals("completed")) return "succeeded"; if (value.contains("fail") || value.equals("error") || value.equals("banned")) return "failed"; if (value.contains("cancel")) return "cancelled"; return "running"; } private int progress(Map response, String status) { Object raw = response.get("progress"); if (raw == null && response.get("data") instanceof Map data) raw = data.get("progress"); if (raw instanceof Number number) return Math.max(0, Math.min(100, number.intValue())); return "succeeded".equals(status) ? 100 : 0; } private Map result(Map response) { Object raw = response.get("output"); Map data = response.get("data") instanceof Map value ? value : null; if (raw == null && data != null) raw = data.get("output"); if (raw instanceof Map map) { Map value = new LinkedHashMap<>(); map.forEach((k,v)->value.put(String.valueOf(k),v)); copyAlias(value, "model_url", "modelUrl"); copyAlias(value, "rendered_image_url", "previewUrl"); copyAlias(value, "video_url", "videoUrl"); if (value.get("url") == null && value.get("modelUrl") != null) value.put("url", value.get("modelUrl")); if (value.get("url") == null && value.get("videoUrl") != null) value.put("url", value.get("videoUrl")); if (data != null && data.get("credits_consumed") != null) value.put("creditsConsumed", data.get("credits_consumed")); return value; } if (raw instanceof String value && !value.isBlank()) return Map.of("url", value); return null; } private void copyAlias(Map value, String source, String target) { if (value.get(target) == null && value.get(source) != null) value.put(target, value.get(source)); } private String writeJson(Object value) { try { return objectMapper.writeValueAsString(value); } catch (JsonProcessingException exception) { return "{}"; } } private BigDecimal taskCharge(Map model) { if ("provider-mock".equals(model.get("providerId"))) return BigDecimal.ZERO; if (!(model.get("pricing") instanceof Map pricing)) throw new ApiException("MODEL_NOT_PRICED", "该异步模型尚未配置按任务计费价格", 503); Object unitValue = pricing.get("unit"); String unit = String.valueOf(unitValue == null ? "" : unitValue).toLowerCase(java.util.Locale.ROOT); if (unit.contains("token")) throw new ApiException("ASYNC_BILLING_UNSUPPORTED", "异步任务不能使用 Token 价格,需配置按次、按生成或按媒体单位的固定售价", 503); Object value = pricing.get("price"); if (value == null) value = pricing.get("input"); try { BigDecimal amount = new BigDecimal(String.valueOf(value)); if (amount.signum() < 0 || amount.scale() > 8 || amount.compareTo(new BigDecimal("1000000")) > 0) throw new NumberFormatException(); return amount.setScale(8, RoundingMode.CEILING); } catch (Exception exception) { throw new ApiException("MODEL_NOT_PRICED", "该异步模型需配置有效的固定任务售价", 503); } } private void reserveWallet(UUID userId, BigDecimal amount) { if (amount.signum() == 0) return; jdbc.update("INSERT INTO user_wallets(user_id) VALUES (?) ON CONFLICT DO NOTHING", userId); Map wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId); BigDecimal available = ((BigDecimal) wallet.get("balance")).subtract((BigDecimal) wallet.get("reserved_balance")); if (available.compareTo(amount) < 0) throw new ApiException("INSUFFICIENT_BALANCE", "可用余额不足以创建此异步任务", 402); jdbc.update("UPDATE user_wallets SET reserved_balance=reserved_balance+?,updated_at=now() WHERE user_id=?", amount, userId); } private void releaseWallet(UUID userId, BigDecimal amount) { if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, userId); } private void settleTask(String taskId) { if (transactions == null) { settleTaskInTransaction(taskId); return; } transactions.executeWithoutResult(status -> settleTaskInTransaction(taskId)); } private void settleTaskInTransaction(String taskId) { Map task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId); BigDecimal amount = (BigDecimal) task.get("reserved_amount"); if (amount == null || amount.signum() == 0 || task.get("cost_amount") != null) return; UUID userId = (UUID) task.get("user_id"); Map wallet = jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE", userId); BigDecimal before = (BigDecimal) wallet.get("balance"); BigDecimal after = before.subtract(amount); jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", after, amount, userId); jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_after,currency,reason,operation,requested_amount,balance_before,actor_user_id) VALUES (?,?,?,?,?,'CNY',?,'usage',?,?,NULL)", UUID.randomUUID(), userId, taskRequestId(taskId), amount.negate(), after, "异步模型任务 · " + taskId, amount, before); jdbc.update("UPDATE model_tasks SET cost_amount=? WHERE id=?", amount, taskId); jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId); } private void releaseTask(String taskId) { if (transactions == null) { releaseTaskInTransaction(taskId); return; } transactions.executeWithoutResult(status -> releaseTaskInTransaction(taskId)); } private void releaseTaskInTransaction(String taskId) { Map task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId); BigDecimal amount = (BigDecimal) task.get("reserved_amount"); if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) { releaseWallet((UUID) task.get("user_id"), amount); jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId); } } private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); } private void putIfPresent(Map target, String name, ResultSet resultSet, String column) throws SQLException { Object value = resultSet.getObject(column); if (value != null) target.put(name, value); } }