fix async queue and provider key rotation
This commit is contained in:
1 parent
962eef26e5
commit
9750aceb03
4 files changed
+133
-26
No files matched your search
+103
-16
@@ -22,6 +22,8 @@ import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Service
|
||||
@@ -30,14 +32,16 @@ public class AsyncTaskService {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final CatalogService catalogService;
|
||||
private final int maxQueuedPerModel;
|
||||
private final int defaultModelConcurrency;
|
||||
private final GatewayGuard gatewayGuard;
|
||||
private final UsageAuditService usage;
|
||||
private final AsyncProviderClient providerClient;
|
||||
private final TransactionTemplate transactions;
|
||||
|
||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, gatewayGuard, usage, null);
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, gatewayGuard, usage, null, null);
|
||||
}
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
@@ -46,18 +50,22 @@ public class AsyncTaskService {
|
||||
ObjectMapper objectMapper,
|
||||
CatalogService catalogService,
|
||||
@Value("${moyi.tasks.max-queued-per-model:100}") int maxQueuedPerModel,
|
||||
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
|
||||
GatewayGuard gatewayGuard,
|
||||
UsageAuditService usage,
|
||||
AsyncProviderClient providerClient
|
||||
AsyncProviderClient providerClient,
|
||||
PlatformTransactionManager transactionManager
|
||||
) {
|
||||
if (maxQueuedPerModel < 1) throw new IllegalArgumentException("Task queue limit must be positive");
|
||||
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
|
||||
this.jdbc = jdbc;
|
||||
this.objectMapper = objectMapper;
|
||||
this.catalogService = catalogService;
|
||||
this.maxQueuedPerModel = maxQueuedPerModel;
|
||||
this.defaultModelConcurrency = defaultModelConcurrency;
|
||||
this.gatewayGuard = gatewayGuard;
|
||||
this.usage = usage;
|
||||
this.providerClient = providerClient;
|
||||
this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -74,6 +82,10 @@ public class AsyncTaskService {
|
||||
if (!catalogService.keyCanInvoke(key, modelId)) throw new ApiException("MODEL_FORBIDDEN", "当前 Key 无权调用该模型", 403);
|
||||
if (!taskType.equals(model.get("type")) || !"async".equals(model.get("mode"))) throw new ApiException("NOT_ASYNC_MEDIA_MODEL", "该模型不是对应的异步媒体模型", 400);
|
||||
if (!"provider-mock".equals(model.get("providerId")) && providerClient == null) throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "异步执行器未配置", 501);
|
||||
// Serialize the queue-capacity check with task insertion for this
|
||||
// model. Without the transaction advisory lock concurrent creates
|
||||
// could all observe the same count and exceed maxQueuedPerModel.
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId);
|
||||
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId);
|
||||
if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429);
|
||||
String prompt = String.valueOf(input.getOrDefault("prompt", "")).trim();
|
||||
@@ -82,18 +94,12 @@ public class AsyncTaskService {
|
||||
UUID userId = catalogService.keyOwnerId(apiKeyId);
|
||||
BigDecimal reservedAmount = taskCharge(model);
|
||||
reserveWallet(userId, reservedAmount);
|
||||
String providerTaskId;
|
||||
try {
|
||||
if ("provider-mock".equals(model.get("providerId"))) providerTaskId = "mock";
|
||||
else {
|
||||
if (providerClient == null) throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "异步执行器未配置", 501);
|
||||
Map<String,Object> upstream = providerClient.submit(String.valueOf(model.get("providerId")), String.valueOf(model.get("upstreamModel")), taskType, prompt);
|
||||
providerTaskId = providerTaskId(upstream);
|
||||
}
|
||||
} catch (RuntimeException exception) {
|
||||
releaseWallet(userId, reservedAmount);
|
||||
throw exception;
|
||||
}
|
||||
// Real async providers are submitted by the worker below. Creating a
|
||||
// task must only reserve the wallet and enqueue the work; submitting
|
||||
// here bypassed the model concurrency limit and made the queue merely
|
||||
// cosmetic. Mock tasks keep their sentinel so existing deterministic
|
||||
// tests can complete them without an upstream call.
|
||||
String providerTaskId = "provider-mock".equals(model.get("providerId")) ? "mock" : null;
|
||||
String id = "task_" + UUID.randomUUID().toString().replace("-", "").substring(0, 24);
|
||||
OffsetDateTime now = now();
|
||||
try {
|
||||
@@ -137,7 +143,6 @@ public class AsyncTaskService {
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}")
|
||||
@Transactional
|
||||
public void completeMockTasks() {
|
||||
List<String> taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id"));
|
||||
jdbc.update(
|
||||
@@ -148,6 +153,68 @@ public class AsyncTaskService {
|
||||
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.provider_task_id, m.provider_id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' ORDER BY t.created_at LIMIT 50");
|
||||
for (Map<String,Object> row : pending) pollProviderTask(row);
|
||||
dispatchQueuedTasks();
|
||||
}
|
||||
|
||||
/**
|
||||
* Claim and submit queued real tasks up to each model's configured
|
||||
* concurrency. The claim is a single PostgreSQL statement: a per-model
|
||||
* advisory lock plus a compare-and-set update prevents two API replicas
|
||||
* from starting more than the configured number of upstream tasks.
|
||||
*/
|
||||
private void dispatchQueuedTasks() {
|
||||
if (providerClient == null) return;
|
||||
// A process can die after claiming a task but before receiving an
|
||||
// upstream task id. Return those claims to the queue after a short
|
||||
// grace period so they neither consume a concurrency slot forever nor
|
||||
// leave the user's reservation stuck.
|
||||
jdbc.update("UPDATE model_tasks SET status='queued', updated_at=now() WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes'");
|
||||
for (int i = 0; i < 50; i++) {
|
||||
Map<String,Object> task = claimQueuedTask();
|
||||
if (task == null) return;
|
||||
String taskId = String.valueOf(task.get("id"));
|
||||
usage.updateTask(taskId, "running", null, null, null);
|
||||
try {
|
||||
Map<String,Object> upstream = providerClient.submit(
|
||||
String.valueOf(task.get("provider_id")),
|
||||
String.valueOf(task.get("upstream_model")),
|
||||
String.valueOf(task.get("task_type")),
|
||||
String.valueOf(task.get("prompt"))
|
||||
);
|
||||
jdbc.update("UPDATE model_tasks SET provider_task_id=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), now(), taskId);
|
||||
} catch (RuntimeException exception) {
|
||||
jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status='running'",
|
||||
writeJson(Map.of("message", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage())), now(), now(), taskId);
|
||||
releaseTask(taskId);
|
||||
usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_SUBMIT_FAILED", "上游异步任务创建失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String,Object> claimQueuedTask() {
|
||||
String sql = """
|
||||
WITH candidate AS (
|
||||
SELECT t.id
|
||||
FROM model_tasks t
|
||||
JOIN model_catalog m ON m.id=t.model_id
|
||||
WHERE t.status='queued'
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
AND pg_try_advisory_xact_lock(hashtextextended('moyi:async:' || t.model_id, 0))
|
||||
AND (SELECT COUNT(*) FROM model_tasks running
|
||||
WHERE running.model_id=t.model_id AND running.status='running')
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
ORDER BY t.created_at
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE model_tasks t
|
||||
SET status='running', updated_at=now()
|
||||
FROM candidate c
|
||||
WHERE t.id=c.id AND t.status='queued'
|
||||
RETURNING t.id, t.provider_id, t.upstream_model, t.task_type, t.prompt
|
||||
""";
|
||||
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency);
|
||||
return rows.isEmpty() ? null : rows.getFirst();
|
||||
}
|
||||
|
||||
private Map<String, Object> requireKey(String authorization) {
|
||||
@@ -207,6 +274,10 @@ public class AsyncTaskService {
|
||||
}
|
||||
} catch (Exception exception) {
|
||||
jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", writeJson(Map.of("message", "上游异步任务查询失败")), now(), now(), taskId);
|
||||
// A polling failure is terminal for this task. Release the
|
||||
// reserved wallet amount as part of the same failure path so a
|
||||
// transient/provider error cannot permanently lock user credit.
|
||||
releaseTask(taskId);
|
||||
usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_POLL_FAILED", "上游异步任务查询失败");
|
||||
}
|
||||
}
|
||||
@@ -284,6 +355,14 @@ public class AsyncTaskService {
|
||||
}
|
||||
|
||||
private void settleTask(String taskId) {
|
||||
if (transactions == null) {
|
||||
settleTaskInTransaction(taskId);
|
||||
return;
|
||||
}
|
||||
transactions.executeWithoutResult(status -> settleTaskInTransaction(taskId));
|
||||
}
|
||||
|
||||
private void settleTaskInTransaction(String taskId) {
|
||||
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
|
||||
if (amount == null || amount.signum() == 0 || task.get("cost_amount") != null) return;
|
||||
@@ -298,6 +377,14 @@ public class AsyncTaskService {
|
||||
}
|
||||
|
||||
private void releaseTask(String taskId) {
|
||||
if (transactions == null) {
|
||||
releaseTaskInTransaction(taskId);
|
||||
return;
|
||||
}
|
||||
transactions.executeWithoutResult(status -> releaseTaskInTransaction(taskId));
|
||||
}
|
||||
|
||||
private void releaseTaskInTransaction(String taskId) {
|
||||
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
|
||||
if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) {
|
||||
|
||||
Reference in new issue
Block a user