fix async queue and provider key rotation

This commit is contained in:
wlx committed 2026-09-27 18:05:35 +08:00
1 parent 962eef26e5
commit 9750aceb03
4 files changed
+133 -26

No files matched your search

@@ -17,6 +17,8 @@ import java.util.Map;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import jakarta.annotation.PreDestroy;
import org.springframework.dao.EmptyResultDataAccessException;
@@ -31,6 +33,9 @@ public class OpenAiCompatibleProvider {
private final SecretCipher secretCipher;
private final Map<String, ProviderConfig> providers;
private final org.springframework.core.env.Environment environment;
/** Cursor state must survive dynamic provider resolution; otherwise every
* DB-backed channel request starts at key 0 and never rotates. */
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
private final int timeoutSeconds;
@@ -189,7 +194,8 @@ public class OpenAiCompatibleProvider {
String ciphertext = row.get("credential_ciphertext") == null ? "" : String.valueOf(row.get("credential_ciphertext"));
String apiKey = ciphertext.isBlank() ? "" : secretCipher.decrypt(ciphertext);
String baseUrl = row.get("base_url") == null ? "" : String.valueOf(row.get("base_url"));
return configured(providerId, providerCode, apiKey, baseUrl, environment);
return configured(providerId, providerCode, apiKey, baseUrl, environment,
keyCursors.computeIfAbsent(providerId, ignored -> new AtomicInteger()));
} catch (EmptyResultDataAccessException exception) {
return fixed;
}
@@ -204,15 +210,19 @@ public class OpenAiCompatibleProvider {
}
private ProviderConfig configured(String providerId, String fallbackKey, String baseUrl, org.springframework.core.env.Environment environment) {
return configured(providerId, providerId, fallbackKey, baseUrl, environment);
return configured(providerId, providerId, fallbackKey, baseUrl, environment, new AtomicInteger());
}
private ProviderConfig configured(String providerId, String providerCode, String fallbackKey, String baseUrl, org.springframework.core.env.Environment environment) {
return configured(providerId, providerCode, fallbackKey, baseUrl, environment, new AtomicInteger());
}
private ProviderConfig configured(String providerId, String providerCode, String fallbackKey, String baseUrl, org.springframework.core.env.Environment environment, AtomicInteger cursor) {
List<String> keys = new ArrayList<>();
if (fallbackKey != null && !fallbackKey.isBlank()) keys.add(fallbackKey.trim());
addEnvironmentKeys(keys, providerId, environment);
if (providerCode != null && !providerCode.isBlank() && !providerCode.equals(providerId)) addEnvironmentKeys(keys, providerCode, environment);
return new ProviderConfig(keys.stream().distinct().toList(), baseUrl);
return new ProviderConfig(keys.stream().distinct().toList(), baseUrl, cursor);
}
private void addEnvironmentKeys(List<String> keys, String alias, org.springframework.core.env.Environment environment) {
@@ -222,7 +232,6 @@ public class OpenAiCompatibleProvider {
}
private record ProviderConfig(List<String> apiKeys, String baseUrl, AtomicInteger cursor) {
private ProviderConfig(List<String> apiKeys, String baseUrl) { this(apiKeys, baseUrl, new AtomicInteger()); }
private boolean hasKey() { return !apiKeys.isEmpty(); }
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
}
@@ -15,6 +15,8 @@ import java.util.List;
import java.util.Map;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.springframework.core.env.Environment;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
@@ -27,6 +29,9 @@ public class AsyncProviderClient {
private final SecretCipher cipher;
private final ObjectMapper mapper;
private final Environment environment;
/** Keep rotation state across submit/poll calls; rebuilding a Provider on
* every request otherwise selected the first upstream key forever. */
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
public AsyncProviderClient(JdbcTemplate jdbc, SecretCipher cipher, ObjectMapper mapper, Environment environment) {
@@ -97,7 +102,7 @@ public class AsyncProviderClient {
addEnvironmentKeys(keys, "MOYI_TRIPO_API_KEY");
}
if (keys.isEmpty()) throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道尚未配置 API Key", 503);
return new Provider(code, String.valueOf(row.get("base_url")), keys.stream().distinct().toList());
return new Provider(code, String.valueOf(row.get("base_url")), keys.stream().distinct().toList(), keyCursors.computeIfAbsent(id, ignored -> new AtomicInteger()));
} catch (EmptyResultDataAccessException exception) { throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道不存在或已停用", 503); }
}
@@ -116,7 +121,6 @@ public class AsyncProviderClient {
return normalizedCode.contains("tripo") || normalizedUrl.contains("tripo3d");
}
private record Provider(String code, String baseUrl, List<String> apiKeys, AtomicInteger cursor) {
private Provider(String code, String baseUrl, List<String> apiKeys) { this(code, baseUrl, apiKeys, new AtomicInteger()); }
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
}
}
@@ -22,6 +22,8 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.transaction.annotation.Transactional;
@Service
@@ -30,14 +32,16 @@ public class AsyncTaskService {
private final ObjectMapper objectMapper;
private final CatalogService catalogService;
private final int maxQueuedPerModel;
private final int defaultModelConcurrency;
private final GatewayGuard gatewayGuard;
private final UsageAuditService usage;
private final AsyncProviderClient providerClient;
private final TransactionTemplate transactions;
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, gatewayGuard, usage, null);
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, gatewayGuard, usage, null, null);
}
@org.springframework.beans.factory.annotation.Autowired
@@ -46,18 +50,22 @@ public class AsyncTaskService {
ObjectMapper objectMapper,
CatalogService catalogService,
@Value("${moyi.tasks.max-queued-per-model:100}") int maxQueuedPerModel,
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
GatewayGuard gatewayGuard,
UsageAuditService usage,
AsyncProviderClient providerClient
AsyncProviderClient providerClient,
PlatformTransactionManager transactionManager
) {
if (maxQueuedPerModel < 1) throw new IllegalArgumentException("Task queue limit must be positive");
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
this.jdbc = jdbc;
this.objectMapper = objectMapper;
this.catalogService = catalogService;
this.maxQueuedPerModel = maxQueuedPerModel;
this.defaultModelConcurrency = defaultModelConcurrency;
this.gatewayGuard = gatewayGuard;
this.usage = usage;
this.providerClient = providerClient;
this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager);
}
@Transactional
@@ -74,6 +82,10 @@ public class AsyncTaskService {
if (!catalogService.keyCanInvoke(key, modelId)) throw new ApiException("MODEL_FORBIDDEN", "当前 Key 无权调用该模型", 403);
if (!taskType.equals(model.get("type")) || !"async".equals(model.get("mode"))) throw new ApiException("NOT_ASYNC_MEDIA_MODEL", "该模型不是对应的异步媒体模型", 400);
if (!"provider-mock".equals(model.get("providerId")) && providerClient == null) throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "异步执行器未配置", 501);
// Serialize the queue-capacity check with task insertion for this
// model. Without the transaction advisory lock concurrent creates
// could all observe the same count and exceed maxQueuedPerModel.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId);
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId);
if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429);
String prompt = String.valueOf(input.getOrDefault("prompt", "")).trim();
@@ -82,18 +94,12 @@ public class AsyncTaskService {
UUID userId = catalogService.keyOwnerId(apiKeyId);
BigDecimal reservedAmount = taskCharge(model);
reserveWallet(userId, reservedAmount);
String providerTaskId;
try {
if ("provider-mock".equals(model.get("providerId"))) providerTaskId = "mock";
else {
if (providerClient == null) throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "异步执行器未配置", 501);
Map<String,Object> upstream = providerClient.submit(String.valueOf(model.get("providerId")), String.valueOf(model.get("upstreamModel")), taskType, prompt);
providerTaskId = providerTaskId(upstream);
}
} catch (RuntimeException exception) {
releaseWallet(userId, reservedAmount);
throw exception;
}
// Real async providers are submitted by the worker below. Creating a
// task must only reserve the wallet and enqueue the work; submitting
// here bypassed the model concurrency limit and made the queue merely
// cosmetic. Mock tasks keep their sentinel so existing deterministic
// tests can complete them without an upstream call.
String providerTaskId = "provider-mock".equals(model.get("providerId")) ? "mock" : null;
String id = "task_" + UUID.randomUUID().toString().replace("-", "").substring(0, 24);
OffsetDateTime now = now();
try {
@@ -137,7 +143,6 @@ public class AsyncTaskService {
}
@Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}")
@Transactional
public void completeMockTasks() {
List<String> taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id"));
jdbc.update(
@@ -148,6 +153,68 @@ public class AsyncTaskService {
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.provider_task_id, m.provider_id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' ORDER BY t.created_at LIMIT 50");
for (Map<String,Object> row : pending) pollProviderTask(row);
dispatchQueuedTasks();
}
/**
* Claim and submit queued real tasks up to each model's configured
* concurrency. The claim is a single PostgreSQL statement: a per-model
* advisory lock plus a compare-and-set update prevents two API replicas
* from starting more than the configured number of upstream tasks.
*/
private void dispatchQueuedTasks() {
if (providerClient == null) return;
// A process can die after claiming a task but before receiving an
// upstream task id. Return those claims to the queue after a short
// grace period so they neither consume a concurrency slot forever nor
// leave the user's reservation stuck.
jdbc.update("UPDATE model_tasks SET status='queued', updated_at=now() WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes'");
for (int i = 0; i < 50; i++) {
Map<String,Object> task = claimQueuedTask();
if (task == null) return;
String taskId = String.valueOf(task.get("id"));
usage.updateTask(taskId, "running", null, null, null);
try {
Map<String,Object> upstream = providerClient.submit(
String.valueOf(task.get("provider_id")),
String.valueOf(task.get("upstream_model")),
String.valueOf(task.get("task_type")),
String.valueOf(task.get("prompt"))
);
jdbc.update("UPDATE model_tasks SET provider_task_id=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), now(), taskId);
} catch (RuntimeException exception) {
jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status='running'",
writeJson(Map.of("message", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage())), now(), now(), taskId);
releaseTask(taskId);
usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_SUBMIT_FAILED", "上游异步任务创建失败");
}
}
}
private Map<String,Object> claimQueuedTask() {
String sql = """
WITH candidate AS (
SELECT t.id
FROM model_tasks t
JOIN model_catalog m ON m.id=t.model_id
WHERE t.status='queued'
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
AND pg_try_advisory_xact_lock(hashtextextended('moyi:async:' || t.model_id, 0))
AND (SELECT COUNT(*) FROM model_tasks running
WHERE running.model_id=t.model_id AND running.status='running')
< COALESCE(m.max_concurrency, ?)
ORDER BY t.created_at
LIMIT 1
)
UPDATE model_tasks t
SET status='running', updated_at=now()
FROM candidate c
WHERE t.id=c.id AND t.status='queued'
RETURNING t.id, t.provider_id, t.upstream_model, t.task_type, t.prompt
""";
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency);
return rows.isEmpty() ? null : rows.getFirst();
}
private Map<String, Object> requireKey(String authorization) {
@@ -207,6 +274,10 @@ public class AsyncTaskService {
}
} catch (Exception exception) {
jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", writeJson(Map.of("message", "上游异步任务查询失败")), now(), now(), taskId);
// A polling failure is terminal for this task. Release the
// reserved wallet amount as part of the same failure path so a
// transient/provider error cannot permanently lock user credit.
releaseTask(taskId);
usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_POLL_FAILED", "上游异步任务查询失败");
}
}
@@ -284,6 +355,14 @@ public class AsyncTaskService {
}
private void settleTask(String taskId) {
if (transactions == null) {
settleTaskInTransaction(taskId);
return;
}
transactions.executeWithoutResult(status -> settleTaskInTransaction(taskId));
}
private void settleTaskInTransaction(String taskId) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
if (amount == null || amount.signum() == 0 || task.get("cost_amount") != null) return;
@@ -298,6 +377,14 @@ public class AsyncTaskService {
}
private void releaseTask(String taskId) {
if (transactions == null) {
releaseTaskInTransaction(taskId);
return;
}
transactions.executeWithoutResult(status -> releaseTaskInTransaction(taskId));
}
private void releaseTaskInTransaction(String taskId) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) {