fix: remove artificial model group concurrency cap

This commit is contained in:
wlx committed 2026-09-29 12:46:10 +08:00
1 parent 25114ca6b8
commit ce0a718686
13 files changed
+165 -34

No files matched your search

@@ -199,7 +199,7 @@ public class CatalogService {
Integer priority = integerOrNull(input.get("priority"));
Integer maxConcurrency = integerOrNull(input.get("maxConcurrency"));
if (priority != null && priority < 0) throw new ApiException("INVALID_MODEL_PRIORITY", "模型路由优先级必须大于等于 0");
if (maxConcurrency != null && maxConcurrency < 1) throw new ApiException("INVALID_MODEL_CONCURRENCY", "模型并发上限必须大于等于 1");
if (maxConcurrency != null && maxConcurrency < 0) throw new ApiException("INVALID_MODEL_CONCURRENCY", "模型并发上限必须大于等于 0;0 表示不设置平台分组上限");
jdbc.update(
"INSERT INTO model_catalog (id, provider_id, model_key, source_label, public_name, upstream_model, model_type, call_mode, description, input_types, output_types, capabilities, pricing, priority, max_concurrency, logical_model_id, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, CAST(? AS jsonb), CAST(? AS jsonb), CAST(? AS jsonb), CAST(? AS jsonb), ?, ?, ?, ?)",
id, providerId, modelKey, sourceLabel, publicName, upstreamModel, type, mode, description,
@@ -223,7 +223,7 @@ public class CatalogService {
if (findProvider(providerId)==null) throw new ApiException("PROVIDER_NOT_FOUND","渠道不存在",404);
if(groupName.length()>160 || upstream.length()>160) throw new ApiException("INVALID_MODEL","分组名称或上游标识过长");
Integer priority=integerOrNull(value.get("priority")), concurrency=integerOrNull(value.get("maxConcurrency"));
if ((priority!=null && priority<0) || (concurrency!=null && concurrency<1)) throw new ApiException("INVALID_MODEL_LIMIT","优先级不能小于 0,并发上限至少为 1");
if ((priority!=null && priority<0) || (concurrency!=null && concurrency<0)) throw new ApiException("INVALID_MODEL_LIMIT","优先级和并发上限不能小于 0;并发填 0 表示不设置平台分组上限");
validatePricing(value.get("pricing"));
jdbc.update("UPDATE model_catalog SET logical_model_id=?, provider_id=?, source_label=?, upstream_model=?, public_name=?, model_key=?, model_type=?, call_mode=?, description=?, input_types=CAST(? AS jsonb), output_types=CAST(? AS jsonb), pricing=CAST(? AS jsonb), priority=?, max_concurrency=?, status=?, updated_at=now() WHERE id=?",logicalId,providerId,groupName,upstream,definition.get("publicName"),definition.get("modelKey"),definition.get("type"),definition.get("mode"),definition.get("description"),jsonArray(definition.get("inputTypes")),jsonArray(definition.get("outputTypes")),jsonObjectOrNull(value.get("pricing")),priority,concurrency,status(value),id);
auth.audit("model_group.updated",admin.id(),new AuthService.RequestMeta(null,null),Map.of("groupId",id,"modelId",logicalId,"providerId",providerId));
@@ -38,6 +38,7 @@ public class GatewayGuard {
private final int readsPerMinute;
private final int defaultModelConcurrency;
private final int defaultKeyConcurrency;
private final boolean modelConcurrencyEnabled;
private final long queueWaitMs;
private final long queuePollMs;
private final String redisPrefix;
@@ -58,9 +59,14 @@ public class GatewayGuard {
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
@Value("${moyi.gateway.queue-wait-ms:30000}") long queueWaitMs,
@Value("${moyi.gateway.queue-poll-ms:100}") long queuePollMs,
@Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix
@Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix,
@Value("${moyi.gateway.disable-model-concurrency:false}") boolean disableModelConcurrency
) {
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1
// A model concurrency of 0 means "unlimited at the MOYI layer". It
// is useful when measuring the actual upstream/channel limit: the
// provider remains responsible for its own quota while each platform
// API key can still retain an independent limit.
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 0 || defaultKeyConcurrency < 0
|| queueWaitMs < 0 || queuePollMs < 10) {
throw new IllegalArgumentException("Gateway limits must be positive");
}
@@ -69,6 +75,7 @@ public class GatewayGuard {
this.readsPerMinute = readsPerMinute;
this.defaultModelConcurrency = defaultModelConcurrency;
this.defaultKeyConcurrency = defaultKeyConcurrency;
this.modelConcurrencyEnabled = !disableModelConcurrency;
this.queueWaitMs = queueWaitMs;
this.queuePollMs = queuePollMs;
if (redisPrefix == null || !redisPrefix.matches("[A-Za-z0-9:_-]{2,80}:")) {
@@ -77,6 +84,14 @@ public class GatewayGuard {
this.redisPrefix = redisPrefix;
}
/** Compatibility constructor for callers that do not configure the measurement switch. */
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
int defaultModelConcurrency, int defaultKeyConcurrency, long queueWaitMs, long queuePollMs,
String redisPrefix) {
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, defaultKeyConcurrency,
queueWaitMs, queuePollMs, redisPrefix, false);
}
/** Compatibility constructor for isolated unit tests and local callers. */
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
int defaultModelConcurrency, String redisPrefix) {
@@ -99,9 +114,12 @@ public class GatewayGuard {
public int modelConcurrency(Map<String, Object> model) {
Object limit = model.get("maxConcurrency");
return limit instanceof Number number && number.intValue() > 0 ? number.intValue() : defaultModelConcurrency;
return limit instanceof Number number && number.intValue() >= 0 ? number.intValue() : defaultModelConcurrency;
}
/** True when MOYI enforces the model/group concurrency slot. */
public boolean modelConcurrencyEnabled() { return modelConcurrencyEnabled; }
public Slot acquire(Map<String, Object> model) {
return acquire(model, 0);
}
@@ -112,6 +130,7 @@ public class GatewayGuard {
* synchronous while preventing an unbounded servlet-thread backlog.
*/
public Slot acquire(Map<String, Object> model, long requestedWaitMs) {
if (!modelConcurrencyEnabled) return new Slot(null, null);
String bucket = redisPrefix + "gateway:slots:model:" + model.get("id");
return acquireSlot(bucket, modelConcurrency(model), "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs);
}
@@ -137,6 +156,7 @@ public class GatewayGuard {
/** Reads live leases rather than historical usage rows. Expired workers do not count as running. */
public CapacitySnapshot capacitySnapshot(List<String> modelIds) {
if (!modelConcurrencyEnabled) return new CapacitySnapshot(true, Map.of());
if (modelIds.isEmpty()) return new CapacitySnapshot(true, Map.of());
List<String> buckets = new ArrayList<>();
for (String id : modelIds) {
@@ -159,6 +179,12 @@ public class GatewayGuard {
/** Atomically reserves the model and API-Key slots, so queued requests do not hold one scarce slot while waiting for the other. */
public Admission acquire(Map<String, Object> model, UUID keyId, long requestedWaitMs) {
if (!modelConcurrencyEnabled) {
// Measurement mode removes only MOYI's per-model/group guard. The
// independent per-platform-Key limit remains active so one client
// cannot accidentally consume the entire gateway indefinitely.
return new Admission(new Slot(null, null), acquireKey(keyId, requestedWaitMs));
}
String modelBucket = redisPrefix + "gateway:slots:model:" + model.get("id");
String keyBucket = redisPrefix + "gateway:slots:key:" + keyId;
String waitingBucket = redisPrefix + "gateway:waiting:model:" + model.get("id");
@@ -167,9 +193,23 @@ public class GatewayGuard {
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
boolean admitted = false;
try {
int modelLimit = modelConcurrency(model);
// Do not create a Redis model lease when the route (or the
// server default) is explicitly unlimited. This removes the
// MOYI-side upstream-group cap while preserving API-Key slots.
if (modelLimit <= 0) {
Slot keySlot = acquireSlot(keyBucket, defaultKeyConcurrency, "KEY_BUSY", "当前 API Key 的并发已满,已进入排队,请稍后重试", requestedWaitMs);
admitted = true;
return new Admission(null, keySlot);
}
if (defaultKeyConcurrency <= 0) {
Slot modelSlot = acquireSlot(modelBucket, modelLimit, "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs);
admitted = true;
return new Admission(modelSlot, null);
}
while (true) {
long remaining = Math.max(0, java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime()));
long acquired = executePair(modelBucket, keyBucket, waitingBucket, Integer.toString(modelConcurrency(model)), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token, Long.toString(remaining), Long.toString(Math.max(SLOT_TTL_MS, waitMs + 1000)));
long acquired = executePair(modelBucket, keyBucket, waitingBucket, Integer.toString(modelLimit), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token, Long.toString(remaining), Long.toString(Math.max(SLOT_TTL_MS, waitMs + 1000)));
if (acquired == 1) {
admitted = true; // The acquisition script also removed the waiting entry.
return new Admission(modelBucket, keyBucket, token);
@@ -191,6 +231,7 @@ public class GatewayGuard {
}
private Slot acquireSlot(String bucket, int limit, String code, String message, long requestedWaitMs) {
if (limit <= 0) return new Slot(null, null);
String token = UUID.randomUUID().toString();
long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs));
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
@@ -244,15 +285,20 @@ public class GatewayGuard {
private final AtomicBoolean closed = new AtomicBoolean();
private Admission(String modelBucket, String keyBucket, String token) {
this.modelSlot = new Slot(modelBucket, token);
this.keySlot = new Slot(keyBucket, token);
this.modelSlot = modelBucket == null ? null : new Slot(modelBucket, token);
this.keySlot = keyBucket == null ? null : new Slot(keyBucket, token);
}
private Admission(Slot modelSlot, Slot keySlot) {
this.modelSlot = modelSlot;
this.keySlot = keySlot;
}
@Override
public void close() {
if (!closed.compareAndSet(false, true)) return;
modelSlot.close();
keySlot.close();
if (modelSlot != null) modelSlot.close();
if (keySlot != null) keySlot.close();
}
}
@@ -266,6 +312,7 @@ public class GatewayGuard {
@Override
public void close() {
if (!closed.compareAndSet(false, true)) return;
if (key == null || token == null) return;
// A failed release expires automatically and must not replace the upstream response.
releaseQuietly(key, token);
}
@@ -36,6 +36,7 @@ public class AsyncTaskService {
private final int maxQueuedPerModel;
private final int defaultModelConcurrency;
private final int defaultKeyConcurrency;
private final boolean modelConcurrencyEnabled;
private final long providerTimeoutMs;
private final GatewayGuard gatewayGuard;
private final UsageAuditService usage;
@@ -45,7 +46,18 @@ public class AsyncTaskService {
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, gatewayGuard, usage, null, null);
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, true, gatewayGuard, usage, null, null);
}
/** Compatibility constructor for isolated tests using explicit limits. */
public AsyncTaskService(
JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
int maxQueuedPerModel, int defaultModelConcurrency, int defaultKeyConcurrency,
long providerTimeoutMs, GatewayGuard gatewayGuard, UsageAuditService usage,
AsyncProviderClient providerClient, PlatformTransactionManager transactionManager
) {
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, defaultModelConcurrency,
defaultKeyConcurrency, providerTimeoutMs, true, gatewayGuard, usage, providerClient, transactionManager);
}
@org.springframework.beans.factory.annotation.Autowired
@@ -57,12 +69,16 @@ public class AsyncTaskService {
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long providerTimeoutMs,
@Value("${moyi.gateway.disable-model-concurrency:false}") boolean disableModelConcurrency,
GatewayGuard gatewayGuard,
UsageAuditService usage,
AsyncProviderClient providerClient,
PlatformTransactionManager transactionManager
) {
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
// A model concurrency of 0 means no MOYI-side model-group cap. The
// upstream provider still enforces its own quota, while API-Key
// concurrency remains independently configurable.
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 0 || defaultKeyConcurrency < 0) throw new IllegalArgumentException("Task queue and concurrency limits must not be negative");
if (providerTimeoutMs < 1_000) throw new IllegalArgumentException("Provider task timeout must be at least one second");
this.jdbc = jdbc;
this.objectMapper = objectMapper;
@@ -70,6 +86,7 @@ public class AsyncTaskService {
this.maxQueuedPerModel = maxQueuedPerModel;
this.defaultModelConcurrency = defaultModelConcurrency;
this.defaultKeyConcurrency = defaultKeyConcurrency;
this.modelConcurrencyEnabled = !disableModelConcurrency;
this.providerTimeoutMs = providerTimeoutMs;
this.gatewayGuard = gatewayGuard;
this.usage = usage;
@@ -225,17 +242,26 @@ public class AsyncTaskService {
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
AND p.status='active' AND p.deleted_at IS NULL
AND (SELECT COUNT(*) FROM model_tasks running_model
WHERE running_model.model_id=t.model_id AND running_model.status='running')
< COALESCE(m.max_concurrency, ?)
AND (SELECT COUNT(*) FROM model_tasks running_key
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
< ?
AND (
? <= 0
OR COALESCE(m.max_concurrency, ?) <= 0
OR (SELECT COUNT(*) FROM model_tasks running_model
WHERE running_model.model_id=t.model_id AND running_model.status='running')
< COALESCE(m.max_concurrency, ?)
)
AND (
? <= 0
OR (SELECT COUNT(*) FROM model_tasks running_key
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
< ?
)
ORDER BY t.created_at
LIMIT 1
FOR UPDATE OF t SKIP LOCKED
""";
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency, defaultKeyConcurrency);
List<Map<String,Object>> rows = jdbc.queryForList(sql,
modelConcurrencyEnabled ? 1 : 0, defaultModelConcurrency, defaultModelConcurrency,
defaultKeyConcurrency, defaultKeyConcurrency);
if (rows.isEmpty()) return null;
Map<String,Object> row = rows.getFirst();
jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));
@@ -129,6 +129,7 @@ public class UsageAuditService {
"LEFT JOIN usage_events ue ON ue.deleted_at IS NULL AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " +
"GROUP BY day ORDER BY day",
(rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count"))));
boolean modelGuardEnabled = gatewayGuard == null || gatewayGuard.modelConcurrencyEnabled();
int defaultModelLimit = gatewayGuard == null ? 4 : gatewayGuard.defaultModelConcurrency();
List<Map<String, Object>> groups = jdbc.query("""
WITH pending AS (
@@ -138,7 +139,8 @@ public class UsageAuditService {
FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id
)
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
COALESCE(NULLIF(m.max_concurrency, 0), ?) AS max_concurrency,
CASE WHEN m.max_concurrency = 0 THEN 0
ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency,
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued
@@ -155,7 +157,7 @@ public class UsageAuditService {
group.put("providerName", rs.getString("provider_name"));
group.put("mode", rs.getString("call_mode"));
group.put("routeEnabled", rs.getBoolean("route_enabled"));
group.put("limit", rs.getInt("max_concurrency"));
group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0);
group.put("running", rs.getLong("running"));
group.put("queued", rs.getLong("queued"));
return group;
@@ -179,6 +181,7 @@ public class UsageAuditService {
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
"defaultModelConcurrency", defaultModelLimit,
"defaultKeyConcurrency", gatewayGuard == null ? 4 : gatewayGuard.defaultKeyConcurrency(),
"modelConcurrencyGuardEnabled", gatewayGuard == null || gatewayGuard.modelConcurrencyEnabled(),
"asyncQueueLimit", asyncQueueLimit,
"asyncWorkerIntervalMs", asyncWorkerIntervalMs,
"synchronousSnapshotAvailable", snapshot.available()
@@ -43,8 +43,15 @@ moyi:
redis-prefix: ${MOYI_REDIS_KEY_PREFIX:moyi:}
requests-per-minute: ${GATEWAY_REQUESTS_PER_MINUTE:60}
reads-per-minute: ${GATEWAY_READS_PER_MINUTE:300}
default-model-concurrency: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:4}
# 0 disables the MOYI-side model-group cap so channel concurrency can be
# measured directly. Set a positive value only when a provider confirms
# a shared per-model limit.
default-model-concurrency: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:0}
default-key-concurrency: ${GATEWAY_DEFAULT_KEY_CONCURRENCY:4}
# Temporary measurement switch. When true, skip MOYI model/group slots
# while retaining the independent API-Key limit, RPM, billing and the
# provider's own response/limit.
disable-model-concurrency: ${GATEWAY_DISABLE_MODEL_CONCURRENCY:false}
queue-wait-ms: ${GATEWAY_QUEUE_WAIT_MS:30000}
queue-poll-ms: ${GATEWAY_QUEUE_POLL_MS:100}
tasks:
@@ -0,0 +1,14 @@
-- The original relay catalog seeds used max_concurrency = 1 as a temporary
-- safety value. That value is a MOYI-side admission limit; it is not a
-- channel/provider quota and made every group look serial even when the
-- upstream account could process more requests.
--
-- 0 means "no MOYI model-group cap". The gateway still enforces each
-- platform API-Key limit, request-rate limit, wallet/usage accounting, and
-- the provider's own quota/rate limit. Administrators can still set an
-- explicit positive cap for a route when a channel reports one.
UPDATE model_catalog
SET max_concurrency = 0,
updated_at = now()
WHERE deleted_at IS NULL
AND max_concurrency = 1;