fix: remove artificial model group concurrency cap
This commit is contained in:
1 parent
25114ca6b8
commit
ce0a718686
13 files changed
+165
-34
No files matched your search
@@ -199,7 +199,7 @@ public class CatalogService {
|
||||
Integer priority = integerOrNull(input.get("priority"));
|
||||
Integer maxConcurrency = integerOrNull(input.get("maxConcurrency"));
|
||||
if (priority != null && priority < 0) throw new ApiException("INVALID_MODEL_PRIORITY", "模型路由优先级必须大于等于 0");
|
||||
if (maxConcurrency != null && maxConcurrency < 1) throw new ApiException("INVALID_MODEL_CONCURRENCY", "模型并发上限必须大于等于 1");
|
||||
if (maxConcurrency != null && maxConcurrency < 0) throw new ApiException("INVALID_MODEL_CONCURRENCY", "模型并发上限必须大于等于 0;0 表示不设置平台分组上限");
|
||||
jdbc.update(
|
||||
"INSERT INTO model_catalog (id, provider_id, model_key, source_label, public_name, upstream_model, model_type, call_mode, description, input_types, output_types, capabilities, pricing, priority, max_concurrency, logical_model_id, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, CAST(? AS jsonb), CAST(? AS jsonb), CAST(? AS jsonb), CAST(? AS jsonb), ?, ?, ?, ?)",
|
||||
id, providerId, modelKey, sourceLabel, publicName, upstreamModel, type, mode, description,
|
||||
@@ -223,7 +223,7 @@ public class CatalogService {
|
||||
if (findProvider(providerId)==null) throw new ApiException("PROVIDER_NOT_FOUND","渠道不存在",404);
|
||||
if(groupName.length()>160 || upstream.length()>160) throw new ApiException("INVALID_MODEL","分组名称或上游标识过长");
|
||||
Integer priority=integerOrNull(value.get("priority")), concurrency=integerOrNull(value.get("maxConcurrency"));
|
||||
if ((priority!=null && priority<0) || (concurrency!=null && concurrency<1)) throw new ApiException("INVALID_MODEL_LIMIT","优先级不能小于 0,并发上限至少为 1");
|
||||
if ((priority!=null && priority<0) || (concurrency!=null && concurrency<0)) throw new ApiException("INVALID_MODEL_LIMIT","优先级和并发上限不能小于 0;并发填 0 表示不设置平台分组上限");
|
||||
validatePricing(value.get("pricing"));
|
||||
jdbc.update("UPDATE model_catalog SET logical_model_id=?, provider_id=?, source_label=?, upstream_model=?, public_name=?, model_key=?, model_type=?, call_mode=?, description=?, input_types=CAST(? AS jsonb), output_types=CAST(? AS jsonb), pricing=CAST(? AS jsonb), priority=?, max_concurrency=?, status=?, updated_at=now() WHERE id=?",logicalId,providerId,groupName,upstream,definition.get("publicName"),definition.get("modelKey"),definition.get("type"),definition.get("mode"),definition.get("description"),jsonArray(definition.get("inputTypes")),jsonArray(definition.get("outputTypes")),jsonObjectOrNull(value.get("pricing")),priority,concurrency,status(value),id);
|
||||
auth.audit("model_group.updated",admin.id(),new AuthService.RequestMeta(null,null),Map.of("groupId",id,"modelId",logicalId,"providerId",providerId));
|
||||
|
||||
@@ -38,6 +38,7 @@ public class GatewayGuard {
|
||||
private final int readsPerMinute;
|
||||
private final int defaultModelConcurrency;
|
||||
private final int defaultKeyConcurrency;
|
||||
private final boolean modelConcurrencyEnabled;
|
||||
private final long queueWaitMs;
|
||||
private final long queuePollMs;
|
||||
private final String redisPrefix;
|
||||
@@ -58,9 +59,14 @@ public class GatewayGuard {
|
||||
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
|
||||
@Value("${moyi.gateway.queue-wait-ms:30000}") long queueWaitMs,
|
||||
@Value("${moyi.gateway.queue-poll-ms:100}") long queuePollMs,
|
||||
@Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix
|
||||
@Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix,
|
||||
@Value("${moyi.gateway.disable-model-concurrency:false}") boolean disableModelConcurrency
|
||||
) {
|
||||
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1
|
||||
// A model concurrency of 0 means "unlimited at the MOYI layer". It
|
||||
// is useful when measuring the actual upstream/channel limit: the
|
||||
// provider remains responsible for its own quota while each platform
|
||||
// API key can still retain an independent limit.
|
||||
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 0 || defaultKeyConcurrency < 0
|
||||
|| queueWaitMs < 0 || queuePollMs < 10) {
|
||||
throw new IllegalArgumentException("Gateway limits must be positive");
|
||||
}
|
||||
@@ -69,6 +75,7 @@ public class GatewayGuard {
|
||||
this.readsPerMinute = readsPerMinute;
|
||||
this.defaultModelConcurrency = defaultModelConcurrency;
|
||||
this.defaultKeyConcurrency = defaultKeyConcurrency;
|
||||
this.modelConcurrencyEnabled = !disableModelConcurrency;
|
||||
this.queueWaitMs = queueWaitMs;
|
||||
this.queuePollMs = queuePollMs;
|
||||
if (redisPrefix == null || !redisPrefix.matches("[A-Za-z0-9:_-]{2,80}:")) {
|
||||
@@ -77,6 +84,14 @@ public class GatewayGuard {
|
||||
this.redisPrefix = redisPrefix;
|
||||
}
|
||||
|
||||
/** Compatibility constructor for callers that do not configure the measurement switch. */
|
||||
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
|
||||
int defaultModelConcurrency, int defaultKeyConcurrency, long queueWaitMs, long queuePollMs,
|
||||
String redisPrefix) {
|
||||
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, defaultKeyConcurrency,
|
||||
queueWaitMs, queuePollMs, redisPrefix, false);
|
||||
}
|
||||
|
||||
/** Compatibility constructor for isolated unit tests and local callers. */
|
||||
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
|
||||
int defaultModelConcurrency, String redisPrefix) {
|
||||
@@ -99,9 +114,12 @@ public class GatewayGuard {
|
||||
|
||||
public int modelConcurrency(Map<String, Object> model) {
|
||||
Object limit = model.get("maxConcurrency");
|
||||
return limit instanceof Number number && number.intValue() > 0 ? number.intValue() : defaultModelConcurrency;
|
||||
return limit instanceof Number number && number.intValue() >= 0 ? number.intValue() : defaultModelConcurrency;
|
||||
}
|
||||
|
||||
/** True when MOYI enforces the model/group concurrency slot. */
|
||||
public boolean modelConcurrencyEnabled() { return modelConcurrencyEnabled; }
|
||||
|
||||
public Slot acquire(Map<String, Object> model) {
|
||||
return acquire(model, 0);
|
||||
}
|
||||
@@ -112,6 +130,7 @@ public class GatewayGuard {
|
||||
* synchronous while preventing an unbounded servlet-thread backlog.
|
||||
*/
|
||||
public Slot acquire(Map<String, Object> model, long requestedWaitMs) {
|
||||
if (!modelConcurrencyEnabled) return new Slot(null, null);
|
||||
String bucket = redisPrefix + "gateway:slots:model:" + model.get("id");
|
||||
return acquireSlot(bucket, modelConcurrency(model), "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs);
|
||||
}
|
||||
@@ -137,6 +156,7 @@ public class GatewayGuard {
|
||||
|
||||
/** Reads live leases rather than historical usage rows. Expired workers do not count as running. */
|
||||
public CapacitySnapshot capacitySnapshot(List<String> modelIds) {
|
||||
if (!modelConcurrencyEnabled) return new CapacitySnapshot(true, Map.of());
|
||||
if (modelIds.isEmpty()) return new CapacitySnapshot(true, Map.of());
|
||||
List<String> buckets = new ArrayList<>();
|
||||
for (String id : modelIds) {
|
||||
@@ -159,6 +179,12 @@ public class GatewayGuard {
|
||||
|
||||
/** Atomically reserves the model and API-Key slots, so queued requests do not hold one scarce slot while waiting for the other. */
|
||||
public Admission acquire(Map<String, Object> model, UUID keyId, long requestedWaitMs) {
|
||||
if (!modelConcurrencyEnabled) {
|
||||
// Measurement mode removes only MOYI's per-model/group guard. The
|
||||
// independent per-platform-Key limit remains active so one client
|
||||
// cannot accidentally consume the entire gateway indefinitely.
|
||||
return new Admission(new Slot(null, null), acquireKey(keyId, requestedWaitMs));
|
||||
}
|
||||
String modelBucket = redisPrefix + "gateway:slots:model:" + model.get("id");
|
||||
String keyBucket = redisPrefix + "gateway:slots:key:" + keyId;
|
||||
String waitingBucket = redisPrefix + "gateway:waiting:model:" + model.get("id");
|
||||
@@ -167,9 +193,23 @@ public class GatewayGuard {
|
||||
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
|
||||
boolean admitted = false;
|
||||
try {
|
||||
int modelLimit = modelConcurrency(model);
|
||||
// Do not create a Redis model lease when the route (or the
|
||||
// server default) is explicitly unlimited. This removes the
|
||||
// MOYI-side upstream-group cap while preserving API-Key slots.
|
||||
if (modelLimit <= 0) {
|
||||
Slot keySlot = acquireSlot(keyBucket, defaultKeyConcurrency, "KEY_BUSY", "当前 API Key 的并发已满,已进入排队,请稍后重试", requestedWaitMs);
|
||||
admitted = true;
|
||||
return new Admission(null, keySlot);
|
||||
}
|
||||
if (defaultKeyConcurrency <= 0) {
|
||||
Slot modelSlot = acquireSlot(modelBucket, modelLimit, "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs);
|
||||
admitted = true;
|
||||
return new Admission(modelSlot, null);
|
||||
}
|
||||
while (true) {
|
||||
long remaining = Math.max(0, java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime()));
|
||||
long acquired = executePair(modelBucket, keyBucket, waitingBucket, Integer.toString(modelConcurrency(model)), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token, Long.toString(remaining), Long.toString(Math.max(SLOT_TTL_MS, waitMs + 1000)));
|
||||
long acquired = executePair(modelBucket, keyBucket, waitingBucket, Integer.toString(modelLimit), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token, Long.toString(remaining), Long.toString(Math.max(SLOT_TTL_MS, waitMs + 1000)));
|
||||
if (acquired == 1) {
|
||||
admitted = true; // The acquisition script also removed the waiting entry.
|
||||
return new Admission(modelBucket, keyBucket, token);
|
||||
@@ -191,6 +231,7 @@ public class GatewayGuard {
|
||||
}
|
||||
|
||||
private Slot acquireSlot(String bucket, int limit, String code, String message, long requestedWaitMs) {
|
||||
if (limit <= 0) return new Slot(null, null);
|
||||
String token = UUID.randomUUID().toString();
|
||||
long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs));
|
||||
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
|
||||
@@ -244,15 +285,20 @@ public class GatewayGuard {
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
private Admission(String modelBucket, String keyBucket, String token) {
|
||||
this.modelSlot = new Slot(modelBucket, token);
|
||||
this.keySlot = new Slot(keyBucket, token);
|
||||
this.modelSlot = modelBucket == null ? null : new Slot(modelBucket, token);
|
||||
this.keySlot = keyBucket == null ? null : new Slot(keyBucket, token);
|
||||
}
|
||||
|
||||
private Admission(Slot modelSlot, Slot keySlot) {
|
||||
this.modelSlot = modelSlot;
|
||||
this.keySlot = keySlot;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
modelSlot.close();
|
||||
keySlot.close();
|
||||
if (modelSlot != null) modelSlot.close();
|
||||
if (keySlot != null) keySlot.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,6 +312,7 @@ public class GatewayGuard {
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
if (key == null || token == null) return;
|
||||
// A failed release expires automatically and must not replace the upstream response.
|
||||
releaseQuietly(key, token);
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ public class AsyncTaskService {
|
||||
private final int maxQueuedPerModel;
|
||||
private final int defaultModelConcurrency;
|
||||
private final int defaultKeyConcurrency;
|
||||
private final boolean modelConcurrencyEnabled;
|
||||
private final long providerTimeoutMs;
|
||||
private final GatewayGuard gatewayGuard;
|
||||
private final UsageAuditService usage;
|
||||
@@ -45,7 +46,18 @@ public class AsyncTaskService {
|
||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, gatewayGuard, usage, null, null);
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, true, gatewayGuard, usage, null, null);
|
||||
}
|
||||
|
||||
/** Compatibility constructor for isolated tests using explicit limits. */
|
||||
public AsyncTaskService(
|
||||
JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
int maxQueuedPerModel, int defaultModelConcurrency, int defaultKeyConcurrency,
|
||||
long providerTimeoutMs, GatewayGuard gatewayGuard, UsageAuditService usage,
|
||||
AsyncProviderClient providerClient, PlatformTransactionManager transactionManager
|
||||
) {
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, defaultModelConcurrency,
|
||||
defaultKeyConcurrency, providerTimeoutMs, true, gatewayGuard, usage, providerClient, transactionManager);
|
||||
}
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
@@ -57,12 +69,16 @@ public class AsyncTaskService {
|
||||
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
|
||||
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
|
||||
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long providerTimeoutMs,
|
||||
@Value("${moyi.gateway.disable-model-concurrency:false}") boolean disableModelConcurrency,
|
||||
GatewayGuard gatewayGuard,
|
||||
UsageAuditService usage,
|
||||
AsyncProviderClient providerClient,
|
||||
PlatformTransactionManager transactionManager
|
||||
) {
|
||||
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
|
||||
// A model concurrency of 0 means no MOYI-side model-group cap. The
|
||||
// upstream provider still enforces its own quota, while API-Key
|
||||
// concurrency remains independently configurable.
|
||||
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 0 || defaultKeyConcurrency < 0) throw new IllegalArgumentException("Task queue and concurrency limits must not be negative");
|
||||
if (providerTimeoutMs < 1_000) throw new IllegalArgumentException("Provider task timeout must be at least one second");
|
||||
this.jdbc = jdbc;
|
||||
this.objectMapper = objectMapper;
|
||||
@@ -70,6 +86,7 @@ public class AsyncTaskService {
|
||||
this.maxQueuedPerModel = maxQueuedPerModel;
|
||||
this.defaultModelConcurrency = defaultModelConcurrency;
|
||||
this.defaultKeyConcurrency = defaultKeyConcurrency;
|
||||
this.modelConcurrencyEnabled = !disableModelConcurrency;
|
||||
this.providerTimeoutMs = providerTimeoutMs;
|
||||
this.gatewayGuard = gatewayGuard;
|
||||
this.usage = usage;
|
||||
@@ -225,17 +242,26 @@ public class AsyncTaskService {
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND (SELECT COUNT(*) FROM model_tasks running_model
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
AND (SELECT COUNT(*) FROM model_tasks running_key
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
||||
< ?
|
||||
AND (
|
||||
? <= 0
|
||||
OR COALESCE(m.max_concurrency, ?) <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_model
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
)
|
||||
AND (
|
||||
? <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_key
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
||||
< ?
|
||||
)
|
||||
ORDER BY t.created_at
|
||||
LIMIT 1
|
||||
FOR UPDATE OF t SKIP LOCKED
|
||||
""";
|
||||
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency, defaultKeyConcurrency);
|
||||
List<Map<String,Object>> rows = jdbc.queryForList(sql,
|
||||
modelConcurrencyEnabled ? 1 : 0, defaultModelConcurrency, defaultModelConcurrency,
|
||||
defaultKeyConcurrency, defaultKeyConcurrency);
|
||||
if (rows.isEmpty()) return null;
|
||||
Map<String,Object> row = rows.getFirst();
|
||||
jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));
|
||||
|
||||
@@ -129,6 +129,7 @@ public class UsageAuditService {
|
||||
"LEFT JOIN usage_events ue ON ue.deleted_at IS NULL AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " +
|
||||
"GROUP BY day ORDER BY day",
|
||||
(rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count"))));
|
||||
boolean modelGuardEnabled = gatewayGuard == null || gatewayGuard.modelConcurrencyEnabled();
|
||||
int defaultModelLimit = gatewayGuard == null ? 4 : gatewayGuard.defaultModelConcurrency();
|
||||
List<Map<String, Object>> groups = jdbc.query("""
|
||||
WITH pending AS (
|
||||
@@ -138,7 +139,8 @@ public class UsageAuditService {
|
||||
FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id
|
||||
)
|
||||
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
|
||||
COALESCE(NULLIF(m.max_concurrency, 0), ?) AS max_concurrency,
|
||||
CASE WHEN m.max_concurrency = 0 THEN 0
|
||||
ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency,
|
||||
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
|
||||
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued
|
||||
@@ -155,7 +157,7 @@ public class UsageAuditService {
|
||||
group.put("providerName", rs.getString("provider_name"));
|
||||
group.put("mode", rs.getString("call_mode"));
|
||||
group.put("routeEnabled", rs.getBoolean("route_enabled"));
|
||||
group.put("limit", rs.getInt("max_concurrency"));
|
||||
group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0);
|
||||
group.put("running", rs.getLong("running"));
|
||||
group.put("queued", rs.getLong("queued"));
|
||||
return group;
|
||||
@@ -179,6 +181,7 @@ public class UsageAuditService {
|
||||
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
|
||||
"defaultModelConcurrency", defaultModelLimit,
|
||||
"defaultKeyConcurrency", gatewayGuard == null ? 4 : gatewayGuard.defaultKeyConcurrency(),
|
||||
"modelConcurrencyGuardEnabled", gatewayGuard == null || gatewayGuard.modelConcurrencyEnabled(),
|
||||
"asyncQueueLimit", asyncQueueLimit,
|
||||
"asyncWorkerIntervalMs", asyncWorkerIntervalMs,
|
||||
"synchronousSnapshotAvailable", snapshot.available()
|
||||
|
||||
Reference in new issue
Block a user