From ce0a7186860efe384c4914423fd7a1bfc9c03cff Mon Sep 17 00:00:00 2001 From: WangLX <519407855@qq.com> Date: Tue, 29 Sep 2026 12:46:10 +0800 Subject: [PATCH] fix: remove artificial model group concurrency cap --- .../controlplane/catalog/CatalogService.java | 4 +- .../controlplane/gateway/GatewayGuard.java | 63 ++++++++++++++++--- .../controlplane/task/AsyncTaskService.java | 44 ++++++++++--- .../controlplane/usage/UsageAuditService.java | 7 ++- .../src/main/resources/application.yml | 9 ++- .../V32__remove_moyi_model_group_cap.sql | 14 +++++ .../gateway/GatewayDeploymentTest.java | 26 ++++++++ docs/09-java-control-plane-node.md | 2 +- docs/17-catalog-and-key-selection.md | 4 +- infra/moyi/docker-compose.cloud.yml | 7 ++- prototype/src/components/CatalogAdmin.vue | 10 +-- prototype/src/services/catalog.ts | 1 + prototype/src/views/AdminConsole.vue | 8 ++- 13 files changed, 165 insertions(+), 34 deletions(-) create mode 100644 control-plane-java/src/main/resources/db/migration/V32__remove_moyi_model_group_cap.sql diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java index d732e01..6c12880 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java @@ -199,7 +199,7 @@ public class CatalogService { Integer priority = integerOrNull(input.get("priority")); Integer maxConcurrency = integerOrNull(input.get("maxConcurrency")); if (priority != null && priority < 0) throw new ApiException("INVALID_MODEL_PRIORITY", "模型路由优先级必须大于等于 0"); - if (maxConcurrency != null && maxConcurrency < 1) throw new ApiException("INVALID_MODEL_CONCURRENCY", "模型并发上限必须大于等于 1"); + if (maxConcurrency != null && maxConcurrency < 0) throw new ApiException("INVALID_MODEL_CONCURRENCY", "模型并发上限必须大于等于 0;0 表示不设置平台分组上限"); jdbc.update( "INSERT INTO model_catalog (id, provider_id, model_key, source_label, public_name, upstream_model, model_type, call_mode, description, input_types, output_types, capabilities, pricing, priority, max_concurrency, logical_model_id, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, CAST(? AS jsonb), CAST(? AS jsonb), CAST(? AS jsonb), CAST(? AS jsonb), ?, ?, ?, ?)", id, providerId, modelKey, sourceLabel, publicName, upstreamModel, type, mode, description, @@ -223,7 +223,7 @@ public class CatalogService { if (findProvider(providerId)==null) throw new ApiException("PROVIDER_NOT_FOUND","渠道不存在",404); if(groupName.length()>160 || upstream.length()>160) throw new ApiException("INVALID_MODEL","分组名称或上游标识过长"); Integer priority=integerOrNull(value.get("priority")), concurrency=integerOrNull(value.get("maxConcurrency")); - if ((priority!=null && priority<0) || (concurrency!=null && concurrency<1)) throw new ApiException("INVALID_MODEL_LIMIT","优先级不能小于 0,并发上限至少为 1"); + if ((priority!=null && priority<0) || (concurrency!=null && concurrency<0)) throw new ApiException("INVALID_MODEL_LIMIT","优先级和并发上限不能小于 0;并发填 0 表示不设置平台分组上限"); validatePricing(value.get("pricing")); jdbc.update("UPDATE model_catalog SET logical_model_id=?, provider_id=?, source_label=?, upstream_model=?, public_name=?, model_key=?, model_type=?, call_mode=?, description=?, input_types=CAST(? AS jsonb), output_types=CAST(? AS jsonb), pricing=CAST(? AS jsonb), priority=?, max_concurrency=?, status=?, updated_at=now() WHERE id=?",logicalId,providerId,groupName,upstream,definition.get("publicName"),definition.get("modelKey"),definition.get("type"),definition.get("mode"),definition.get("description"),jsonArray(definition.get("inputTypes")),jsonArray(definition.get("outputTypes")),jsonObjectOrNull(value.get("pricing")),priority,concurrency,status(value),id); auth.audit("model_group.updated",admin.id(),new AuthService.RequestMeta(null,null),Map.of("groupId",id,"modelId",logicalId,"providerId",providerId)); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java b/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java index ac4ef94..48ff5de 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java @@ -38,6 +38,7 @@ public class GatewayGuard { private final int readsPerMinute; private final int defaultModelConcurrency; private final int defaultKeyConcurrency; + private final boolean modelConcurrencyEnabled; private final long queueWaitMs; private final long queuePollMs; private final String redisPrefix; @@ -58,9 +59,14 @@ public class GatewayGuard { @Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency, @Value("${moyi.gateway.queue-wait-ms:30000}") long queueWaitMs, @Value("${moyi.gateway.queue-poll-ms:100}") long queuePollMs, - @Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix + @Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix, + @Value("${moyi.gateway.disable-model-concurrency:false}") boolean disableModelConcurrency ) { - if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1 + // A model concurrency of 0 means "unlimited at the MOYI layer". It + // is useful when measuring the actual upstream/channel limit: the + // provider remains responsible for its own quota while each platform + // API key can still retain an independent limit. + if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 0 || defaultKeyConcurrency < 0 || queueWaitMs < 0 || queuePollMs < 10) { throw new IllegalArgumentException("Gateway limits must be positive"); } @@ -69,6 +75,7 @@ public class GatewayGuard { this.readsPerMinute = readsPerMinute; this.defaultModelConcurrency = defaultModelConcurrency; this.defaultKeyConcurrency = defaultKeyConcurrency; + this.modelConcurrencyEnabled = !disableModelConcurrency; this.queueWaitMs = queueWaitMs; this.queuePollMs = queuePollMs; if (redisPrefix == null || !redisPrefix.matches("[A-Za-z0-9:_-]{2,80}:")) { @@ -77,6 +84,14 @@ public class GatewayGuard { this.redisPrefix = redisPrefix; } + /** Compatibility constructor for callers that do not configure the measurement switch. */ + public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute, + int defaultModelConcurrency, int defaultKeyConcurrency, long queueWaitMs, long queuePollMs, + String redisPrefix) { + this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, defaultKeyConcurrency, + queueWaitMs, queuePollMs, redisPrefix, false); + } + /** Compatibility constructor for isolated unit tests and local callers. */ public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute, int defaultModelConcurrency, String redisPrefix) { @@ -99,9 +114,12 @@ public class GatewayGuard { public int modelConcurrency(Map model) { Object limit = model.get("maxConcurrency"); - return limit instanceof Number number && number.intValue() > 0 ? number.intValue() : defaultModelConcurrency; + return limit instanceof Number number && number.intValue() >= 0 ? number.intValue() : defaultModelConcurrency; } + /** True when MOYI enforces the model/group concurrency slot. */ + public boolean modelConcurrencyEnabled() { return modelConcurrencyEnabled; } + public Slot acquire(Map model) { return acquire(model, 0); } @@ -112,6 +130,7 @@ public class GatewayGuard { * synchronous while preventing an unbounded servlet-thread backlog. */ public Slot acquire(Map model, long requestedWaitMs) { + if (!modelConcurrencyEnabled) return new Slot(null, null); String bucket = redisPrefix + "gateway:slots:model:" + model.get("id"); return acquireSlot(bucket, modelConcurrency(model), "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs); } @@ -137,6 +156,7 @@ public class GatewayGuard { /** Reads live leases rather than historical usage rows. Expired workers do not count as running. */ public CapacitySnapshot capacitySnapshot(List modelIds) { + if (!modelConcurrencyEnabled) return new CapacitySnapshot(true, Map.of()); if (modelIds.isEmpty()) return new CapacitySnapshot(true, Map.of()); List buckets = new ArrayList<>(); for (String id : modelIds) { @@ -159,6 +179,12 @@ public class GatewayGuard { /** Atomically reserves the model and API-Key slots, so queued requests do not hold one scarce slot while waiting for the other. */ public Admission acquire(Map model, UUID keyId, long requestedWaitMs) { + if (!modelConcurrencyEnabled) { + // Measurement mode removes only MOYI's per-model/group guard. The + // independent per-platform-Key limit remains active so one client + // cannot accidentally consume the entire gateway indefinitely. + return new Admission(new Slot(null, null), acquireKey(keyId, requestedWaitMs)); + } String modelBucket = redisPrefix + "gateway:slots:model:" + model.get("id"); String keyBucket = redisPrefix + "gateway:slots:key:" + keyId; String waitingBucket = redisPrefix + "gateway:waiting:model:" + model.get("id"); @@ -167,9 +193,23 @@ public class GatewayGuard { long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs); boolean admitted = false; try { + int modelLimit = modelConcurrency(model); + // Do not create a Redis model lease when the route (or the + // server default) is explicitly unlimited. This removes the + // MOYI-side upstream-group cap while preserving API-Key slots. + if (modelLimit <= 0) { + Slot keySlot = acquireSlot(keyBucket, defaultKeyConcurrency, "KEY_BUSY", "当前 API Key 的并发已满,已进入排队,请稍后重试", requestedWaitMs); + admitted = true; + return new Admission(null, keySlot); + } + if (defaultKeyConcurrency <= 0) { + Slot modelSlot = acquireSlot(modelBucket, modelLimit, "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs); + admitted = true; + return new Admission(modelSlot, null); + } while (true) { long remaining = Math.max(0, java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime())); - long acquired = executePair(modelBucket, keyBucket, waitingBucket, Integer.toString(modelConcurrency(model)), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token, Long.toString(remaining), Long.toString(Math.max(SLOT_TTL_MS, waitMs + 1000))); + long acquired = executePair(modelBucket, keyBucket, waitingBucket, Integer.toString(modelLimit), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token, Long.toString(remaining), Long.toString(Math.max(SLOT_TTL_MS, waitMs + 1000))); if (acquired == 1) { admitted = true; // The acquisition script also removed the waiting entry. return new Admission(modelBucket, keyBucket, token); @@ -191,6 +231,7 @@ public class GatewayGuard { } private Slot acquireSlot(String bucket, int limit, String code, String message, long requestedWaitMs) { + if (limit <= 0) return new Slot(null, null); String token = UUID.randomUUID().toString(); long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs)); long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs); @@ -244,15 +285,20 @@ public class GatewayGuard { private final AtomicBoolean closed = new AtomicBoolean(); private Admission(String modelBucket, String keyBucket, String token) { - this.modelSlot = new Slot(modelBucket, token); - this.keySlot = new Slot(keyBucket, token); + this.modelSlot = modelBucket == null ? null : new Slot(modelBucket, token); + this.keySlot = keyBucket == null ? null : new Slot(keyBucket, token); + } + + private Admission(Slot modelSlot, Slot keySlot) { + this.modelSlot = modelSlot; + this.keySlot = keySlot; } @Override public void close() { if (!closed.compareAndSet(false, true)) return; - modelSlot.close(); - keySlot.close(); + if (modelSlot != null) modelSlot.close(); + if (keySlot != null) keySlot.close(); } } @@ -266,6 +312,7 @@ public class GatewayGuard { @Override public void close() { if (!closed.compareAndSet(false, true)) return; + if (key == null || token == null) return; // A failed release expires automatically and must not replace the upstream response. releaseQuietly(key, token); } diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java index 74c8947..3aeca98 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java @@ -36,6 +36,7 @@ public class AsyncTaskService { private final int maxQueuedPerModel; private final int defaultModelConcurrency; private final int defaultKeyConcurrency; + private final boolean modelConcurrencyEnabled; private final long providerTimeoutMs; private final GatewayGuard gatewayGuard; private final UsageAuditService usage; @@ -45,7 +46,18 @@ public class AsyncTaskService { /** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */ public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService, int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) { - this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, gatewayGuard, usage, null, null); + this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, true, gatewayGuard, usage, null, null); + } + + /** Compatibility constructor for isolated tests using explicit limits. */ + public AsyncTaskService( + JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService, + int maxQueuedPerModel, int defaultModelConcurrency, int defaultKeyConcurrency, + long providerTimeoutMs, GatewayGuard gatewayGuard, UsageAuditService usage, + AsyncProviderClient providerClient, PlatformTransactionManager transactionManager + ) { + this(jdbc, objectMapper, catalogService, maxQueuedPerModel, defaultModelConcurrency, + defaultKeyConcurrency, providerTimeoutMs, true, gatewayGuard, usage, providerClient, transactionManager); } @org.springframework.beans.factory.annotation.Autowired @@ -57,12 +69,16 @@ public class AsyncTaskService { @Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency, @Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency, @Value("${moyi.tasks.provider-timeout-ms:3600000}") long providerTimeoutMs, + @Value("${moyi.gateway.disable-model-concurrency:false}") boolean disableModelConcurrency, GatewayGuard gatewayGuard, UsageAuditService usage, AsyncProviderClient providerClient, PlatformTransactionManager transactionManager ) { - if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive"); + // A model concurrency of 0 means no MOYI-side model-group cap. The + // upstream provider still enforces its own quota, while API-Key + // concurrency remains independently configurable. + if (maxQueuedPerModel < 1 || defaultModelConcurrency < 0 || defaultKeyConcurrency < 0) throw new IllegalArgumentException("Task queue and concurrency limits must not be negative"); if (providerTimeoutMs < 1_000) throw new IllegalArgumentException("Provider task timeout must be at least one second"); this.jdbc = jdbc; this.objectMapper = objectMapper; @@ -70,6 +86,7 @@ public class AsyncTaskService { this.maxQueuedPerModel = maxQueuedPerModel; this.defaultModelConcurrency = defaultModelConcurrency; this.defaultKeyConcurrency = defaultKeyConcurrency; + this.modelConcurrencyEnabled = !disableModelConcurrency; this.providerTimeoutMs = providerTimeoutMs; this.gatewayGuard = gatewayGuard; this.usage = usage; @@ -225,17 +242,26 @@ public class AsyncTaskService { AND t.provider_task_id IS NULL AND m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL - AND (SELECT COUNT(*) FROM model_tasks running_model - WHERE running_model.model_id=t.model_id AND running_model.status='running') - < COALESCE(m.max_concurrency, ?) - AND (SELECT COUNT(*) FROM model_tasks running_key - WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running') - < ? + AND ( + ? <= 0 + OR COALESCE(m.max_concurrency, ?) <= 0 + OR (SELECT COUNT(*) FROM model_tasks running_model + WHERE running_model.model_id=t.model_id AND running_model.status='running') + < COALESCE(m.max_concurrency, ?) + ) + AND ( + ? <= 0 + OR (SELECT COUNT(*) FROM model_tasks running_key + WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running') + < ? + ) ORDER BY t.created_at LIMIT 1 FOR UPDATE OF t SKIP LOCKED """; - List> rows = jdbc.queryForList(sql, defaultModelConcurrency, defaultKeyConcurrency); + List> rows = jdbc.queryForList(sql, + modelConcurrencyEnabled ? 1 : 0, defaultModelConcurrency, defaultModelConcurrency, + defaultKeyConcurrency, defaultKeyConcurrency); if (rows.isEmpty()) return null; Map row = rows.getFirst(); jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id")); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java b/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java index 68a01fc..e09ff9f 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java @@ -129,6 +129,7 @@ public class UsageAuditService { "LEFT JOIN usage_events ue ON ue.deleted_at IS NULL AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " + "GROUP BY day ORDER BY day", (rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count")))); + boolean modelGuardEnabled = gatewayGuard == null || gatewayGuard.modelConcurrencyEnabled(); int defaultModelLimit = gatewayGuard == null ? 4 : gatewayGuard.defaultModelConcurrency(); List> groups = jdbc.query(""" WITH pending AS ( @@ -138,7 +139,8 @@ public class UsageAuditService { FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id ) SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode, - COALESCE(NULLIF(m.max_concurrency, 0), ?) AS max_concurrency, + CASE WHEN m.max_concurrency = 0 THEN 0 + ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency, (m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled, COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued @@ -155,7 +157,7 @@ public class UsageAuditService { group.put("providerName", rs.getString("provider_name")); group.put("mode", rs.getString("call_mode")); group.put("routeEnabled", rs.getBoolean("route_enabled")); - group.put("limit", rs.getInt("max_concurrency")); + group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0); group.put("running", rs.getLong("running")); group.put("queued", rs.getLong("queued")); return group; @@ -179,6 +181,7 @@ public class UsageAuditService { "queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(), "defaultModelConcurrency", defaultModelLimit, "defaultKeyConcurrency", gatewayGuard == null ? 4 : gatewayGuard.defaultKeyConcurrency(), + "modelConcurrencyGuardEnabled", gatewayGuard == null || gatewayGuard.modelConcurrencyEnabled(), "asyncQueueLimit", asyncQueueLimit, "asyncWorkerIntervalMs", asyncWorkerIntervalMs, "synchronousSnapshotAvailable", snapshot.available() diff --git a/control-plane-java/src/main/resources/application.yml b/control-plane-java/src/main/resources/application.yml index d7f4d36..867db6f 100644 --- a/control-plane-java/src/main/resources/application.yml +++ b/control-plane-java/src/main/resources/application.yml @@ -43,8 +43,15 @@ moyi: redis-prefix: ${MOYI_REDIS_KEY_PREFIX:moyi:} requests-per-minute: ${GATEWAY_REQUESTS_PER_MINUTE:60} reads-per-minute: ${GATEWAY_READS_PER_MINUTE:300} - default-model-concurrency: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:4} + # 0 disables the MOYI-side model-group cap so channel concurrency can be + # measured directly. Set a positive value only when a provider confirms + # a shared per-model limit. + default-model-concurrency: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:0} default-key-concurrency: ${GATEWAY_DEFAULT_KEY_CONCURRENCY:4} + # Temporary measurement switch. When true, skip MOYI model/group slots + # while retaining the independent API-Key limit, RPM, billing and the + # provider's own response/limit. + disable-model-concurrency: ${GATEWAY_DISABLE_MODEL_CONCURRENCY:false} queue-wait-ms: ${GATEWAY_QUEUE_WAIT_MS:30000} queue-poll-ms: ${GATEWAY_QUEUE_POLL_MS:100} tasks: diff --git a/control-plane-java/src/main/resources/db/migration/V32__remove_moyi_model_group_cap.sql b/control-plane-java/src/main/resources/db/migration/V32__remove_moyi_model_group_cap.sql new file mode 100644 index 0000000..100dbf1 --- /dev/null +++ b/control-plane-java/src/main/resources/db/migration/V32__remove_moyi_model_group_cap.sql @@ -0,0 +1,14 @@ +-- The original relay catalog seeds used max_concurrency = 1 as a temporary +-- safety value. That value is a MOYI-side admission limit; it is not a +-- channel/provider quota and made every group look serial even when the +-- upstream account could process more requests. +-- +-- 0 means "no MOYI model-group cap". The gateway still enforces each +-- platform API-Key limit, request-rate limit, wallet/usage accounting, and +-- the provider's own quota/rate limit. Administrators can still set an +-- explicit positive cap for a route when a channel reports one. +UPDATE model_catalog + SET max_concurrency = 0, + updated_at = now() + WHERE deleted_at IS NULL + AND max_concurrency = 1; diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java index 53f38a3..985176b 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java @@ -66,6 +66,32 @@ class GatewayDeploymentTest { verify(redis, atLeastOnce()).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:slots:key:" + keyId)), any(Object[].class)); } + @Test + void zeroModelConcurrencyUsesOnlyTheIndependentKeyLease() { + StringRedisTemplate redis = mock(StringRedisTemplate.class); + doReturn(1L).when(redis).execute(any(RedisScript.class), anyList(), any(Object[].class)); + UUID keyId = UUID.randomUUID(); + Map unlimited = Map.of("id", "model-unlimited", "maxConcurrency", 0); + try (GatewayGuard.Admission ignored = new GatewayGuard(redis, 60, 300, 4, 2, 15, 10, "moyi:test:").acquire(unlimited, keyId, 15)) { + // The route is deliberately unlimited at the MOYI model layer. + } + verify(redis, never()).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:slots:model:model-unlimited")), any(Object[].class)); + verify(redis, atLeastOnce()).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:slots:key:" + keyId)), any(Object[].class)); + } + + @Test + void disabledModelGuardDoesNotCreateModelLease() { + StringRedisTemplate redis = mock(StringRedisTemplate.class); + doReturn(1L).when(redis).execute(any(RedisScript.class), anyList(), any(Object[].class)); + UUID keyId = UUID.randomUUID(); + Map model = Map.of("id", "model-measurement", "maxConcurrency", 1); + try (GatewayGuard.Admission ignored = new GatewayGuard(redis, 60, 300, 4, 2, 15, 10, "moyi:test:", true).acquire(model, keyId, 15)) { + // Measurement mode bypasses only the MOYI model-group lease. + } + verify(redis, never()).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:slots:model:model-measurement")), any(Object[].class)); + verify(redis, atLeastOnce()).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:slots:key:" + keyId)), any(Object[].class)); + } + @Test void modelsExposeReadableIdAndKeepRouteIdForDiagnostics() { CatalogService catalog = mock(CatalogService.class); diff --git a/docs/09-java-control-plane-node.md b/docs/09-java-control-plane-node.md index 64976eb..301758c 100644 --- a/docs/09-java-control-plane-node.md +++ b/docs/09-java-control-plane-node.md @@ -77,7 +77,7 @@ Invoke-WebRequest http://localhost:8080/health 本节点完成的是“控制面基础闭环 + 同步供应商转发 + Redis 网关保护 + 持久化任务队列”,不是生产网关完成版: -- 数据已写入 PostgreSQL;Redis 已用于 API Key 频率限制、API Key 并发租约、上游分组并发租约和 Redis Lua 原子操作。默认同步调用每 Key 每分钟 60 次(`GATEWAY_REQUESTS_PER_MINUTE`)、每 Key 同时 4 个请求(`GATEWAY_DEFAULT_KEY_CONCURRENCY`);分组并发由管理端模型分组的 `maxConcurrency` 配置,未配置时使用 `GATEWAY_DEFAULT_MODEL_CONCURRENCY`。Redis 不可用时网关返回 503,不会绕过保护直接请求上游; +- 数据已写入 PostgreSQL;Redis 已用于 API Key 频率限制、API Key 并发租约、上游分组并发租约和 Redis Lua 原子操作。默认同步调用每 Key 每分钟 60 次(`GATEWAY_REQUESTS_PER_MINUTE`)、每 Key 同时 4 个请求(`GATEWAY_DEFAULT_KEY_CONCURRENCY`);分组并发由管理端模型分组的 `maxConcurrency` 配置,填 0 表示不设置 MOYI 分组上限,未配置时使用 `GATEWAY_DEFAULT_MODEL_CONCURRENCY`。Redis 不可用时网关返回 503,不会绕过保护直接请求上游; - `/v1/chat/completions` 支持 OpenAI 兼容同步/流式文本聊天;真实供应商未配置 Key 时返回 503,只有 `provider-mock` 可以返回 Mock。V13 起按人民币分组售价预留并结算,详见 [流式与计费](16-streaming-chat-billing.md); - `/v1/3d/tasks` 和 `/v1/video/tasks` 已提供数据库持久化 Mock worker;真实供应商执行器尚未替换,返回 501 而非伪造任务成功; - Seedance、Tripo/VAST 的异步任务和 OSS 结果存储尚未迁移到 Java; diff --git a/docs/17-catalog-and-key-selection.md b/docs/17-catalog-and-key-selection.md index 14efaad..ccfb11f 100644 --- a/docs/17-catalog-and-key-selection.md +++ b/docs/17-catalog-and-key-selection.md @@ -13,13 +13,13 @@ API Key 权限窗口支持同时勾选多个模型,再选择各自的分组; 一次同步模型调用要同时通过两层并发保护: 1. **平台 API Key 并发**:限制同一个用户 Key 同时进行中的请求数量。默认值为 4,可在服务端环境变量 `GATEWAY_DEFAULT_KEY_CONCURRENCY` 调整;每个 Key 独立计数,不会因为其他用户占满而互相阻塞。 -2. **上游分组并发**:限制同一个渠道分组向中转站发出的同时请求数量,供使用该分组的多个用户共享。管理员在模型与分组中填写“上游分组并发上限”;1 表示串行,4 表示最多同时处理 4 个请求。 +2. **上游分组并发**:可限制同一个渠道分组向中转站发出的同时请求数量,供使用该分组的多个用户共享。管理员在模型与分组中填写“MOYI 分组并发上限”;填 0 表示不设置 MOYI 分组上限,直接观察渠道方的真实限制;只有渠道明确给出共享上限时才填写正整数,1 才表示串行。 > **注意:这个数字来自 MOYI,不是渠道商返回值。** 早期目录迁移为了防止未经验证的路由压垮上游,把新建分组的 `max_concurrency` 初始写成了 `1`;因此管理端看到的 `1` 不能证明 ApiHub、APIPro、TokenFoundryX 或 Tripo 只允许一个并发。渠道商的账户并发、RPM、TPM 和跨模型共享额度必须通过渠道文档或实测确认。若要做渠道容量压测,先为目标分组临时设置足够大的 MOYI 上限,并同步提高测试 API Key 的并发和 RPM,避免平台自身的保护层先返回 `MODEL_BUSY`、`KEY_BUSY` 或 `RATE_LIMITED`。 两层限制会同时生效。同步/流式请求先尝试原子取得“模型分组 + 平台 Key”两个并发槽;若任一层满额,则在平台内进行有界等待,默认最多 30 秒(`GATEWAY_QUEUE_WAIT_MS`),每 100 毫秒重试一次(`GATEWAY_QUEUE_POLL_MS`)。等待时不会占着另一层资源;获得两个槽后才向上游发起请求,超时才返回 429 和 `Retry-After`。API Key 的每分钟请求频率和并发数是不同设置;异步任务另受任务队列上限控制。 -这里的数值是 MOYI 配置的保护上限,不是性能压测上限,也不是中转站承诺的账号额度。上游账号可能还有跨模型共享的并发/RPM/TPM限制;需要用有效上游密钥做阶梯压测或由渠道方确认后,再提高 MOYI 分组上限。 +这里的正整数是 MOYI 配置的保护上限,不是性能压测上限,也不是中转站承诺的账号额度。默认值为 0(不限制模型分组);上游账号仍可能有跨模型共享的并发/RPM/TPM 限制,达到时会返回上游限流错误。API Key 并发和请求频率仍然生效,方便先测渠道能力,再按渠道方确认的额度设置保护值。 ### 渠道最大并发的实测方法 diff --git a/infra/moyi/docker-compose.cloud.yml b/infra/moyi/docker-compose.cloud.yml index 0c79887..cdc6067 100644 --- a/infra/moyi/docker-compose.cloud.yml +++ b/infra/moyi/docker-compose.cloud.yml @@ -46,8 +46,13 @@ services: TRIPO_API_KEY: ${TRIPO_API_KEY:-} GATEWAY_REQUESTS_PER_MINUTE: ${GATEWAY_REQUESTS_PER_MINUTE:-60} GATEWAY_READS_PER_MINUTE: ${GATEWAY_READS_PER_MINUTE:-300} - GATEWAY_DEFAULT_MODEL_CONCURRENCY: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:-4} + # 0 means no MOYI-side model-group cap; the upstream provider remains + # responsible for its own concurrency and rate limits. + GATEWAY_DEFAULT_MODEL_CONCURRENCY: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:-0} GATEWAY_DEFAULT_KEY_CONCURRENCY: ${GATEWAY_DEFAULT_KEY_CONCURRENCY:-4} + # Set true for an upstream concurrency measurement run. This bypasses + # all MOYI model/group leases while retaining per-Key protection. + GATEWAY_DISABLE_MODEL_CONCURRENCY: ${GATEWAY_DISABLE_MODEL_CONCURRENCY:-false} GATEWAY_QUEUE_WAIT_MS: ${GATEWAY_QUEUE_WAIT_MS:-30000} GATEWAY_QUEUE_POLL_MS: ${GATEWAY_QUEUE_POLL_MS:-100} TASK_MAX_QUEUED_PER_MODEL: ${TASK_MAX_QUEUED_PER_MODEL:-100} diff --git a/prototype/src/components/CatalogAdmin.vue b/prototype/src/components/CatalogAdmin.vue index e684111..e2f5804 100644 --- a/prototype/src/components/CatalogAdmin.vue +++ b/prototype/src/components/CatalogAdmin.vue @@ -29,7 +29,7 @@ async function remove() { } const channel = reactive({ code: '', name: '', protocol: 'openai-compatible' as CatalogProvider['protocol'], sourceType: 'relay' as CatalogProvider['sourceType'], baseUrl: '', credentialType: 'api-key' as NonNullable, credentialSecret: '', description: '', status: 'active' as CatalogProvider['status'] }) const model = reactive({ publicName: '', modelKey: '', vendorName: '', type: 'text' as LogicalModel['type'], mode: 'sync' as LogicalModel['mode'], description: '', inputTypes: 'text', outputTypes: 'text', status: 'active' as LogicalModel['status'] }) -const group = reactive({ logicalModelId: '', providerId: '', sourceLabel: '', upstreamModel: '', input: '', output: '', cachedInput: '', unit: 'CNY / 1M tokens', discount: '', discountLabel: '', exchangeRate: '', priceSource: '', listPriceInput: '', listPriceOutput: '', listPriceCachedInput: '', priority: '100', maxConcurrency: '4', status: 'active' as CatalogModel['status'] }) +const group = reactive({ logicalModelId: '', providerId: '', sourceLabel: '', upstreamModel: '', input: '', output: '', cachedInput: '', unit: 'CNY / 1M tokens', discount: '', discountLabel: '', exchangeRate: '', priceSource: '', listPriceInput: '', listPriceOutput: '', listPriceCachedInput: '', priority: '100', maxConcurrency: '0', status: 'active' as CatalogModel['status'] }) const shownModels = computed(() => models.value.filter(item => `${item.publicName} ${item.vendorName || ''} ${item.modelKey || ''}`.toLowerCase().includes(query.value.toLowerCase()))) const shownChannels = computed(() => channels.value.filter(item => `${item.name} ${item.code}`.toLowerCase().includes(query.value.toLowerCase()))) const { page: modelPage, pageSize: modelPageSize, pagedItems: pagedModels } = usePagination(shownModels) @@ -64,7 +64,7 @@ function editModel(item?: LogicalModel) { } function editGroup(parent: LogicalModel, item?: CatalogModel) { editingId.value = item?.id - Object.assign(group, { logicalModelId: parent.id, providerId: item?.providerId || '', sourceLabel: item?.sourceLabel || '', upstreamModel: item?.upstreamModel || '', input: item?.pricing?.input?.toString() || '', output: item?.pricing?.output?.toString() || '', cachedInput: item?.pricing?.cachedInput?.toString() || '', unit: item?.pricing?.unit || 'CNY / 1M tokens', discount: item?.pricing?.discount?.toString() || '', discountLabel: item?.pricing?.discountLabel || '', exchangeRate: item?.pricing?.exchangeRate?.toString() || '', priceSource: item?.pricing?.priceSource || '', listPriceInput: item?.pricing?.listPriceUsd?.input?.toString() || '', listPriceOutput: item?.pricing?.listPriceUsd?.output?.toString() || '', listPriceCachedInput: item?.pricing?.listPriceUsd?.cachedInput?.toString() || '', priority: String(item?.priority ?? 100), maxConcurrency: String(item?.maxConcurrency ?? 4), status: item?.status || 'active' }) + Object.assign(group, { logicalModelId: parent.id, providerId: item?.providerId || '', sourceLabel: item?.sourceLabel || '', upstreamModel: item?.upstreamModel || '', input: item?.pricing?.input?.toString() || '', output: item?.pricing?.output?.toString() || '', cachedInput: item?.pricing?.cachedInput?.toString() || '', unit: item?.pricing?.unit || 'CNY / 1M tokens', discount: item?.pricing?.discount?.toString() || '', discountLabel: item?.pricing?.discountLabel || '', exchangeRate: item?.pricing?.exchangeRate?.toString() || '', priceSource: item?.pricing?.priceSource || '', listPriceInput: item?.pricing?.listPriceUsd?.input?.toString() || '', listPriceOutput: item?.pricing?.listPriceUsd?.output?.toString() || '', listPriceCachedInput: item?.pricing?.listPriceUsd?.cachedInput?.toString() || '', priority: String(item?.priority ?? 100), maxConcurrency: String(item?.maxConcurrency ?? 0), status: item?.status || 'active' }) editor.value = 'group' } async function save() { @@ -108,7 +108,7 @@ async function toggle(kind: 'channel' | 'model' | 'group', item: CatalogProvider

{{ props.section === 'channels' ? '渠道管理' : '模型与分组' }}

{{ error }}

-

启用模型后会显示在用户端模型广场。模型厂商表示模型的实际研发方,渠道表示请求经过的中转站或官方接口;分组与所属渠道也启用后,用户才能为 API Key 选择该分组。平台 API Key 并发和上游渠道/模型并发是两层独立限制:前者限制同一个 Key 的同时请求数,后者在下方按分组填写,填 1 才是串行,填 4 表示同一渠道分组最多同时处理 4 个请求。

+

启用模型后会显示在用户端模型广场。模型厂商表示模型的实际研发方,渠道表示请求经过的中转站或官方接口;分组与所属渠道也启用后,用户才能为 API Key 选择该分组。平台 API Key 并发和上游渠道/模型并发是两层独立限制:前者限制同一个 Key 的同时请求数,后者在下方按分组填写;填 0 表示不设置 MOYI 分组上限,直接测量渠道方的真实并发。只有渠道方明确给出共享并发上限时才填写正整数。

加载中…

渠道来源 / 协议Base URL凭据状态操作
{{ item.name }}{{ item.code }}{{ item.sourceType }}{{ item.protocol }}{{ item.baseUrl || '未配置' }}{{ item.credentialMasked || '未配置' }}
暂无渠道
@@ -116,7 +116,7 @@ async function toggle(kind: 'channel' | 'model' | 'group', item: CatalogProvider

{{ item.publicName }}

{{ item.vendorName || '未标注厂商' }} · {{ item.modelKey || item.id }} · {{ item.type }} · {{ item.mode }}

{{ item.description }}

-
分组名称渠道上游模型输入 / 输出价格对外折扣并发状态操作
{{ route.sourceLabel }}{{ channelName(route.providerId) }}{{ route.upstreamModel }}{{ route.pricing?.input ?? '—' }} / {{ route.pricing?.output ?? '—' }}{{ route.pricing?.unit }}{{ route.pricing?.discountLabel || '—' }}{{ route.maxConcurrency ?? '默认' }}
暂无分组
+
分组名称渠道上游模型输入 / 输出价格对外折扣并发状态操作
{{ route.sourceLabel }}{{ channelName(route.providerId) }}{{ route.upstreamModel }}{{ route.pricing?.input ?? '—' }} / {{ route.pricing?.output ?? '—' }}{{ route.pricing?.unit }}{{ route.pricing?.discountLabel || '—' }}{{ route.maxConcurrency === undefined || route.maxConcurrency === null || route.maxConcurrency === 0 ? '不限(由渠道限流)' : route.maxConcurrency }}
暂无分组

暂无模型

@@ -155,7 +155,7 @@ async function toggle(kind: 'channel' | 'model' | 'group', item: CatalogProvider 输入、输出价格是实际扣费值,单位为 CNY / 1M tokens。折扣只用于向用户展示,不会替代人民币售价;留空表示不展示折扣。 - + diff --git a/prototype/src/services/catalog.ts b/prototype/src/services/catalog.ts index 3212574..512865a 100644 --- a/prototype/src/services/catalog.ts +++ b/prototype/src/services/catalog.ts @@ -174,6 +174,7 @@ export interface AdminUsageSummary { queueWaitMs: number defaultModelConcurrency: number defaultKeyConcurrency: number + modelConcurrencyGuardEnabled: boolean asyncQueueLimit: number asyncWorkerIntervalMs: number synchronousSnapshotAvailable: boolean diff --git a/prototype/src/views/AdminConsole.vue b/prototype/src/views/AdminConsole.vue index 4f965e4..3c15440 100644 --- a/prototype/src/views/AdminConsole.vue +++ b/prototype/src/views/AdminConsole.vue @@ -202,7 +202,9 @@ const queueGroups = computed(() => (adminSummary.value?.capacityGroups || []).ma warning: group.running === null || group.routeEnabled === false || !!group.queued, }))) const { page: capacityPage, pageSize: capacityPageSize, pagedItems: capacityPageGroups } = usePagination(queueGroups) -const configuredCapacity = computed(() => adminSummary.value?.capacityGroups.filter(group => group.routeEnabled !== false).reduce((total, group) => total + group.limit, 0)) +const configuredCapacityGroups = computed(() => adminSummary.value?.capacityGroups.filter(group => group.routeEnabled !== false) || []) +const hasUnlimitedCapacity = computed(() => configuredCapacityGroups.value.some(group => group.limit <= 0)) +const configuredCapacity = computed(() => hasUnlimitedCapacity.value ? null : configuredCapacityGroups.value.reduce((total, group) => total + group.limit, 0)) const pricing = computed(() => models.value.filter((model) => model.pricingConfigured).map((model) => ({ model: model.name, unit: model.pricing?.unit || 'CNY / 1M tokens', @@ -293,12 +295,12 @@ function mapModels(items: CatalogModel[], providersList: CatalogProvider[]): Mod