fix: remove artificial model group concurrency cap
This commit is contained in:
1 parent
25114ca6b8
commit
ce0a718686
13 files changed
+165
-34
No files matched your search
@@ -36,6 +36,7 @@ public class AsyncTaskService {
|
||||
private final int maxQueuedPerModel;
|
||||
private final int defaultModelConcurrency;
|
||||
private final int defaultKeyConcurrency;
|
||||
private final boolean modelConcurrencyEnabled;
|
||||
private final long providerTimeoutMs;
|
||||
private final GatewayGuard gatewayGuard;
|
||||
private final UsageAuditService usage;
|
||||
@@ -45,7 +46,18 @@ public class AsyncTaskService {
|
||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, gatewayGuard, usage, null, null);
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, true, gatewayGuard, usage, null, null);
|
||||
}
|
||||
|
||||
/** Compatibility constructor for isolated tests using explicit limits. */
|
||||
public AsyncTaskService(
|
||||
JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
int maxQueuedPerModel, int defaultModelConcurrency, int defaultKeyConcurrency,
|
||||
long providerTimeoutMs, GatewayGuard gatewayGuard, UsageAuditService usage,
|
||||
AsyncProviderClient providerClient, PlatformTransactionManager transactionManager
|
||||
) {
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, defaultModelConcurrency,
|
||||
defaultKeyConcurrency, providerTimeoutMs, true, gatewayGuard, usage, providerClient, transactionManager);
|
||||
}
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
@@ -57,12 +69,16 @@ public class AsyncTaskService {
|
||||
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
|
||||
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
|
||||
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long providerTimeoutMs,
|
||||
@Value("${moyi.gateway.disable-model-concurrency:false}") boolean disableModelConcurrency,
|
||||
GatewayGuard gatewayGuard,
|
||||
UsageAuditService usage,
|
||||
AsyncProviderClient providerClient,
|
||||
PlatformTransactionManager transactionManager
|
||||
) {
|
||||
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
|
||||
// A model concurrency of 0 means no MOYI-side model-group cap. The
|
||||
// upstream provider still enforces its own quota, while API-Key
|
||||
// concurrency remains independently configurable.
|
||||
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 0 || defaultKeyConcurrency < 0) throw new IllegalArgumentException("Task queue and concurrency limits must not be negative");
|
||||
if (providerTimeoutMs < 1_000) throw new IllegalArgumentException("Provider task timeout must be at least one second");
|
||||
this.jdbc = jdbc;
|
||||
this.objectMapper = objectMapper;
|
||||
@@ -70,6 +86,7 @@ public class AsyncTaskService {
|
||||
this.maxQueuedPerModel = maxQueuedPerModel;
|
||||
this.defaultModelConcurrency = defaultModelConcurrency;
|
||||
this.defaultKeyConcurrency = defaultKeyConcurrency;
|
||||
this.modelConcurrencyEnabled = !disableModelConcurrency;
|
||||
this.providerTimeoutMs = providerTimeoutMs;
|
||||
this.gatewayGuard = gatewayGuard;
|
||||
this.usage = usage;
|
||||
@@ -225,17 +242,26 @@ public class AsyncTaskService {
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND (SELECT COUNT(*) FROM model_tasks running_model
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
AND (SELECT COUNT(*) FROM model_tasks running_key
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
||||
< ?
|
||||
AND (
|
||||
? <= 0
|
||||
OR COALESCE(m.max_concurrency, ?) <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_model
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
)
|
||||
AND (
|
||||
? <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_key
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
||||
< ?
|
||||
)
|
||||
ORDER BY t.created_at
|
||||
LIMIT 1
|
||||
FOR UPDATE OF t SKIP LOCKED
|
||||
""";
|
||||
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency, defaultKeyConcurrency);
|
||||
List<Map<String,Object>> rows = jdbc.queryForList(sql,
|
||||
modelConcurrencyEnabled ? 1 : 0, defaultModelConcurrency, defaultModelConcurrency,
|
||||
defaultKeyConcurrency, defaultKeyConcurrency);
|
||||
if (rows.isEmpty()) return null;
|
||||
Map<String,Object> row = rows.getFirst();
|
||||
jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));
|
||||
|
||||
Reference in new issue
Block a user