report key and model queue saturation separately
This commit is contained in:
1 parent
16981fd307
commit
b54b6a8ac4
2 files changed
+5
-2
No files matched your search
@@ -125,11 +125,14 @@ public class GatewayGuard {
|
||||
String token = UUID.randomUUID().toString();
|
||||
long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs));
|
||||
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
|
||||
long lastBusyCode = 0;
|
||||
while (true) {
|
||||
long acquired = executePair(modelBucket, keyBucket, Integer.toString(modelConcurrency(model)), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token);
|
||||
if (acquired == 1) return new Admission(modelBucket, keyBucket, token);
|
||||
lastBusyCode = acquired;
|
||||
if (waitMs == 0 || System.nanoTime() >= deadline) {
|
||||
throw new GatewayLimitException("MODEL_BUSY", "当前模型或 API Key 并发已满,已排队等待但仍未获得资源", 429, Math.max(1, (int) Math.ceil(queuePollMs / 1000.0)));
|
||||
boolean keyBusy = lastBusyCode == 2;
|
||||
throw new GatewayLimitException(keyBusy ? "KEY_BUSY" : "MODEL_BUSY", keyBusy ? "当前 API Key 并发已满,已排队等待但仍未获得资源" : "当前模型分组并发已满,已排队等待但仍未获得资源", 429, Math.max(1, (int) Math.ceil(queuePollMs / 1000.0)));
|
||||
}
|
||||
try {
|
||||
long remaining = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime());
|
||||
|
||||
@@ -3,7 +3,7 @@ local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now)
|
||||
if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[1]) then return 0 end
|
||||
if redis.call('ZCARD', KEYS[2]) >= tonumber(ARGV[2]) then return 0 end
|
||||
if redis.call('ZCARD', KEYS[2]) >= tonumber(ARGV[2]) then return 2 end
|
||||
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[3]), ARGV[4])
|
||||
redis.call('ZADD', KEYS[2], now + tonumber(ARGV[3]), ARGV[4])
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[3])
|
||||
|
||||
Reference in new issue
Block a user