feat: separate api key and upstream concurrency
This commit is contained in:
1 parent
281849ceb8
commit
f5f855c751
13 files changed
+74
-8
No files matched your search
+2
-1
@@ -80,7 +80,8 @@ public class GatewayController {
|
||||
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||
try (GatewayGuard.Slot slot = guard.acquire(model)) {
|
||||
try (GatewayGuard.Slot keySlot = guard.acquireKey(UUID.fromString(String.valueOf(key.get("id"))));
|
||||
GatewayGuard.Slot modelSlot = guard.acquire(model)) {
|
||||
var ticket=billing.reserve(key,model,request,idempotencyKey);
|
||||
servletResponse.setHeader("X-Request-Id",ticket.id().toString());
|
||||
try {
|
||||
|
||||
@@ -25,6 +25,7 @@ public class GatewayGuard {
|
||||
private final int requestsPerMinute;
|
||||
private final int readsPerMinute;
|
||||
private final int defaultModelConcurrency;
|
||||
private final int defaultKeyConcurrency;
|
||||
private final String redisPrefix;
|
||||
|
||||
private static DefaultRedisScript<Long> script(String location) {
|
||||
@@ -34,26 +35,35 @@ public class GatewayGuard {
|
||||
return script;
|
||||
}
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public GatewayGuard(
|
||||
StringRedisTemplate redis,
|
||||
@Value("${moyi.gateway.requests-per-minute:60}") int requestsPerMinute,
|
||||
@Value("${moyi.gateway.reads-per-minute:300}") int readsPerMinute,
|
||||
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
|
||||
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
|
||||
@Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix
|
||||
) {
|
||||
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1) {
|
||||
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) {
|
||||
throw new IllegalArgumentException("Gateway limits must be positive");
|
||||
}
|
||||
this.redis = redis;
|
||||
this.requestsPerMinute = requestsPerMinute;
|
||||
this.readsPerMinute = readsPerMinute;
|
||||
this.defaultModelConcurrency = defaultModelConcurrency;
|
||||
this.defaultKeyConcurrency = defaultKeyConcurrency;
|
||||
if (redisPrefix == null || !redisPrefix.matches("[A-Za-z0-9:_-]{2,80}:")) {
|
||||
throw new IllegalArgumentException("Redis prefix must end in ':' and contain only letters, numbers, ':', '_' or '-'");
|
||||
}
|
||||
this.redisPrefix = redisPrefix;
|
||||
}
|
||||
|
||||
/** Compatibility constructor for isolated unit tests and local callers. */
|
||||
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
|
||||
int defaultModelConcurrency, String redisPrefix) {
|
||||
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, 4, redisPrefix);
|
||||
}
|
||||
|
||||
public void checkRate(UUID keyId, boolean readOnly) {
|
||||
String bucket = redisPrefix + "gateway:rate:" + (readOnly ? "read:" : "invoke:") + keyId;
|
||||
long retryMs = execute(RATE, bucket, Integer.toString(readOnly ? readsPerMinute : requestsPerMinute), "60000");
|
||||
@@ -68,13 +78,22 @@ public class GatewayGuard {
|
||||
}
|
||||
|
||||
public Slot acquire(Map<String, Object> model) {
|
||||
String bucket = redisPrefix + "gateway:slots:" + model.get("id");
|
||||
String bucket = redisPrefix + "gateway:slots:model:" + model.get("id");
|
||||
String token = UUID.randomUUID().toString();
|
||||
long acquired = execute(ACQUIRE, bucket, Integer.toString(modelConcurrency(model)), Long.toString(SLOT_TTL_MS), token);
|
||||
if (acquired != 1) throw new GatewayLimitException("MODEL_BUSY", "当前模型并发已满,请稍后重试", 429, 1);
|
||||
return new Slot(bucket, token);
|
||||
}
|
||||
|
||||
/** Limits simultaneous in-flight requests made through one MOYI platform API Key. */
|
||||
public Slot acquireKey(UUID keyId) {
|
||||
String bucket = redisPrefix + "gateway:slots:key:" + keyId;
|
||||
String token = UUID.randomUUID().toString();
|
||||
long acquired = execute(ACQUIRE, bucket, Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token);
|
||||
if (acquired != 1) throw new GatewayLimitException("KEY_BUSY", "当前 API Key 的并发已满,请稍后重试", 429, 1);
|
||||
return new Slot(bucket, token);
|
||||
}
|
||||
|
||||
private long execute(DefaultRedisScript<Long> script, String key, String... args) {
|
||||
try {
|
||||
Long value = redis.execute(script, List.of(key), (Object[]) args);
|
||||
|
||||
@@ -44,6 +44,7 @@ moyi:
|
||||
requests-per-minute: ${GATEWAY_REQUESTS_PER_MINUTE:60}
|
||||
reads-per-minute: ${GATEWAY_READS_PER_MINUTE:300}
|
||||
default-model-concurrency: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:4}
|
||||
default-key-concurrency: ${GATEWAY_DEFAULT_KEY_CONCURRENCY:4}
|
||||
tasks:
|
||||
max-queued-per-model: ${TASK_MAX_QUEUED_PER_MODEL:100}
|
||||
worker-interval-ms: ${MOYI_TASK_WORKER_INTERVAL_MS:1000}
|
||||
|
||||
+11
@@ -29,6 +29,17 @@ class GatewayDeploymentTest {
|
||||
verify(redis).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:rate:invoke:" + keyId)), any(Object[].class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void keyConcurrencyUsesDedicatedRedisBucket() {
|
||||
StringRedisTemplate redis = mock(StringRedisTemplate.class);
|
||||
doReturn(1L).when(redis).execute(any(RedisScript.class), anyList(), any(Object[].class));
|
||||
UUID keyId = UUID.randomUUID();
|
||||
try (GatewayGuard.Slot ignored = new GatewayGuard(redis, 60, 300, 4, 2, "moyi:test:").acquireKey(keyId)) {
|
||||
// The lease is intentionally held for the duration of this assertion.
|
||||
}
|
||||
verify(redis, atLeastOnce()).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:slots:key:" + keyId)), any(Object[].class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidRedisPrefixIsRejected() {
|
||||
assertThrows(IllegalArgumentException.class, () -> new GatewayGuard(mock(StringRedisTemplate.class), 60, 300, 4, ""));
|
||||
|
||||
Reference in new issue
Block a user