add queued gateway admission and finish admin monitoring

This commit is contained in:
wlx committed 2026-09-27 02:58:16 +08:00
1 parent 8f75ebdf96
commit 0e6fca81b3
12 files changed
+288 -35

No files matched your search

@@ -80,8 +80,9 @@ public class GatewayController {
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
ChatMeter.Request request=ChatMeter.request(input,json);
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
try (GatewayGuard.Slot keySlot = guard.acquireKey(UUID.fromString(String.valueOf(key.get("id"))));
GatewayGuard.Slot modelSlot = guard.acquire(model)) {
// Reserve both limits atomically. A queued request never consumes a
// model slot while waiting for the API-Key slot (or vice versa).
try (GatewayGuard.Admission admission = guard.acquire(model, UUID.fromString(String.valueOf(key.get("id"))), guard.queueWaitMs())) {
var ticket=billing.reserve(key,model,request,idempotencyKey);
servletResponse.setHeader("X-Request-Id",ticket.id().toString());
try {
@@ -20,12 +20,15 @@ public class GatewayGuard {
private static final long SLOT_TTL_MS = 180_000;
private static final DefaultRedisScript<Long> RATE = script("redis/rate-limit.lua");
private static final DefaultRedisScript<Long> ACQUIRE = script("redis/acquire-slot.lua");
private static final DefaultRedisScript<Long> ACQUIRE_PAIR = script("redis/acquire-pair.lua");
private static final DefaultRedisScript<Long> RELEASE = new DefaultRedisScript<>("return redis.call('ZREM', KEYS[1], ARGV[1])", Long.class);
private final StringRedisTemplate redis;
private final int requestsPerMinute;
private final int readsPerMinute;
private final int defaultModelConcurrency;
private final int defaultKeyConcurrency;
private final long queueWaitMs;
private final long queuePollMs;
private final String redisPrefix;
private static DefaultRedisScript<Long> script(String location) {
@@ -42,9 +45,12 @@ public class GatewayGuard {
@Value("${moyi.gateway.reads-per-minute:300}") int readsPerMinute,
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
@Value("${moyi.gateway.queue-wait-ms:30000}") long queueWaitMs,
@Value("${moyi.gateway.queue-poll-ms:100}") long queuePollMs,
@Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix
) {
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) {
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1
|| queueWaitMs < 0 || queuePollMs < 10) {
throw new IllegalArgumentException("Gateway limits must be positive");
}
this.redis = redis;
@@ -52,6 +58,8 @@ public class GatewayGuard {
this.readsPerMinute = readsPerMinute;
this.defaultModelConcurrency = defaultModelConcurrency;
this.defaultKeyConcurrency = defaultKeyConcurrency;
this.queueWaitMs = queueWaitMs;
this.queuePollMs = queuePollMs;
if (redisPrefix == null || !redisPrefix.matches("[A-Za-z0-9:_-]{2,80}:")) {
throw new IllegalArgumentException("Redis prefix must end in ':' and contain only letters, numbers, ':', '_' or '-'");
}
@@ -61,7 +69,13 @@ public class GatewayGuard {
/** Compatibility constructor for isolated unit tests and local callers. */
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
int defaultModelConcurrency, String redisPrefix) {
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, 4, redisPrefix);
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, 4, 0, 100, redisPrefix);
}
/** Compatibility constructor for tests that need a configurable key limit. */
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
int defaultModelConcurrency, int defaultKeyConcurrency, String redisPrefix) {
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, defaultKeyConcurrency, 0, 100, redisPrefix);
}
public void checkRate(UUID keyId, boolean readOnly) {
@@ -78,20 +92,73 @@ public class GatewayGuard {
}
public Slot acquire(Map<String, Object> model) {
return acquire(model, 0);
}
/**
* Acquire a model slot, waiting in a bounded queue when the configured
* concurrency is full. A bounded wait keeps OpenAI-compatible callers
* synchronous while preventing an unbounded servlet-thread backlog.
*/
public Slot acquire(Map<String, Object> model, long requestedWaitMs) {
String bucket = redisPrefix + "gateway:slots:model:" + model.get("id");
String token = UUID.randomUUID().toString();
long acquired = execute(ACQUIRE, bucket, Integer.toString(modelConcurrency(model)), Long.toString(SLOT_TTL_MS), token);
if (acquired != 1) throw new GatewayLimitException("MODEL_BUSY", "当前模型并发已满,请稍后重试", 429, 1);
return new Slot(bucket, token);
return acquireSlot(bucket, modelConcurrency(model), "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs);
}
/** Limits simultaneous in-flight requests made through one MOYI platform API Key. */
public Slot acquireKey(UUID keyId) {
return acquireKey(keyId, 0);
}
/** Acquire the per-platform-Key slot, using the same bounded queue. */
public Slot acquireKey(UUID keyId, long requestedWaitMs) {
String bucket = redisPrefix + "gateway:slots:key:" + keyId;
return acquireSlot(bucket, defaultKeyConcurrency, "KEY_BUSY", "当前 API Key 的并发已满,已进入排队,请稍后重试", requestedWaitMs);
}
public long queueWaitMs() { return queueWaitMs; }
/** Atomically reserves the model and API-Key slots, so queued requests do not hold one scarce slot while waiting for the other. */
public Admission acquire(Map<String, Object> model, UUID keyId, long requestedWaitMs) {
String modelBucket = redisPrefix + "gateway:slots:model:" + model.get("id");
String keyBucket = redisPrefix + "gateway:slots:key:" + keyId;
String token = UUID.randomUUID().toString();
long acquired = execute(ACQUIRE, bucket, Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token);
if (acquired != 1) throw new GatewayLimitException("KEY_BUSY", "当前 API Key 的并发已满,请稍后重试", 429, 1);
return new Slot(bucket, token);
long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs));
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
while (true) {
long acquired = executePair(modelBucket, keyBucket, Integer.toString(modelConcurrency(model)), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token);
if (acquired == 1) return new Admission(modelBucket, keyBucket, token);
if (waitMs == 0 || System.nanoTime() >= deadline) {
throw new GatewayLimitException("MODEL_BUSY", "当前模型或 API Key 并发已满,已排队等待但仍未获得资源", 429, Math.max(1, (int) Math.ceil(queuePollMs / 1000.0)));
}
try {
long remaining = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime());
Thread.sleep(Math.max(1, Math.min(queuePollMs, remaining)));
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw new GatewayLimitException("MODEL_BUSY", "排队被中断,请稍后重试", 429, 1);
}
}
}
private Slot acquireSlot(String bucket, int limit, String code, String message, long requestedWaitMs) {
String token = UUID.randomUUID().toString();
long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs));
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
while (true) {
long acquired = execute(ACQUIRE, bucket, Integer.toString(limit), Long.toString(SLOT_TTL_MS), token);
if (acquired == 1) return new Slot(bucket, token);
if (waitMs == 0 || System.nanoTime() >= deadline) {
throw new GatewayLimitException(code, message, 429, Math.max(1, (int) Math.ceil(queuePollMs / 1000.0)));
}
try {
long remaining = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime());
Thread.sleep(Math.max(1, Math.min(queuePollMs, remaining)));
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw new GatewayLimitException(code, "排队被中断,请稍后重试", 429, 1);
}
}
}
private long execute(DefaultRedisScript<Long> script, String key, String... args) {
@@ -104,6 +171,34 @@ public class GatewayGuard {
throw new GatewayLimitException("GATEWAY_GUARD_UNAVAILABLE", "网关流量控制暂不可用,请稍后重试", 503, 3);
}
private long executePair(String modelKey, String keyKey, String modelLimit, String keyLimit, String ttl, String token) {
try {
Long value = redis.execute(ACQUIRE_PAIR, List.of(modelKey, keyKey), modelLimit, keyLimit, ttl, token);
if (value != null) return value;
} catch (DataAccessException exception) {
log.warn("Gateway pair admission Redis operation failed ({})", exception.getClass().getSimpleName());
}
throw new GatewayLimitException("GATEWAY_GUARD_UNAVAILABLE", "网关流量控制暂不可用,请稍后重试", 503, 3);
}
public final class Admission implements AutoCloseable {
private final Slot modelSlot;
private final Slot keySlot;
private final AtomicBoolean closed = new AtomicBoolean();
private Admission(String modelBucket, String keyBucket, String token) {
this.modelSlot = new Slot(modelBucket, token);
this.keySlot = new Slot(keyBucket, token);
}
@Override
public void close() {
if (!closed.compareAndSet(false, true)) return;
modelSlot.close();
keySlot.close();
}
}
public final class Slot implements AutoCloseable {
private final String key;
private final String token;
@@ -45,6 +45,11 @@ public class UsageAuditController {
return usage.listAdmin(AuthController.sessionToken(request), filters(q, keyId, modelId, status, eventKind, from, to, page, size), auth);
}
@GetMapping("/summary")
public Map<String, Object> summary(jakarta.servlet.http.HttpServletRequest request) {
return usage.adminSummary(AuthController.sessionToken(request), auth);
}
@GetMapping("/{id}")
public Map<String, Object> detail(@PathVariable String id, jakarta.servlet.http.HttpServletRequest request) {
return Map.of("record", usage.detail(AuthController.sessionToken(request), id, auth));
@@ -94,6 +94,56 @@ public class UsageAuditService {
return result;
}
/**
* Small, read-only aggregate used by the admin dashboard. Keeping these
* counters in the control plane means the dashboard no longer presents
* placeholder dashes while the detailed usage table remains paginated.
*/
@Transactional(readOnly = true)
public Map<String, Object> adminSummary(String token, AuthService auth) {
auth.requireAdmin(token);
Map<String, Object> result = new LinkedHashMap<>();
result.put("callsToday", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND event_kind = 'call' AND created_at >= CURRENT_DATE"));
result.put("callsThisMonth", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND event_kind = 'call' AND created_at >= date_trunc('month', now())"));
result.put("waiting", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND status IN ('queued','running')"));
result.put("costToday", decimal("SELECT COALESCE(SUM(cost_amount),0) FROM usage_events WHERE deleted_at IS NULL AND status = 'succeeded' AND created_at >= CURRENT_DATE"));
result.put("costThisMonth", decimal("SELECT COALESCE(SUM(cost_amount),0) FROM usage_events WHERE deleted_at IS NULL AND status = 'succeeded' AND created_at >= date_trunc('month', now())"));
result.put("series", jdbc.query(
"SELECT to_char(day, 'YYYY-MM-DD') AS day, COUNT(ue.id) AS count " +
"FROM generate_series(CURRENT_DATE - INTERVAL '6 days', CURRENT_DATE, INTERVAL '1 day') day " +
"LEFT JOIN usage_events ue ON ue.deleted_at IS NULL AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " +
"GROUP BY day ORDER BY day",
(rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count"))));
result.put("capacityGroups", jdbc.query(
"SELECT m.id, m.public_name, COALESCE(m.max_concurrency, 0) AS max_concurrency, " +
"COUNT(*) FILTER (WHERE ue.status = 'running') AS running, " +
"COUNT(*) FILTER (WHERE ue.status IN ('queued','running')) AS queued " +
"FROM model_catalog m LEFT JOIN usage_events ue ON ue.model_id = m.id " +
"AND ue.deleted_at IS NULL AND ue.created_at >= now() - INTERVAL '10 minutes' " +
"WHERE m.deleted_at IS NULL AND m.status = 'active' " +
"GROUP BY m.id, m.public_name, m.max_concurrency ORDER BY m.public_name LIMIT 100",
(rs, row) -> {
Map<String, Object> group = new LinkedHashMap<>();
group.put("id", rs.getString("id"));
group.put("name", rs.getString("public_name"));
group.put("limit", rs.getInt("max_concurrency"));
group.put("running", rs.getLong("running"));
group.put("queued", rs.getLong("queued"));
return group;
}));
return result;
}
private long count(String sql) {
Number value = jdbc.queryForObject(sql, Number.class);
return value == null ? 0L : value.longValue();
}
private java.math.BigDecimal decimal(String sql) {
java.math.BigDecimal value = jdbc.queryForObject(sql, java.math.BigDecimal.class);
return value == null ? java.math.BigDecimal.ZERO : value;
}
@Transactional(readOnly = true)
public Map<String, Object> detail(String token, String id, AuthService auth) {
auth.requireAdmin(token);