add queued gateway admission and finish admin monitoring
This commit is contained in:
1 parent
8f75ebdf96
commit
0e6fca81b3
12 files changed
+288
-35
No files matched your search
+3
-2
@@ -80,8 +80,9 @@ public class GatewayController {
|
||||
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||
try (GatewayGuard.Slot keySlot = guard.acquireKey(UUID.fromString(String.valueOf(key.get("id"))));
|
||||
GatewayGuard.Slot modelSlot = guard.acquire(model)) {
|
||||
// Reserve both limits atomically. A queued request never consumes a
|
||||
// model slot while waiting for the API-Key slot (or vice versa).
|
||||
try (GatewayGuard.Admission admission = guard.acquire(model, UUID.fromString(String.valueOf(key.get("id"))), guard.queueWaitMs())) {
|
||||
var ticket=billing.reserve(key,model,request,idempotencyKey);
|
||||
servletResponse.setHeader("X-Request-Id",ticket.id().toString());
|
||||
try {
|
||||
|
||||
@@ -20,12 +20,15 @@ public class GatewayGuard {
|
||||
private static final long SLOT_TTL_MS = 180_000;
|
||||
private static final DefaultRedisScript<Long> RATE = script("redis/rate-limit.lua");
|
||||
private static final DefaultRedisScript<Long> ACQUIRE = script("redis/acquire-slot.lua");
|
||||
private static final DefaultRedisScript<Long> ACQUIRE_PAIR = script("redis/acquire-pair.lua");
|
||||
private static final DefaultRedisScript<Long> RELEASE = new DefaultRedisScript<>("return redis.call('ZREM', KEYS[1], ARGV[1])", Long.class);
|
||||
private final StringRedisTemplate redis;
|
||||
private final int requestsPerMinute;
|
||||
private final int readsPerMinute;
|
||||
private final int defaultModelConcurrency;
|
||||
private final int defaultKeyConcurrency;
|
||||
private final long queueWaitMs;
|
||||
private final long queuePollMs;
|
||||
private final String redisPrefix;
|
||||
|
||||
private static DefaultRedisScript<Long> script(String location) {
|
||||
@@ -42,9 +45,12 @@ public class GatewayGuard {
|
||||
@Value("${moyi.gateway.reads-per-minute:300}") int readsPerMinute,
|
||||
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
|
||||
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
|
||||
@Value("${moyi.gateway.queue-wait-ms:30000}") long queueWaitMs,
|
||||
@Value("${moyi.gateway.queue-poll-ms:100}") long queuePollMs,
|
||||
@Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix
|
||||
) {
|
||||
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) {
|
||||
if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1
|
||||
|| queueWaitMs < 0 || queuePollMs < 10) {
|
||||
throw new IllegalArgumentException("Gateway limits must be positive");
|
||||
}
|
||||
this.redis = redis;
|
||||
@@ -52,6 +58,8 @@ public class GatewayGuard {
|
||||
this.readsPerMinute = readsPerMinute;
|
||||
this.defaultModelConcurrency = defaultModelConcurrency;
|
||||
this.defaultKeyConcurrency = defaultKeyConcurrency;
|
||||
this.queueWaitMs = queueWaitMs;
|
||||
this.queuePollMs = queuePollMs;
|
||||
if (redisPrefix == null || !redisPrefix.matches("[A-Za-z0-9:_-]{2,80}:")) {
|
||||
throw new IllegalArgumentException("Redis prefix must end in ':' and contain only letters, numbers, ':', '_' or '-'");
|
||||
}
|
||||
@@ -61,7 +69,13 @@ public class GatewayGuard {
|
||||
/** Compatibility constructor for isolated unit tests and local callers. */
|
||||
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
|
||||
int defaultModelConcurrency, String redisPrefix) {
|
||||
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, 4, redisPrefix);
|
||||
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, 4, 0, 100, redisPrefix);
|
||||
}
|
||||
|
||||
/** Compatibility constructor for tests that need a configurable key limit. */
|
||||
public GatewayGuard(StringRedisTemplate redis, int requestsPerMinute, int readsPerMinute,
|
||||
int defaultModelConcurrency, int defaultKeyConcurrency, String redisPrefix) {
|
||||
this(redis, requestsPerMinute, readsPerMinute, defaultModelConcurrency, defaultKeyConcurrency, 0, 100, redisPrefix);
|
||||
}
|
||||
|
||||
public void checkRate(UUID keyId, boolean readOnly) {
|
||||
@@ -78,20 +92,73 @@ public class GatewayGuard {
|
||||
}
|
||||
|
||||
public Slot acquire(Map<String, Object> model) {
|
||||
return acquire(model, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire a model slot, waiting in a bounded queue when the configured
|
||||
* concurrency is full. A bounded wait keeps OpenAI-compatible callers
|
||||
* synchronous while preventing an unbounded servlet-thread backlog.
|
||||
*/
|
||||
public Slot acquire(Map<String, Object> model, long requestedWaitMs) {
|
||||
String bucket = redisPrefix + "gateway:slots:model:" + model.get("id");
|
||||
String token = UUID.randomUUID().toString();
|
||||
long acquired = execute(ACQUIRE, bucket, Integer.toString(modelConcurrency(model)), Long.toString(SLOT_TTL_MS), token);
|
||||
if (acquired != 1) throw new GatewayLimitException("MODEL_BUSY", "当前模型并发已满,请稍后重试", 429, 1);
|
||||
return new Slot(bucket, token);
|
||||
return acquireSlot(bucket, modelConcurrency(model), "MODEL_BUSY", "当前模型并发已满,已进入排队,请稍后重试", requestedWaitMs);
|
||||
}
|
||||
|
||||
/** Limits simultaneous in-flight requests made through one MOYI platform API Key. */
|
||||
public Slot acquireKey(UUID keyId) {
|
||||
return acquireKey(keyId, 0);
|
||||
}
|
||||
|
||||
/** Acquire the per-platform-Key slot, using the same bounded queue. */
|
||||
public Slot acquireKey(UUID keyId, long requestedWaitMs) {
|
||||
String bucket = redisPrefix + "gateway:slots:key:" + keyId;
|
||||
return acquireSlot(bucket, defaultKeyConcurrency, "KEY_BUSY", "当前 API Key 的并发已满,已进入排队,请稍后重试", requestedWaitMs);
|
||||
}
|
||||
|
||||
public long queueWaitMs() { return queueWaitMs; }
|
||||
|
||||
/** Atomically reserves the model and API-Key slots, so queued requests do not hold one scarce slot while waiting for the other. */
|
||||
public Admission acquire(Map<String, Object> model, UUID keyId, long requestedWaitMs) {
|
||||
String modelBucket = redisPrefix + "gateway:slots:model:" + model.get("id");
|
||||
String keyBucket = redisPrefix + "gateway:slots:key:" + keyId;
|
||||
String token = UUID.randomUUID().toString();
|
||||
long acquired = execute(ACQUIRE, bucket, Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token);
|
||||
if (acquired != 1) throw new GatewayLimitException("KEY_BUSY", "当前 API Key 的并发已满,请稍后重试", 429, 1);
|
||||
return new Slot(bucket, token);
|
||||
long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs));
|
||||
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
|
||||
while (true) {
|
||||
long acquired = executePair(modelBucket, keyBucket, Integer.toString(modelConcurrency(model)), Integer.toString(defaultKeyConcurrency), Long.toString(SLOT_TTL_MS), token);
|
||||
if (acquired == 1) return new Admission(modelBucket, keyBucket, token);
|
||||
if (waitMs == 0 || System.nanoTime() >= deadline) {
|
||||
throw new GatewayLimitException("MODEL_BUSY", "当前模型或 API Key 并发已满,已排队等待但仍未获得资源", 429, Math.max(1, (int) Math.ceil(queuePollMs / 1000.0)));
|
||||
}
|
||||
try {
|
||||
long remaining = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime());
|
||||
Thread.sleep(Math.max(1, Math.min(queuePollMs, remaining)));
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new GatewayLimitException("MODEL_BUSY", "排队被中断,请稍后重试", 429, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Slot acquireSlot(String bucket, int limit, String code, String message, long requestedWaitMs) {
|
||||
String token = UUID.randomUUID().toString();
|
||||
long waitMs = Math.max(0, Math.min(queueWaitMs, requestedWaitMs > 0 ? requestedWaitMs : queueWaitMs));
|
||||
long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(waitMs);
|
||||
while (true) {
|
||||
long acquired = execute(ACQUIRE, bucket, Integer.toString(limit), Long.toString(SLOT_TTL_MS), token);
|
||||
if (acquired == 1) return new Slot(bucket, token);
|
||||
if (waitMs == 0 || System.nanoTime() >= deadline) {
|
||||
throw new GatewayLimitException(code, message, 429, Math.max(1, (int) Math.ceil(queuePollMs / 1000.0)));
|
||||
}
|
||||
try {
|
||||
long remaining = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime());
|
||||
Thread.sleep(Math.max(1, Math.min(queuePollMs, remaining)));
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new GatewayLimitException(code, "排队被中断,请稍后重试", 429, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private long execute(DefaultRedisScript<Long> script, String key, String... args) {
|
||||
@@ -104,6 +171,34 @@ public class GatewayGuard {
|
||||
throw new GatewayLimitException("GATEWAY_GUARD_UNAVAILABLE", "网关流量控制暂不可用,请稍后重试", 503, 3);
|
||||
}
|
||||
|
||||
private long executePair(String modelKey, String keyKey, String modelLimit, String keyLimit, String ttl, String token) {
|
||||
try {
|
||||
Long value = redis.execute(ACQUIRE_PAIR, List.of(modelKey, keyKey), modelLimit, keyLimit, ttl, token);
|
||||
if (value != null) return value;
|
||||
} catch (DataAccessException exception) {
|
||||
log.warn("Gateway pair admission Redis operation failed ({})", exception.getClass().getSimpleName());
|
||||
}
|
||||
throw new GatewayLimitException("GATEWAY_GUARD_UNAVAILABLE", "网关流量控制暂不可用,请稍后重试", 503, 3);
|
||||
}
|
||||
|
||||
public final class Admission implements AutoCloseable {
|
||||
private final Slot modelSlot;
|
||||
private final Slot keySlot;
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
private Admission(String modelBucket, String keyBucket, String token) {
|
||||
this.modelSlot = new Slot(modelBucket, token);
|
||||
this.keySlot = new Slot(keyBucket, token);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
modelSlot.close();
|
||||
keySlot.close();
|
||||
}
|
||||
}
|
||||
|
||||
public final class Slot implements AutoCloseable {
|
||||
private final String key;
|
||||
private final String token;
|
||||
|
||||
@@ -45,6 +45,11 @@ public class UsageAuditController {
|
||||
return usage.listAdmin(AuthController.sessionToken(request), filters(q, keyId, modelId, status, eventKind, from, to, page, size), auth);
|
||||
}
|
||||
|
||||
@GetMapping("/summary")
|
||||
public Map<String, Object> summary(jakarta.servlet.http.HttpServletRequest request) {
|
||||
return usage.adminSummary(AuthController.sessionToken(request), auth);
|
||||
}
|
||||
|
||||
@GetMapping("/{id}")
|
||||
public Map<String, Object> detail(@PathVariable String id, jakarta.servlet.http.HttpServletRequest request) {
|
||||
return Map.of("record", usage.detail(AuthController.sessionToken(request), id, auth));
|
||||
|
||||
@@ -94,6 +94,56 @@ public class UsageAuditService {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Small, read-only aggregate used by the admin dashboard. Keeping these
|
||||
* counters in the control plane means the dashboard no longer presents
|
||||
* placeholder dashes while the detailed usage table remains paginated.
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public Map<String, Object> adminSummary(String token, AuthService auth) {
|
||||
auth.requireAdmin(token);
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("callsToday", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND event_kind = 'call' AND created_at >= CURRENT_DATE"));
|
||||
result.put("callsThisMonth", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND event_kind = 'call' AND created_at >= date_trunc('month', now())"));
|
||||
result.put("waiting", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND status IN ('queued','running')"));
|
||||
result.put("costToday", decimal("SELECT COALESCE(SUM(cost_amount),0) FROM usage_events WHERE deleted_at IS NULL AND status = 'succeeded' AND created_at >= CURRENT_DATE"));
|
||||
result.put("costThisMonth", decimal("SELECT COALESCE(SUM(cost_amount),0) FROM usage_events WHERE deleted_at IS NULL AND status = 'succeeded' AND created_at >= date_trunc('month', now())"));
|
||||
result.put("series", jdbc.query(
|
||||
"SELECT to_char(day, 'YYYY-MM-DD') AS day, COUNT(ue.id) AS count " +
|
||||
"FROM generate_series(CURRENT_DATE - INTERVAL '6 days', CURRENT_DATE, INTERVAL '1 day') day " +
|
||||
"LEFT JOIN usage_events ue ON ue.deleted_at IS NULL AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " +
|
||||
"GROUP BY day ORDER BY day",
|
||||
(rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count"))));
|
||||
result.put("capacityGroups", jdbc.query(
|
||||
"SELECT m.id, m.public_name, COALESCE(m.max_concurrency, 0) AS max_concurrency, " +
|
||||
"COUNT(*) FILTER (WHERE ue.status = 'running') AS running, " +
|
||||
"COUNT(*) FILTER (WHERE ue.status IN ('queued','running')) AS queued " +
|
||||
"FROM model_catalog m LEFT JOIN usage_events ue ON ue.model_id = m.id " +
|
||||
"AND ue.deleted_at IS NULL AND ue.created_at >= now() - INTERVAL '10 minutes' " +
|
||||
"WHERE m.deleted_at IS NULL AND m.status = 'active' " +
|
||||
"GROUP BY m.id, m.public_name, m.max_concurrency ORDER BY m.public_name LIMIT 100",
|
||||
(rs, row) -> {
|
||||
Map<String, Object> group = new LinkedHashMap<>();
|
||||
group.put("id", rs.getString("id"));
|
||||
group.put("name", rs.getString("public_name"));
|
||||
group.put("limit", rs.getInt("max_concurrency"));
|
||||
group.put("running", rs.getLong("running"));
|
||||
group.put("queued", rs.getLong("queued"));
|
||||
return group;
|
||||
}));
|
||||
return result;
|
||||
}
|
||||
|
||||
private long count(String sql) {
|
||||
Number value = jdbc.queryForObject(sql, Number.class);
|
||||
return value == null ? 0L : value.longValue();
|
||||
}
|
||||
|
||||
private java.math.BigDecimal decimal(String sql) {
|
||||
java.math.BigDecimal value = jdbc.queryForObject(sql, java.math.BigDecimal.class);
|
||||
return value == null ? java.math.BigDecimal.ZERO : value;
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public Map<String, Object> detail(String token, String id, AuthService auth) {
|
||||
auth.requireAdmin(token);
|
||||
|
||||
Reference in new issue
Block a user