feat:"拆分上游首字节/空闲/总期限并在执行期间续期预留与并发租约"
- OpenAiCompatibleProvider:总期限上限 3600 秒(默认仍 90),新增首字节与空闲超时,流式有数据即重置空闲窗口 - GatewayRequestExecutionService:调用期间定时续期钱包预留与 Redis 租约,结束即停 - ChatBillingService.renew 只延长 pending 预留;GatewayGuard 只续仍持有的租约,丢失不重建 - 新增本地 Docker 启动覆盖文件(预编译产物,仅监听 127.0.0.1) - 新增流式空闲/长流、续期、过期恢复不释放运行中调用、租约续期回归 验证:mvn -B -o clean verify 409 项通过(独立 PostgreSQL/Redis) Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1a608f8ff8
commit
aef5356c54
12 files changed
+263
-10
No files matched your search
@@ -34,6 +34,16 @@ public class GatewayGuard {
|
||||
end
|
||||
return counts
|
||||
""", List.class);
|
||||
// Renew only a lease this request still owns; an expired or released token is never recreated.
|
||||
private static final DefaultRedisScript<Long> RENEW = new DefaultRedisScript<>("""
|
||||
local time = redis.call('TIME')
|
||||
local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
||||
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
|
||||
if not score or tonumber(score) <= now then return 0 end
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + tonumber(ARGV[2]), ARGV[1])
|
||||
if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
|
||||
return 1
|
||||
""", Long.class);
|
||||
private static final DefaultRedisScript<Long> RELEASE = new DefaultRedisScript<>("return redis.call('ZREM', KEYS[1], ARGV[1])", Long.class);
|
||||
private final StringRedisTemplate redis;
|
||||
private final int requestsPerMinute;
|
||||
@@ -340,6 +350,15 @@ public class GatewayGuard {
|
||||
this.keySlot = keySlot;
|
||||
}
|
||||
|
||||
/** Extend every lease held by this admission; returns false if any lease was already lost. */
|
||||
@Override
|
||||
public boolean renew(long ttlMs) {
|
||||
if (closed.get()) return false;
|
||||
boolean held = true;
|
||||
for (Slot slot : new Slot[]{modelSlot, keySlot, userSlot}) if (slot != null && !slot.renew(ttlMs)) held = false;
|
||||
return held;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
@@ -356,6 +375,13 @@ public class GatewayGuard {
|
||||
|
||||
private Slot(String key, String token) { this.key = key; this.token = token; }
|
||||
|
||||
boolean renew(long ttlMs) {
|
||||
if (closed.get()) return false;
|
||||
if (key == null || token == null) return true;
|
||||
try { return execute(RENEW, key, token, Long.toString(Math.max(SLOT_TTL_MS, ttlMs))) == 1; }
|
||||
catch (GatewayLimitException exception) { return false; }
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
|
||||
+3
@@ -2,6 +2,9 @@ package com.moyi.controlplane.gateway.request;
|
||||
|
||||
/** One request's releasable admission lease. */
|
||||
public interface ApiRequestAdmission extends AutoCloseable {
|
||||
/** Extend the lease while the request is still running. Returns false if it was already lost. */
|
||||
default boolean renew(long ttlMs) { return true; }
|
||||
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
+35
-1
@@ -13,14 +13,33 @@ import org.springframework.stereotype.Service;
|
||||
/** Shared application boundary for synchronous and native-protocol model requests. */
|
||||
@Service
|
||||
public class GatewayRequestExecutionService {
|
||||
/** Wallet holds and Redis leases are short; a running call renews both on this cadence. */
|
||||
static final long RENEW_INTERVAL_MS = 60_000;
|
||||
static final long WALLET_HOLD_MS = 300_000;
|
||||
static final long LEASE_TTL_MS = 180_000;
|
||||
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GatewayRequestExecutionService.class);
|
||||
private final ChatBillingService billing;
|
||||
private final ApiRequestRiskPolicy risk;
|
||||
private final long renewIntervalMs;
|
||||
private final java.util.concurrent.ScheduledExecutorService renewals = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
var thread = new Thread(r, "request-lease-renewal"); thread.setDaemon(true); return thread;
|
||||
});
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public GatewayRequestExecutionService(ChatBillingService billing, ApiRequestRiskPolicy risk) {
|
||||
this(billing, risk, RENEW_INTERVAL_MS);
|
||||
}
|
||||
|
||||
GatewayRequestExecutionService(ChatBillingService billing, ApiRequestRiskPolicy risk, long renewIntervalMs) {
|
||||
this.billing = java.util.Objects.requireNonNull(billing, "billing");
|
||||
this.risk = java.util.Objects.requireNonNull(risk, "risk");
|
||||
if (renewIntervalMs < 1) throw new IllegalArgumentException("Renewal interval must be positive");
|
||||
this.renewIntervalMs = renewIntervalMs;
|
||||
}
|
||||
|
||||
@jakarta.annotation.PreDestroy
|
||||
void shutdown() { renewals.shutdownNow(); }
|
||||
|
||||
public <T> T execute(
|
||||
ApiRequestContext context,
|
||||
Map<String, Object> key,
|
||||
@@ -33,8 +52,11 @@ public class GatewayRequestExecutionService {
|
||||
Function<ApiException, T> deliverFailure
|
||||
) {
|
||||
risk.enforce(context);
|
||||
try (ApiRequestAdmission ignored = admission.get()) {
|
||||
try (ApiRequestAdmission lease = admission.get()) {
|
||||
ChatBillingService.Ticket ticket = billing.reserve(key, model, normalizedRequest, idempotencyKey, context.metadata());
|
||||
// Long calls outlive the initial hold and lease. Keep both alive until the work returns,
|
||||
// so expiry recovery cannot release a running call and other callers cannot take its slot.
|
||||
var renewal = renewals.scheduleWithFixedDelay(() -> renew(ticket, lease), renewIntervalMs, renewIntervalMs, java.util.concurrent.TimeUnit.MILLISECONDS);
|
||||
try {
|
||||
return work.execute(ticket);
|
||||
} catch (Exception failure) {
|
||||
@@ -46,10 +68,22 @@ public class GatewayRequestExecutionService {
|
||||
ApiException error = classifyFailure.apply(failure);
|
||||
billing.fail(ticket, error.getCode(), error.getMessage(), context.elapsedMillis());
|
||||
return deliverFailure.apply(error);
|
||||
} finally {
|
||||
renewal.cancel(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void renew(ChatBillingService.Ticket ticket, ApiRequestAdmission lease) {
|
||||
try {
|
||||
billing.renew(ticket.id(), WALLET_HOLD_MS);
|
||||
if (!lease.renew(LEASE_TTL_MS)) log.warn("Request concurrency lease could not be renewed; it may be reused before the call ends");
|
||||
} catch (RuntimeException exception) {
|
||||
// A missed renewal is retried on the next tick; the original hold still bounds recovery.
|
||||
log.warn("Request lease renewal failed ({})", exception.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
public BigDecimal settle(ChatBillingService.Ticket ticket, Map<String, Object> result, int latencyMs, Integer firstTokenMs) {
|
||||
// Validate and reduce to the only data required to price the call. The response body may contain
|
||||
// prompts, generated content, tool arguments or other sensitive data and is never recovery state.
|
||||
|
||||
+33
-8
@@ -30,6 +30,8 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class OpenAiCompatibleProvider {
|
||||
/** Upper bound for the configurable total deadline; leases are renewed at most this long. */
|
||||
public static final int MAX_TIMEOUT_SECONDS = 3600;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final HttpClient httpClient;
|
||||
private final JdbcTemplate jdbc;
|
||||
@@ -41,6 +43,10 @@ public class OpenAiCompatibleProvider {
|
||||
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
|
||||
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
||||
private final int timeoutSeconds;
|
||||
// Long reasoning calls need a larger total deadline without letting a silent upstream hold
|
||||
// the request, its concurrency lease and wallet hold. Both default to the total deadline.
|
||||
private final long firstByteTimeoutMs;
|
||||
private final long idleTimeoutMs;
|
||||
@org.springframework.beans.factory.annotation.Autowired private UpstreamHealth health;
|
||||
|
||||
public OpenAiCompatibleProvider(
|
||||
@@ -54,7 +60,9 @@ public class OpenAiCompatibleProvider {
|
||||
this.jdbc = jdbc;
|
||||
this.secretCipher = secretCipher;
|
||||
this.environment = environment;
|
||||
this.timeoutSeconds = Math.max(1,Math.min(90,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
||||
this.timeoutSeconds = Math.max(1,Math.min(MAX_TIMEOUT_SECONDS,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
||||
this.firstByteTimeoutMs = 1000L*Math.max(1,Math.min(timeoutSeconds,environment.getProperty("moyi.gateway.upstream-first-byte-timeout-seconds",Integer.class,timeoutSeconds)));
|
||||
this.idleTimeoutMs = 1000L*Math.max(1,Math.min(timeoutSeconds,environment.getProperty("moyi.gateway.upstream-idle-timeout-seconds",Integer.class,timeoutSeconds)));
|
||||
this.providers = Map.of(
|
||||
"provider-deepseek", configured("provider-deepseek", environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com"), environment),
|
||||
"provider-kimi", configured("provider-kimi", environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1"), environment),
|
||||
@@ -157,7 +165,7 @@ public class OpenAiCompatibleProvider {
|
||||
String json = writeJson(body);
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder()
|
||||
.uri(URI.create(base + path))
|
||||
.timeout(Duration.ofMillis(remaining))
|
||||
.timeout(Duration.ofMillis(streaming?Math.min(remaining,firstByteTimeoutMs):remaining))
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(json));
|
||||
@@ -170,7 +178,7 @@ public class OpenAiCompatibleProvider {
|
||||
if(health!=null) health.record(healthRoute,apiKey,response.statusCode(),(int)TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started));
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String details;
|
||||
try(var bounded=new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)))) {details=readErrorBody(bounded.body);}
|
||||
try(var bounded=new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)),false)) {details=readErrorBody(bounded.body);}
|
||||
if (requiresLargerOutputBudget(details)) {
|
||||
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
|
||||
}
|
||||
@@ -190,7 +198,7 @@ public class OpenAiCompatibleProvider {
|
||||
response.body().close();
|
||||
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
|
||||
}
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)));
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)),streaming);
|
||||
} catch (ApiException exception) {
|
||||
throw exception;
|
||||
} catch (HttpTimeoutException | SocketTimeoutException exception) {
|
||||
@@ -233,11 +241,28 @@ public class OpenAiCompatibleProvider {
|
||||
private final InputStream body;
|
||||
private final AtomicBoolean expired=new AtomicBoolean();
|
||||
private final ScheduledFuture<?> deadline;
|
||||
private final boolean idleWatch;
|
||||
private volatile ScheduledFuture<?> idle;
|
||||
private volatile boolean closed;
|
||||
private Runnable afterClose=()->{};
|
||||
public StreamResponse onClose(Runnable callback) {afterClose=callback;return this;}
|
||||
private StreamResponse(InputStream body,long remainingMs) {
|
||||
this.body=body;
|
||||
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
|
||||
private StreamResponse(InputStream raw,long remainingMs,boolean streaming) {
|
||||
Runnable expire=() -> { expired.set(true); try { raw.close(); } catch(IOException ignored) {} };
|
||||
// The idle watchdog only matters when it is shorter than the remaining total deadline.
|
||||
this.idleWatch=streaming&&idleTimeoutMs<remainingMs;
|
||||
this.body=!idleWatch?raw:new FilterInputStream(raw) {
|
||||
@Override public int read() throws IOException { int value=super.read(); if(value!=-1) touch(expire); return value; }
|
||||
@Override public int read(byte[] buffer,int offset,int length) throws IOException { int count=super.read(buffer,offset,length); if(count>0) touch(expire); return count; }
|
||||
};
|
||||
this.deadline=deadlines.schedule(expire,remainingMs,TimeUnit.MILLISECONDS);
|
||||
if(idleWatch) touch(expire);
|
||||
}
|
||||
/** Received bytes prove the upstream is alive; restart the idle window. */
|
||||
private void touch(Runnable expire) {
|
||||
if(closed) return;
|
||||
ScheduledFuture<?> previous=idle;
|
||||
idle=deadlines.schedule(expire,idleTimeoutMs,TimeUnit.MILLISECONDS);
|
||||
if(previous!=null) previous.cancel(false);
|
||||
}
|
||||
public void events(EventConsumer consumer) throws IOException {
|
||||
events(consumer,true);
|
||||
@@ -248,7 +273,7 @@ public class OpenAiCompatibleProvider {
|
||||
consumer.accept(chunk);
|
||||
});
|
||||
}
|
||||
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} finally {afterClose.run();afterClose=()->{};} }
|
||||
@Override public void close() { closed=true; deadline.cancel(false); if(idle!=null) idle.cancel(false); try { body.close(); } catch(IOException ignored) {} finally {afterClose.run();afterClose=()->{};} }
|
||||
}
|
||||
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
|
||||
@PreDestroy public void close() { deadlines.shutdownNow(); }
|
||||
|
||||
@@ -48,6 +48,15 @@ public class ChatBillingService {
|
||||
jdbc.update("UPDATE chat_billing_requests SET settlement_usage=?::jsonb, settlement_latency_ms=?, settlement_first_token_ms=?, expires_at=GREATEST(expires_at,now()+interval '5 minutes') WHERE id=? AND status='pending'",
|
||||
encode(normalizedUsage), Math.max(0,latencyMs), firstTokenMs, ticket.id());
|
||||
}
|
||||
/**
|
||||
* Extend a still-running call's hold so expiry recovery cannot release it mid-call.
|
||||
* Only pending tickets move, and the hold never shrinks below its current expiry.
|
||||
*/
|
||||
public boolean renew(UUID id, long holdMs) {
|
||||
long seconds=Math.max(60,(holdMs+999)/1000);
|
||||
return jdbc.update("UPDATE chat_billing_requests SET expires_at=GREATEST(expires_at,now()+make_interval(secs=>?)) WHERE id=? AND status='pending'",seconds,id)>0;
|
||||
}
|
||||
|
||||
@Transactional public void actualRoute(Ticket ticket,Map<String,Object> route) {
|
||||
var snapshots=jdbc.queryForList("SELECT procurement_pricing FROM model_price_history WHERE model_id=? ORDER BY id DESC LIMIT 1",route.get("id"));
|
||||
Object cost=snapshots.isEmpty()?null:snapshots.getFirst().get("procurement_pricing");
|
||||
|
||||
@@ -56,6 +56,12 @@ moyi:
|
||||
# provider's own response/limit.
|
||||
disable-model-concurrency: ${GATEWAY_DISABLE_MODEL_CONCURRENCY:false}
|
||||
queue-wait-ms: ${GATEWAY_QUEUE_WAIT_MS:30000}
|
||||
# Synchronous/stream upstream deadlines (seconds). Total is capped at 3600; first-byte and idle
|
||||
# default to the total and never exceed it. Running calls renew wallet holds and concurrency
|
||||
# leases, but a reverse proxy in front must also allow the longer read time.
|
||||
upstream-timeout-seconds: ${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}
|
||||
upstream-first-byte-timeout-seconds: ${GATEWAY_UPSTREAM_FIRST_BYTE_TIMEOUT_SECONDS:${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}}
|
||||
upstream-idle-timeout-seconds: ${GATEWAY_UPSTREAM_IDLE_TIMEOUT_SECONDS:${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}}
|
||||
queue-poll-ms: ${GATEWAY_QUEUE_POLL_MS:100}
|
||||
tasks:
|
||||
max-queued-per-model: ${TASK_MAX_QUEUED_PER_MODEL:100}
|
||||
|
||||
Reference in new issue
Block a user