feat:"拆分上游首字节/空闲/总期限并在执行期间续期预留与并发租约"
- OpenAiCompatibleProvider:总期限上限 3600 秒(默认仍 90),新增首字节与空闲超时,流式有数据即重置空闲窗口 - GatewayRequestExecutionService:调用期间定时续期钱包预留与 Redis 租约,结束即停 - ChatBillingService.renew 只延长 pending 预留;GatewayGuard 只续仍持有的租约,丢失不重建 - 新增本地 Docker 启动覆盖文件(预编译产物,仅监听 127.0.0.1) - 新增流式空闲/长流、续期、过期恢复不释放运行中调用、租约续期回归 验证:mvn -B -o clean verify 409 项通过(独立 PostgreSQL/Redis) Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1a608f8ff8
commit
aef5356c54
12 files changed
+263
-10
No files matched your search
@@ -34,6 +34,16 @@ public class GatewayGuard {
|
|||||||
end
|
end
|
||||||
return counts
|
return counts
|
||||||
""", List.class);
|
""", List.class);
|
||||||
|
// Renew only a lease this request still owns; an expired or released token is never recreated.
|
||||||
|
private static final DefaultRedisScript<Long> RENEW = new DefaultRedisScript<>("""
|
||||||
|
local time = redis.call('TIME')
|
||||||
|
local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
||||||
|
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
|
||||||
|
if not score or tonumber(score) <= now then return 0 end
|
||||||
|
redis.call('ZADD', KEYS[1], 'XX', now + tonumber(ARGV[2]), ARGV[1])
|
||||||
|
if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
|
||||||
|
return 1
|
||||||
|
""", Long.class);
|
||||||
private static final DefaultRedisScript<Long> RELEASE = new DefaultRedisScript<>("return redis.call('ZREM', KEYS[1], ARGV[1])", Long.class);
|
private static final DefaultRedisScript<Long> RELEASE = new DefaultRedisScript<>("return redis.call('ZREM', KEYS[1], ARGV[1])", Long.class);
|
||||||
private final StringRedisTemplate redis;
|
private final StringRedisTemplate redis;
|
||||||
private final int requestsPerMinute;
|
private final int requestsPerMinute;
|
||||||
@@ -340,6 +350,15 @@ public class GatewayGuard {
|
|||||||
this.keySlot = keySlot;
|
this.keySlot = keySlot;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Extend every lease held by this admission; returns false if any lease was already lost. */
|
||||||
|
@Override
|
||||||
|
public boolean renew(long ttlMs) {
|
||||||
|
if (closed.get()) return false;
|
||||||
|
boolean held = true;
|
||||||
|
for (Slot slot : new Slot[]{modelSlot, keySlot, userSlot}) if (slot != null && !slot.renew(ttlMs)) held = false;
|
||||||
|
return held;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() {
|
public void close() {
|
||||||
if (!closed.compareAndSet(false, true)) return;
|
if (!closed.compareAndSet(false, true)) return;
|
||||||
@@ -356,6 +375,13 @@ public class GatewayGuard {
|
|||||||
|
|
||||||
private Slot(String key, String token) { this.key = key; this.token = token; }
|
private Slot(String key, String token) { this.key = key; this.token = token; }
|
||||||
|
|
||||||
|
boolean renew(long ttlMs) {
|
||||||
|
if (closed.get()) return false;
|
||||||
|
if (key == null || token == null) return true;
|
||||||
|
try { return execute(RENEW, key, token, Long.toString(Math.max(SLOT_TTL_MS, ttlMs))) == 1; }
|
||||||
|
catch (GatewayLimitException exception) { return false; }
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() {
|
public void close() {
|
||||||
if (!closed.compareAndSet(false, true)) return;
|
if (!closed.compareAndSet(false, true)) return;
|
||||||
|
|||||||
+3
@@ -2,6 +2,9 @@ package com.moyi.controlplane.gateway.request;
|
|||||||
|
|
||||||
/** One request's releasable admission lease. */
|
/** One request's releasable admission lease. */
|
||||||
public interface ApiRequestAdmission extends AutoCloseable {
|
public interface ApiRequestAdmission extends AutoCloseable {
|
||||||
|
/** Extend the lease while the request is still running. Returns false if it was already lost. */
|
||||||
|
default boolean renew(long ttlMs) { return true; }
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
void close();
|
void close();
|
||||||
}
|
}
|
||||||
+35
-1
@@ -13,14 +13,33 @@ import org.springframework.stereotype.Service;
|
|||||||
/** Shared application boundary for synchronous and native-protocol model requests. */
|
/** Shared application boundary for synchronous and native-protocol model requests. */
|
||||||
@Service
|
@Service
|
||||||
public class GatewayRequestExecutionService {
|
public class GatewayRequestExecutionService {
|
||||||
|
/** Wallet holds and Redis leases are short; a running call renews both on this cadence. */
|
||||||
|
static final long RENEW_INTERVAL_MS = 60_000;
|
||||||
|
static final long WALLET_HOLD_MS = 300_000;
|
||||||
|
static final long LEASE_TTL_MS = 180_000;
|
||||||
|
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GatewayRequestExecutionService.class);
|
||||||
private final ChatBillingService billing;
|
private final ChatBillingService billing;
|
||||||
private final ApiRequestRiskPolicy risk;
|
private final ApiRequestRiskPolicy risk;
|
||||||
|
private final long renewIntervalMs;
|
||||||
|
private final java.util.concurrent.ScheduledExecutorService renewals = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> {
|
||||||
|
var thread = new Thread(r, "request-lease-renewal"); thread.setDaemon(true); return thread;
|
||||||
|
});
|
||||||
|
|
||||||
|
@org.springframework.beans.factory.annotation.Autowired
|
||||||
public GatewayRequestExecutionService(ChatBillingService billing, ApiRequestRiskPolicy risk) {
|
public GatewayRequestExecutionService(ChatBillingService billing, ApiRequestRiskPolicy risk) {
|
||||||
|
this(billing, risk, RENEW_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
GatewayRequestExecutionService(ChatBillingService billing, ApiRequestRiskPolicy risk, long renewIntervalMs) {
|
||||||
this.billing = java.util.Objects.requireNonNull(billing, "billing");
|
this.billing = java.util.Objects.requireNonNull(billing, "billing");
|
||||||
this.risk = java.util.Objects.requireNonNull(risk, "risk");
|
this.risk = java.util.Objects.requireNonNull(risk, "risk");
|
||||||
|
if (renewIntervalMs < 1) throw new IllegalArgumentException("Renewal interval must be positive");
|
||||||
|
this.renewIntervalMs = renewIntervalMs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@jakarta.annotation.PreDestroy
|
||||||
|
void shutdown() { renewals.shutdownNow(); }
|
||||||
|
|
||||||
public <T> T execute(
|
public <T> T execute(
|
||||||
ApiRequestContext context,
|
ApiRequestContext context,
|
||||||
Map<String, Object> key,
|
Map<String, Object> key,
|
||||||
@@ -33,8 +52,11 @@ public class GatewayRequestExecutionService {
|
|||||||
Function<ApiException, T> deliverFailure
|
Function<ApiException, T> deliverFailure
|
||||||
) {
|
) {
|
||||||
risk.enforce(context);
|
risk.enforce(context);
|
||||||
try (ApiRequestAdmission ignored = admission.get()) {
|
try (ApiRequestAdmission lease = admission.get()) {
|
||||||
ChatBillingService.Ticket ticket = billing.reserve(key, model, normalizedRequest, idempotencyKey, context.metadata());
|
ChatBillingService.Ticket ticket = billing.reserve(key, model, normalizedRequest, idempotencyKey, context.metadata());
|
||||||
|
// Long calls outlive the initial hold and lease. Keep both alive until the work returns,
|
||||||
|
// so expiry recovery cannot release a running call and other callers cannot take its slot.
|
||||||
|
var renewal = renewals.scheduleWithFixedDelay(() -> renew(ticket, lease), renewIntervalMs, renewIntervalMs, java.util.concurrent.TimeUnit.MILLISECONDS);
|
||||||
try {
|
try {
|
||||||
return work.execute(ticket);
|
return work.execute(ticket);
|
||||||
} catch (Exception failure) {
|
} catch (Exception failure) {
|
||||||
@@ -46,10 +68,22 @@ public class GatewayRequestExecutionService {
|
|||||||
ApiException error = classifyFailure.apply(failure);
|
ApiException error = classifyFailure.apply(failure);
|
||||||
billing.fail(ticket, error.getCode(), error.getMessage(), context.elapsedMillis());
|
billing.fail(ticket, error.getCode(), error.getMessage(), context.elapsedMillis());
|
||||||
return deliverFailure.apply(error);
|
return deliverFailure.apply(error);
|
||||||
|
} finally {
|
||||||
|
renewal.cancel(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void renew(ChatBillingService.Ticket ticket, ApiRequestAdmission lease) {
|
||||||
|
try {
|
||||||
|
billing.renew(ticket.id(), WALLET_HOLD_MS);
|
||||||
|
if (!lease.renew(LEASE_TTL_MS)) log.warn("Request concurrency lease could not be renewed; it may be reused before the call ends");
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
// A missed renewal is retried on the next tick; the original hold still bounds recovery.
|
||||||
|
log.warn("Request lease renewal failed ({})", exception.getClass().getSimpleName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public BigDecimal settle(ChatBillingService.Ticket ticket, Map<String, Object> result, int latencyMs, Integer firstTokenMs) {
|
public BigDecimal settle(ChatBillingService.Ticket ticket, Map<String, Object> result, int latencyMs, Integer firstTokenMs) {
|
||||||
// Validate and reduce to the only data required to price the call. The response body may contain
|
// Validate and reduce to the only data required to price the call. The response body may contain
|
||||||
// prompts, generated content, tool arguments or other sensitive data and is never recovery state.
|
// prompts, generated content, tool arguments or other sensitive data and is never recovery state.
|
||||||
|
|||||||
+33
-8
@@ -30,6 +30,8 @@ import org.springframework.stereotype.Service;
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class OpenAiCompatibleProvider {
|
public class OpenAiCompatibleProvider {
|
||||||
|
/** Upper bound for the configurable total deadline; leases are renewed at most this long. */
|
||||||
|
public static final int MAX_TIMEOUT_SECONDS = 3600;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final HttpClient httpClient;
|
private final HttpClient httpClient;
|
||||||
private final JdbcTemplate jdbc;
|
private final JdbcTemplate jdbc;
|
||||||
@@ -41,6 +43,10 @@ public class OpenAiCompatibleProvider {
|
|||||||
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
|
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
|
||||||
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
||||||
private final int timeoutSeconds;
|
private final int timeoutSeconds;
|
||||||
|
// Long reasoning calls need a larger total deadline without letting a silent upstream hold
|
||||||
|
// the request, its concurrency lease and wallet hold. Both default to the total deadline.
|
||||||
|
private final long firstByteTimeoutMs;
|
||||||
|
private final long idleTimeoutMs;
|
||||||
@org.springframework.beans.factory.annotation.Autowired private UpstreamHealth health;
|
@org.springframework.beans.factory.annotation.Autowired private UpstreamHealth health;
|
||||||
|
|
||||||
public OpenAiCompatibleProvider(
|
public OpenAiCompatibleProvider(
|
||||||
@@ -54,7 +60,9 @@ public class OpenAiCompatibleProvider {
|
|||||||
this.jdbc = jdbc;
|
this.jdbc = jdbc;
|
||||||
this.secretCipher = secretCipher;
|
this.secretCipher = secretCipher;
|
||||||
this.environment = environment;
|
this.environment = environment;
|
||||||
this.timeoutSeconds = Math.max(1,Math.min(90,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
this.timeoutSeconds = Math.max(1,Math.min(MAX_TIMEOUT_SECONDS,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
||||||
|
this.firstByteTimeoutMs = 1000L*Math.max(1,Math.min(timeoutSeconds,environment.getProperty("moyi.gateway.upstream-first-byte-timeout-seconds",Integer.class,timeoutSeconds)));
|
||||||
|
this.idleTimeoutMs = 1000L*Math.max(1,Math.min(timeoutSeconds,environment.getProperty("moyi.gateway.upstream-idle-timeout-seconds",Integer.class,timeoutSeconds)));
|
||||||
this.providers = Map.of(
|
this.providers = Map.of(
|
||||||
"provider-deepseek", configured("provider-deepseek", environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com"), environment),
|
"provider-deepseek", configured("provider-deepseek", environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com"), environment),
|
||||||
"provider-kimi", configured("provider-kimi", environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1"), environment),
|
"provider-kimi", configured("provider-kimi", environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1"), environment),
|
||||||
@@ -157,7 +165,7 @@ public class OpenAiCompatibleProvider {
|
|||||||
String json = writeJson(body);
|
String json = writeJson(body);
|
||||||
HttpRequest.Builder builder = HttpRequest.newBuilder()
|
HttpRequest.Builder builder = HttpRequest.newBuilder()
|
||||||
.uri(URI.create(base + path))
|
.uri(URI.create(base + path))
|
||||||
.timeout(Duration.ofMillis(remaining))
|
.timeout(Duration.ofMillis(streaming?Math.min(remaining,firstByteTimeoutMs):remaining))
|
||||||
.header("Authorization", "Bearer " + apiKey)
|
.header("Authorization", "Bearer " + apiKey)
|
||||||
.header("Content-Type", "application/json")
|
.header("Content-Type", "application/json")
|
||||||
.POST(HttpRequest.BodyPublishers.ofString(json));
|
.POST(HttpRequest.BodyPublishers.ofString(json));
|
||||||
@@ -170,7 +178,7 @@ public class OpenAiCompatibleProvider {
|
|||||||
if(health!=null) health.record(healthRoute,apiKey,response.statusCode(),(int)TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started));
|
if(health!=null) health.record(healthRoute,apiKey,response.statusCode(),(int)TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started));
|
||||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||||
String details;
|
String details;
|
||||||
try(var bounded=new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)))) {details=readErrorBody(bounded.body);}
|
try(var bounded=new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)),false)) {details=readErrorBody(bounded.body);}
|
||||||
if (requiresLargerOutputBudget(details)) {
|
if (requiresLargerOutputBudget(details)) {
|
||||||
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
|
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
|
||||||
}
|
}
|
||||||
@@ -190,7 +198,7 @@ public class OpenAiCompatibleProvider {
|
|||||||
response.body().close();
|
response.body().close();
|
||||||
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
|
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
|
||||||
}
|
}
|
||||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)));
|
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)),streaming);
|
||||||
} catch (ApiException exception) {
|
} catch (ApiException exception) {
|
||||||
throw exception;
|
throw exception;
|
||||||
} catch (HttpTimeoutException | SocketTimeoutException exception) {
|
} catch (HttpTimeoutException | SocketTimeoutException exception) {
|
||||||
@@ -233,11 +241,28 @@ public class OpenAiCompatibleProvider {
|
|||||||
private final InputStream body;
|
private final InputStream body;
|
||||||
private final AtomicBoolean expired=new AtomicBoolean();
|
private final AtomicBoolean expired=new AtomicBoolean();
|
||||||
private final ScheduledFuture<?> deadline;
|
private final ScheduledFuture<?> deadline;
|
||||||
|
private final boolean idleWatch;
|
||||||
|
private volatile ScheduledFuture<?> idle;
|
||||||
|
private volatile boolean closed;
|
||||||
private Runnable afterClose=()->{};
|
private Runnable afterClose=()->{};
|
||||||
public StreamResponse onClose(Runnable callback) {afterClose=callback;return this;}
|
public StreamResponse onClose(Runnable callback) {afterClose=callback;return this;}
|
||||||
private StreamResponse(InputStream body,long remainingMs) {
|
private StreamResponse(InputStream raw,long remainingMs,boolean streaming) {
|
||||||
this.body=body;
|
Runnable expire=() -> { expired.set(true); try { raw.close(); } catch(IOException ignored) {} };
|
||||||
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
|
// The idle watchdog only matters when it is shorter than the remaining total deadline.
|
||||||
|
this.idleWatch=streaming&&idleTimeoutMs<remainingMs;
|
||||||
|
this.body=!idleWatch?raw:new FilterInputStream(raw) {
|
||||||
|
@Override public int read() throws IOException { int value=super.read(); if(value!=-1) touch(expire); return value; }
|
||||||
|
@Override public int read(byte[] buffer,int offset,int length) throws IOException { int count=super.read(buffer,offset,length); if(count>0) touch(expire); return count; }
|
||||||
|
};
|
||||||
|
this.deadline=deadlines.schedule(expire,remainingMs,TimeUnit.MILLISECONDS);
|
||||||
|
if(idleWatch) touch(expire);
|
||||||
|
}
|
||||||
|
/** Received bytes prove the upstream is alive; restart the idle window. */
|
||||||
|
private void touch(Runnable expire) {
|
||||||
|
if(closed) return;
|
||||||
|
ScheduledFuture<?> previous=idle;
|
||||||
|
idle=deadlines.schedule(expire,idleTimeoutMs,TimeUnit.MILLISECONDS);
|
||||||
|
if(previous!=null) previous.cancel(false);
|
||||||
}
|
}
|
||||||
public void events(EventConsumer consumer) throws IOException {
|
public void events(EventConsumer consumer) throws IOException {
|
||||||
events(consumer,true);
|
events(consumer,true);
|
||||||
@@ -248,7 +273,7 @@ public class OpenAiCompatibleProvider {
|
|||||||
consumer.accept(chunk);
|
consumer.accept(chunk);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} finally {afterClose.run();afterClose=()->{};} }
|
@Override public void close() { closed=true; deadline.cancel(false); if(idle!=null) idle.cancel(false); try { body.close(); } catch(IOException ignored) {} finally {afterClose.run();afterClose=()->{};} }
|
||||||
}
|
}
|
||||||
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
|
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
|
||||||
@PreDestroy public void close() { deadlines.shutdownNow(); }
|
@PreDestroy public void close() { deadlines.shutdownNow(); }
|
||||||
|
|||||||
@@ -48,6 +48,15 @@ public class ChatBillingService {
|
|||||||
jdbc.update("UPDATE chat_billing_requests SET settlement_usage=?::jsonb, settlement_latency_ms=?, settlement_first_token_ms=?, expires_at=GREATEST(expires_at,now()+interval '5 minutes') WHERE id=? AND status='pending'",
|
jdbc.update("UPDATE chat_billing_requests SET settlement_usage=?::jsonb, settlement_latency_ms=?, settlement_first_token_ms=?, expires_at=GREATEST(expires_at,now()+interval '5 minutes') WHERE id=? AND status='pending'",
|
||||||
encode(normalizedUsage), Math.max(0,latencyMs), firstTokenMs, ticket.id());
|
encode(normalizedUsage), Math.max(0,latencyMs), firstTokenMs, ticket.id());
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* Extend a still-running call's hold so expiry recovery cannot release it mid-call.
|
||||||
|
* Only pending tickets move, and the hold never shrinks below its current expiry.
|
||||||
|
*/
|
||||||
|
public boolean renew(UUID id, long holdMs) {
|
||||||
|
long seconds=Math.max(60,(holdMs+999)/1000);
|
||||||
|
return jdbc.update("UPDATE chat_billing_requests SET expires_at=GREATEST(expires_at,now()+make_interval(secs=>?)) WHERE id=? AND status='pending'",seconds,id)>0;
|
||||||
|
}
|
||||||
|
|
||||||
@Transactional public void actualRoute(Ticket ticket,Map<String,Object> route) {
|
@Transactional public void actualRoute(Ticket ticket,Map<String,Object> route) {
|
||||||
var snapshots=jdbc.queryForList("SELECT procurement_pricing FROM model_price_history WHERE model_id=? ORDER BY id DESC LIMIT 1",route.get("id"));
|
var snapshots=jdbc.queryForList("SELECT procurement_pricing FROM model_price_history WHERE model_id=? ORDER BY id DESC LIMIT 1",route.get("id"));
|
||||||
Object cost=snapshots.isEmpty()?null:snapshots.getFirst().get("procurement_pricing");
|
Object cost=snapshots.isEmpty()?null:snapshots.getFirst().get("procurement_pricing");
|
||||||
|
|||||||
@@ -56,6 +56,12 @@ moyi:
|
|||||||
# provider's own response/limit.
|
# provider's own response/limit.
|
||||||
disable-model-concurrency: ${GATEWAY_DISABLE_MODEL_CONCURRENCY:false}
|
disable-model-concurrency: ${GATEWAY_DISABLE_MODEL_CONCURRENCY:false}
|
||||||
queue-wait-ms: ${GATEWAY_QUEUE_WAIT_MS:30000}
|
queue-wait-ms: ${GATEWAY_QUEUE_WAIT_MS:30000}
|
||||||
|
# Synchronous/stream upstream deadlines (seconds). Total is capped at 3600; first-byte and idle
|
||||||
|
# default to the total and never exceed it. Running calls renew wallet holds and concurrency
|
||||||
|
# leases, but a reverse proxy in front must also allow the longer read time.
|
||||||
|
upstream-timeout-seconds: ${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}
|
||||||
|
upstream-first-byte-timeout-seconds: ${GATEWAY_UPSTREAM_FIRST_BYTE_TIMEOUT_SECONDS:${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}}
|
||||||
|
upstream-idle-timeout-seconds: ${GATEWAY_UPSTREAM_IDLE_TIMEOUT_SECONDS:${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}}
|
||||||
queue-poll-ms: ${GATEWAY_QUEUE_POLL_MS:100}
|
queue-poll-ms: ${GATEWAY_QUEUE_POLL_MS:100}
|
||||||
tasks:
|
tasks:
|
||||||
max-queued-per-model: ${TASK_MAX_QUEUED_PER_MODEL:100}
|
max-queued-per-model: ${TASK_MAX_QUEUED_PER_MODEL:100}
|
||||||
|
|||||||
@@ -450,6 +450,22 @@ class PlatformAdminPostgresTest {
|
|||||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
|
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
|
||||||
assertEquals(2,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE status='failed' AND cost_amount=0",Integer.class));
|
assertEquals(2,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE status='failed' AND cost_amount=0",Integer.class));
|
||||||
}
|
}
|
||||||
|
@Test void renewedRunningCallIsNotReleasedByExpiryRecovery() {
|
||||||
|
var f=chatFixture();
|
||||||
|
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","renewal test",meta);
|
||||||
|
var running=f.reserve(null);
|
||||||
|
jdbc.update("UPDATE chat_billing_requests SET expires_at=now()+interval '5 seconds' WHERE id=?",running.id());
|
||||||
|
assertTrue(f.billing().renew(running.id(),1_800_000));
|
||||||
|
assertTrue(jdbc.queryForObject("SELECT expires_at>now()+interval '29 minutes' FROM chat_billing_requests WHERE id=?",Boolean.class,running.id()));
|
||||||
|
// Renewal never shortens an existing hold.
|
||||||
|
assertTrue(f.billing().renew(running.id(),60_000));
|
||||||
|
assertTrue(jdbc.queryForObject("SELECT expires_at>now()+interval '29 minutes' FROM chat_billing_requests WHERE id=?",Boolean.class,running.id()));
|
||||||
|
f.billing().recoverExpired();
|
||||||
|
assertEquals("pending",jdbc.queryForObject("SELECT status FROM chat_billing_requests WHERE id=?",String.class,running.id()));
|
||||||
|
assertTrue(jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum()>0);
|
||||||
|
f.billing().fail(running,"TEST_DONE","done",1);
|
||||||
|
assertFalse(f.billing().renew(running.id(),1_800_000));
|
||||||
|
}
|
||||||
@Test void unexpectedProviderOverrunIsRecordedAsDebtAndBlocksNewCalls() {
|
@Test void unexpectedProviderOverrunIsRecordedAsDebtAndBlocksNewCalls() {
|
||||||
var f=chatFixture();
|
var f=chatFixture();
|
||||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||||
|
|||||||
+15
@@ -73,6 +73,21 @@ class FairAdmissionRedisTest {
|
|||||||
keyHolder.close();
|
keyHolder.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test void renewalExtendsOnlyLeasesStillHeld() throws Exception {
|
||||||
|
var admission = guard(0).acquire(model, UUID.randomUUID(), 0);
|
||||||
|
String slots = prefix + "gateway:slots:model:fair-model";
|
||||||
|
String token = redis.opsForZSet().range(slots, 0, 0).iterator().next();
|
||||||
|
double before = redis.opsForZSet().score(slots, token);
|
||||||
|
assertTrue(admission.renew(1_800_000));
|
||||||
|
assertTrue(redis.opsForZSet().score(slots, token) > before + 1_000_000);
|
||||||
|
redis.opsForZSet().remove(slots, token);
|
||||||
|
// A lost lease is reported, never silently recreated on top of another caller's capacity.
|
||||||
|
assertFalse(admission.renew(1_800_000));
|
||||||
|
assertNull(redis.opsForZSet().score(slots, token));
|
||||||
|
admission.close();
|
||||||
|
assertFalse(admission.renew(1_800_000));
|
||||||
|
}
|
||||||
|
|
||||||
private long zcard(String kind) {
|
private long zcard(String kind) {
|
||||||
Long value = redis.opsForZSet().zCard(prefix + "gateway:" + kind + ":model:fair-model");
|
Long value = redis.opsForZSet().zCard(prefix + "gateway:" + kind + ":model:fair-model");
|
||||||
return value == null ? 0 : value;
|
return value == null ? 0 : value;
|
||||||
|
|||||||
+20
@@ -96,6 +96,26 @@ class GatewayRequestExecutionServiceTest {
|
|||||||
verify(admission).close();
|
verify(admission).close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void runningCallRenewsItsHoldAndLeaseUntilTheWorkReturns() throws Exception {
|
||||||
|
ChatBillingService billing = mock(ChatBillingService.class);
|
||||||
|
var ticket = new ChatBillingService.Ticket(UUID.randomUUID(), userId, null);
|
||||||
|
when(billing.reserve(key, model, normalized, null, context.metadata())).thenReturn(ticket);
|
||||||
|
var service = new GatewayRequestExecutionService(billing, new ApiRequestRiskPolicy(ApiRequestRiskEngine.allowAll()), 20);
|
||||||
|
ApiRequestAdmission admission = mock(ApiRequestAdmission.class);
|
||||||
|
when(admission.renew(anyLong())).thenReturn(true);
|
||||||
|
try {
|
||||||
|
assertEquals("done", service.execute(context, key, model, normalized, null, () -> admission,
|
||||||
|
ignored -> { Thread.sleep(150); return "done"; }, failure -> new ApiException("CALL_FAILED", "failed", 502), ApiException::getCode));
|
||||||
|
verify(billing, atLeast(2)).renew(eq(ticket.id()), anyLong());
|
||||||
|
verify(admission, atLeast(2)).renew(anyLong());
|
||||||
|
// Renewal stops with the call; a finished request must not keep its hold alive.
|
||||||
|
clearInvocations(billing);
|
||||||
|
Thread.sleep(100);
|
||||||
|
verify(billing, never()).renew(any(), anyLong());
|
||||||
|
} finally { service.shutdown(); }
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void workFailureIsRecordedAndAdmissionLeaseCloses() {
|
void workFailureIsRecordedAndAdmissionLeaseCloses() {
|
||||||
ChatBillingService billing = mock(ChatBillingService.class);
|
ChatBillingService billing = mock(ChatBillingService.class);
|
||||||
|
|||||||
+85
@@ -0,0 +1,85 @@
|
|||||||
|
package com.moyi.controlplane.provider;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.moyi.controlplane.common.ApiException;
|
||||||
|
import com.moyi.controlplane.security.SecretCipher;
|
||||||
|
import com.sun.net.httpserver.HttpServer;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
import org.springframework.mock.env.MockEnvironment;
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
|
/** Long streams survive while bytes flow; a silent upstream ends at the idle window, not the total deadline. */
|
||||||
|
class StreamDeadlineTest {
|
||||||
|
private OpenAiCompatibleProvider provider(HttpServer server, int total, int idle) {
|
||||||
|
var env = new MockEnvironment()
|
||||||
|
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||||
|
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||||
|
.withProperty("moyi.gateway.upstream-timeout-seconds", Integer.toString(total))
|
||||||
|
.withProperty("moyi.gateway.upstream-idle-timeout-seconds", Integer.toString(idle));
|
||||||
|
return new OpenAiCompatibleProvider(new ObjectMapper(), env, mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private HttpServer sse(long gapMs, int events, boolean finish) throws Exception {
|
||||||
|
var server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
|
server.createContext("/v1/chat/completions", exchange -> {
|
||||||
|
exchange.getResponseHeaders().set("Content-Type", "text/event-stream");
|
||||||
|
exchange.sendResponseHeaders(200, 0);
|
||||||
|
try (var out = exchange.getResponseBody()) {
|
||||||
|
for (int i = 0; i < events; i++) {
|
||||||
|
out.write(("data: {\"i\":" + i + "}\n\n").getBytes(StandardCharsets.UTF_8));
|
||||||
|
out.flush();
|
||||||
|
Thread.sleep(gapMs);
|
||||||
|
}
|
||||||
|
if (finish) out.write("data: [DONE]\n\n".getBytes(StandardCharsets.UTF_8));
|
||||||
|
else Thread.sleep(5_000);
|
||||||
|
} catch (Exception ignored) { }
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
return server;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void streamLongerThanTheIdleWindowSucceedsWhileEventsKeepArriving() throws Exception {
|
||||||
|
var server = sse(400, 8, true);
|
||||||
|
var provider = provider(server, 10, 1);
|
||||||
|
try (var stream = provider.stream("provider-deepseek", "model", Map.of("messages", List.of()))) {
|
||||||
|
var chunks = new ArrayList<Map<String,Object>>();
|
||||||
|
long started = System.nanoTime();
|
||||||
|
stream.events(chunks::add);
|
||||||
|
assertEquals(8, chunks.size());
|
||||||
|
assertTrue(System.nanoTime() - started > 2_000_000_000L, "the stream outlived the 1s idle window");
|
||||||
|
} finally { provider.close(); server.stop(0); }
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void silentUpstreamTimesOutAtTheIdleWindow() throws Exception {
|
||||||
|
var server = sse(0, 1, false);
|
||||||
|
var provider = provider(server, 10, 1);
|
||||||
|
try (var stream = provider.stream("provider-deepseek", "model", Map.of("messages", List.of()))) {
|
||||||
|
long started = System.nanoTime();
|
||||||
|
var error = assertThrows(ApiException.class, () -> stream.events(chunk -> { }));
|
||||||
|
assertEquals("UPSTREAM_PROVIDER_TIMEOUT", error.getCode());
|
||||||
|
long elapsedMs = (System.nanoTime() - started) / 1_000_000;
|
||||||
|
assertTrue(elapsedMs < 4_000, "ended by the idle window, not the 10s total: " + elapsedMs);
|
||||||
|
} finally { provider.close(); server.stop(0); }
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void totalDeadlineIsCappedAndIdleNeverExceedsIt() throws Exception {
|
||||||
|
var server = sse(0, 1, true);
|
||||||
|
var env = new MockEnvironment()
|
||||||
|
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||||
|
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||||
|
.withProperty("moyi.gateway.upstream-timeout-seconds", "999999")
|
||||||
|
.withProperty("moyi.gateway.upstream-idle-timeout-seconds", "999999");
|
||||||
|
var provider = new OpenAiCompatibleProvider(new ObjectMapper(), env, mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
|
||||||
|
try {
|
||||||
|
assertEquals(OpenAiCompatibleProvider.MAX_TIMEOUT_SECONDS, org.springframework.test.util.ReflectionTestUtils.getField(provider, "timeoutSeconds"));
|
||||||
|
assertEquals(1000L * OpenAiCompatibleProvider.MAX_TIMEOUT_SECONDS, org.springframework.test.util.ReflectionTestUtils.getField(provider, "idleTimeoutMs"));
|
||||||
|
} finally { provider.close(); server.stop(0); }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,7 +24,7 @@
|
|||||||
| SEC-07 | 整体安全审查与生产验收 | 未完成:已补验证码/账号原子性;本轮进一步分离加密投递、统一已知/未知账号重置受理和限额,外部投递不再持 DB 事务;管理接口全审查、时序残余、默认配置、依赖与备份恢复仍需继续 | 本轮新增 22 项投递数据库回归,全量 390 项通过;真实邮件/短信与规格驱动验收未执行,禁止宣称“全仓库安全” |
|
| SEC-07 | 整体安全审查与生产验收 | 未完成:已补验证码/账号原子性;本轮进一步分离加密投递、统一已知/未知账号重置受理和限额,外部投递不再持 DB 事务;管理接口全审查、时序残余、默认配置、依赖与备份恢复仍需继续 | 本轮新增 22 项投递数据库回归,全量 390 项通过;真实邮件/短信与规格驱动验收未执行,禁止宣称“全仓库安全” |
|
||||||
| COD-01 | Codex 协议能力及验收矩阵 | 未实现完整兼容;保留有限文本 Responses 的明确拒绝语义 | 指定客户端版本、模型、工具和计费联合验收 |
|
| COD-01 | Codex 协议能力及验收矩阵 | 未实现完整兼容;保留有限文本 Responses 的明确拒绝语义 | 指定客户端版本、模型、工具和计费联合验收 |
|
||||||
| COD-02 | Responses WebSocket 与响应归属续接 | 未实现,不可宣传完整 Codex 支持 | 原账号/凭据版本归属、断线、取消、租约、费用一致性 |
|
| COD-02 | Responses WebSocket 与响应归属续接 | 未实现,不可宣传完整 Codex 支持 | 原账号/凭据版本归属、断线、取消、租约、费用一致性 |
|
||||||
| COD-03 | 长请求连接/空闲/总期限与续租 | 未完成,不只修改 90 秒常量 | 并发租约、钱包预留恢复、长流/取消共同验证 |
|
| COD-03 | 长请求连接/空闲/总期限与续租 | 部分完成:同步/流式上游拆分首字节、空闲、总期限(总期限上限 3600 秒,默认仍 90 秒);执行期间每 60 秒续期钱包预留与 Redis 并发租约,结束即停,丢失的租约不会被重建。部署侧 nginx `proxy_read_timeout 120s` 与云端 Compose 未透传新变量,放宽前须同步调整;原生请求 256 KiB / 32K 输出上限未动 | 并发租约、钱包预留恢复、长流/取消共同验证 |
|
||||||
| SCH-01 | 独立上游账号生命周期与调度 | 仅共享 Key 轮换/冷却/指纹,非完整账号实体 | 凭据版本、额度、模型范围、权重及解释能力 |
|
| SCH-01 | 独立上游账号生命周期与调度 | 仅共享 Key 轮换/冷却/指纹,非完整账号实体 | 凭据版本、额度、模型范围、权重及解释能力 |
|
||||||
| SCH-02 | 公平排队 / 取消 / 共享等待预算 | 部分完成:模型+Key 配对准入改为模型级 FIFO(到达顺序 ZSET,过期/超时/放弃即出队;仅因自身 Key 满而等待者不占模型队位),真实 Redis 回归证明旧脚本会被插队;跨路由共享等待预算、优先级与横向压测仍缺 | 多客户端饥饿/FIFO/取消回收与横向并发验证 |
|
| SCH-02 | 公平排队 / 取消 / 共享等待预算 | 部分完成:模型+Key 配对准入改为模型级 FIFO(到达顺序 ZSET,过期/超时/放弃即出队;仅因自身 Key 满而等待者不占模型队位),真实 Redis 回归证明旧脚本会被插队;跨路由共享等待预算、优先级与横向压测仍缺 | 多客户端饥饿/FIFO/取消回收与横向并发验证 |
|
||||||
| OPS-01 | request/attempt 诊断、配置版本、质量门禁 | 部分:已有最终审计/健康/部分快照;本轮加依赖方向测试、CI/Dependabot 配置,远端未运行;逐 attempt/完整配置版本仍缺 | 本地门禁不等于 GitHub CI、CVE 扫描或真实协议验收 |
|
| OPS-01 | request/attempt 诊断、配置版本、质量门禁 | 部分:已有最终审计/健康/部分快照;本轮加依赖方向测试、CI/Dependabot 配置,远端未运行;逐 attempt/完整配置版本仍缺 | 本地门禁不等于 GitHub CI、CVE 扫描或真实协议验收 |
|
||||||
@@ -56,3 +56,5 @@
|
|||||||
2026-10-22 音视频规范化增量(MED-01/02/03):审查时修复音频参数校验中重复的模式变量名(编译错误)与旧目录测试假设,并收紧裸字符串结果只接受 URL。新增结果规范化、音频发送前拒绝、无文件完成结算一次等回归。后端 `mvn -B -o clean verify` **399 项,0 失败/错误/跳过**(独立 PostgreSQL 15.18/Redis 7,2026-10-22 11:47);前端构建与 27 项测试通过。没有真实供应商音视频生成,MED-04/05 仍未完成。
|
2026-10-22 音视频规范化增量(MED-01/02/03):审查时修复音频参数校验中重复的模式变量名(编译错误)与旧目录测试假设,并收紧裸字符串结果只接受 URL。新增结果规范化、音频发送前拒绝、无文件完成结算一次等回归。后端 `mvn -B -o clean verify` **399 项,0 失败/错误/跳过**(独立 PostgreSQL 15.18/Redis 7,2026-10-22 11:47);前端构建与 27 项测试通过。没有真实供应商音视频生成,MED-04/05 仍未完成。
|
||||||
|
|
||||||
2026-10-22 管理授权审查与公平准入增量(SEC-07 / SCH-02):只读审查全部控制器,未发现未鉴权端点、先副作用后校验或跨用户 IDOR。已修复:①修改渠道 Base URL 时必须重新填写凭据,阻止把已存密钥重定向到新主机,审计记录新旧地址;②普通用户 `/api/models` 只返回渠道 id/name/status,不再暴露 Base URL 与凭据掩码;③管理端用量 CSV 导出中和公式前缀。**待决策未改**:`admin` 与 `super_admin` 在钱包加款(含给自己)、定价、渠道、成本核账、用量删除上权限相同;邀请码明文对所有管理员可见;管理变更审计缺 IP/前后值;角色校验未在行锁事务内复核。后端 `mvn -B -o clean verify` **403 项,0 失败/错误/跳过**(2026-10-22 12:06)。
|
2026-10-22 管理授权审查与公平准入增量(SEC-07 / SCH-02):只读审查全部控制器,未发现未鉴权端点、先副作用后校验或跨用户 IDOR。已修复:①修改渠道 Base URL 时必须重新填写凭据,阻止把已存密钥重定向到新主机,审计记录新旧地址;②普通用户 `/api/models` 只返回渠道 id/name/status,不再暴露 Base URL 与凭据掩码;③管理端用量 CSV 导出中和公式前缀。**待决策未改**:`admin` 与 `super_admin` 在钱包加款(含给自己)、定价、渠道、成本核账、用量删除上权限相同;邀请码明文对所有管理员可见;管理变更审计缺 IP/前后值;角色校验未在行锁事务内复核。后端 `mvn -B -o clean verify` **403 项,0 失败/错误/跳过**(2026-10-22 12:06)。
|
||||||
|
|
||||||
|
2026-10-22 长请求底座增量(COD-03):`OpenAiCompatibleProvider` 新增 `GATEWAY_UPSTREAM_FIRST_BYTE_TIMEOUT_SECONDS`/`GATEWAY_UPSTREAM_IDLE_TIMEOUT_SECONDS`,流式响应有字节到达即重置空闲窗口;`GatewayRequestExecutionService` 在调用期间续期 `chat_billing_requests.expires_at`(只延长不缩短,仅 pending)和全部 Redis 租约(仅续仍持有的 token)。默认值不变,部署后行为与之前一致。后端 `mvn -B -o clean verify` **409 项,0 失败/错误/跳过**(2026-10-22 15:23)。另在本机 Docker(`infra/moyi` + `docker-compose.local.override.yml`,仅监听 127.0.0.1:18088)完成启动冒烟:健康检查、超级管理员登录、目录字段收敛、Key 创建、Mock 同步与 SSE 调用和用量记录、401/403/协议拒绝;Tripo 3D 因本地无凭据按预期失败且预留归零。
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# Local test override: use prebuilt artifacts via Dockerfile.runtime and bind only to localhost.
|
||||||
|
services:
|
||||||
|
moyi-api:
|
||||||
|
build:
|
||||||
|
context: ../../control-plane-java
|
||||||
|
dockerfile: Dockerfile.runtime
|
||||||
|
frontend:
|
||||||
|
build:
|
||||||
|
context: ../../prototype
|
||||||
|
dockerfile: Dockerfile.runtime
|
||||||
|
ports: !override
|
||||||
|
- "127.0.0.1:${MOYI_HTTP_PORT:-8080}:80"
|
||||||
Reference in new issue
Block a user