feat: stream chat and settle metered CNY usage atomically

This commit is contained in:
wlx committed 2026-09-20 01:36:22 +08:00
1 parent 7360cf3ebe
commit b46ee72f2d
28 files changed
+1098 -86

No files matched your search

@@ -8,10 +8,14 @@ import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import jakarta.annotation.PreDestroy;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
@@ -23,6 +27,8 @@ public class OpenAiCompatibleProvider {
private final JdbcTemplate jdbc;
private final SecretCipher secretCipher;
private final Map<String, ProviderConfig> providers;
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
private final int timeoutSeconds;
public OpenAiCompatibleProvider(
ObjectMapper objectMapper,
@@ -34,6 +40,7 @@ public class OpenAiCompatibleProvider {
this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
this.jdbc = jdbc;
this.secretCipher = secretCipher;
this.timeoutSeconds = Math.max(1,Math.min(90,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
this.providers = Map.of(
"provider-deepseek", new ProviderConfig(environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com")),
"provider-kimi", new ProviderConfig(environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1")),
@@ -46,29 +53,49 @@ public class OpenAiCompatibleProvider {
return provider != null && !provider.apiKey().isBlank();
}
public Map<String, Object> chat(String providerId, String upstreamModel, List<?> messages) {
public Map<String, Object> chat(String providerId, String upstreamModel, Map<String,Object> input) {
try(var response=open(providerId,upstreamModel,input,false)) {
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
return objectMapper.readValue(bytes, new com.fasterxml.jackson.core.type.TypeReference<>() {});
} catch(ApiException e) { throw e; }
catch(Exception e) { throw upstreamError(); }
}
public StreamResponse stream(String providerId, String upstreamModel, Map<String,Object> input) {
return open(providerId,upstreamModel,input,true);
}
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
ProviderConfig provider = resolve(providerId);
if (provider == null || provider.apiKey().isBlank()) {
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
}
Map<String, Object> body = new LinkedHashMap<>();
Map<String, Object> body = new LinkedHashMap<>(input);
body.put("model", upstreamModel);
body.put("messages", messages == null ? List.of() : messages);
body.put("stream", false);
body.put("stream", streaming);
if(streaming) body.put("stream_options",Map.of("include_usage",true));
else body.remove("stream_options");
String json = writeJson(body);
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(normalizeUrl(provider.baseUrl()) + "/chat/completions"))
.timeout(Duration.ofSeconds(90))
.timeout(Duration.ofSeconds(timeoutSeconds))
.header("Authorization", "Bearer " + provider.apiKey())
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(json))
.build();
try {
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
long started=System.nanoTime();
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502);
response.body().close();
throw upstreamError();
}
return objectMapper.readValue(response.body(), new com.fasterxml.jackson.core.type.TypeReference<>() {});
if(streaming && !response.headers().firstValue("Content-Type").orElse("").toLowerCase().startsWith("text/event-stream")) {
response.body().close();
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
}
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)));
} catch (ApiException exception) {
throw exception;
} catch (InterruptedException exception) {
@@ -79,6 +106,46 @@ public class OpenAiCompatibleProvider {
}
}
/** A hard body deadline is required: HttpRequest.timeout alone does not cover InputStream reads. */
public final class StreamResponse implements AutoCloseable {
private final InputStream body;
private final AtomicBoolean expired=new AtomicBoolean();
private final ScheduledFuture<?> deadline;
private StreamResponse(InputStream body,long remainingMs) {
this.body=body;
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
}
public void events(EventConsumer consumer) throws IOException {
var reader=new BufferedReader(new InputStreamReader(body,StandardCharsets.UTF_8));
StringBuilder event=new StringBuilder(), line=new StringBuilder();
long total=0;
int c;
// Bound both individual frames and the whole response before allocating unbounded strings.
while((c=reader.read())!=-1) {
if(expired.get()) throw new ApiException("UPSTREAM_PROVIDER_TIMEOUT","上游响应超时,本次未扣费",504);
if(++total>4*1024*1024 || line.length()>262144 || event.length()>262144) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
if(c=='\n') {
String text=line.toString(); line.setLength(0);
if(text.endsWith("\r")) text=text.substring(0,text.length()-1);
if(text.isEmpty()) {
if(!event.isEmpty()) {
String data=event.substring(0,event.length()-1); event.setLength(0);
if("[DONE]".equals(data)) return;
Map<String,Object> chunk=objectMapper.readValue(data,new com.fasterxml.jackson.core.type.TypeReference<>(){});
if(chunk.containsKey("error")) throw upstreamError();
consumer.accept(chunk);
}
} else if(text.startsWith("data:")) event.append(text.substring(5).stripLeading()).append('\n');
} else line.append((char)c);
}
throw new ApiException(expired.get()?"UPSTREAM_PROVIDER_TIMEOUT":"UPSTREAM_STREAM_INCOMPLETE","上游流式响应中断,本次未扣费",502);
}
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} }
}
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
@PreDestroy public void close() { deadlines.shutdownNow(); }
private ApiException upstreamError() { return new ApiException("UPSTREAM_PROVIDER_ERROR","上游模型供应商请求失败",502); }
private String normalizeUrl(String value) {
return value == null ? "" : value.replaceAll("/+$", "");
}