feat: stream chat and settle metered CNY usage atomically

This commit is contained in:
wlx committed 2026-09-20 01:36:22 +08:00
1 parent 7360cf3ebe
commit b46ee72f2d
28 files changed
+1098 -86

No files matched your search

@@ -221,11 +221,11 @@ public class CatalogService {
private void validatePricing(Object input) {
if (!(input instanceof Map<?,?> prices)) return;
for(String field: List.of("input","output")) {
for(String field: List.of("input","output","cachedInput")) {
Object amount=prices.get(field);
if(amount==null) continue;
try { if(new java.math.BigDecimal(String.valueOf(amount)).signum()<0) throw new NumberFormatException(); }
catch(NumberFormatException e) { throw new ApiException("INVALID_PRICE","价格必须是非负数"); }
try { var price=new java.math.BigDecimal(String.valueOf(amount)); if(price.signum()<0 || price.scale()>8 || price.compareTo(new java.math.BigDecimal("1000000"))>0) throw new NumberFormatException(); }
catch(NumberFormatException e) { throw new ApiException("INVALID_PRICE","价格须在 0–100 万之间,最多 8 位小数"); }
}
}
@@ -351,8 +351,13 @@ public class CatalogService {
result.put("tasks", Map.of("queued", 0, "running", 0, "succeeded", 0, "failed", 0));
result.put("series", List.of());
result.put("ledger", List.of());
List<java.math.BigDecimal> balances=jdbc.queryForList("SELECT balance FROM user_wallets WHERE user_id=?",java.math.BigDecimal.class,user.id());
result.put("availableCredit",balances.isEmpty()?java.math.BigDecimal.ZERO:balances.getFirst());
var balances=jdbc.queryForList("SELECT balance, reserved_balance FROM user_wallets WHERE user_id=?",user.id());
var balance=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("balance");
var reserved=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("reserved_balance");
result.put("availableCredit",balance.subtract(reserved));
result.put("balance",balance);
result.put("reservedCredit",reserved);
result.put("costThisMonth",jdbc.queryForObject("SELECT COALESCE(-SUM(amount),0) FROM wallet_entries WHERE user_id=? AND operation='usage' AND created_at >= date_trunc('month',now())",java.math.BigDecimal.class,user.id()));
result.put("ledger",jdbc.query("SELECT id, reason, amount, created_at FROM wallet_entries WHERE user_id=? ORDER BY created_at DESC LIMIT 100", (rs,n)->Map.of("id",rs.getString("id"),"title",rs.getString("reason"),"amount",rs.getBigDecimal("amount"),"createdAt",rs.getObject("created_at",OffsetDateTime.class)),user.id()));
return result;
}
@@ -22,7 +22,8 @@ public class WebConfig implements WebMvcConfigurer {
registry.addMapping("/**")
.allowedOrigins(allowedOrigins)
.allowedMethods("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS")
.allowedHeaders("Content-Type", "Authorization", "X-Moyi-Source")
.allowedHeaders("Content-Type", "Authorization", "X-Moyi-Source", "Idempotency-Key")
.exposedHeaders("X-Request-Id", "X-Moyi-Cost-CNY")
.allowCredentials(true)
.maxAge(3600);
}
@@ -0,0 +1,95 @@
package com.moyi.controlplane.gateway;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
/** Monetary calculations never use floating point or locally estimated completion usage. */
public final class ChatMeter {
private ChatMeter() {}
public record Rates(BigDecimal input, BigDecimal output, BigDecimal cachedInput) {
public Map<String,Object> snapshot() {
return Map.of("input",input,"output",output,"cachedInput",cachedInput,"unit","CNY / 1M tokens");
}
public BigDecimal cost(long prompt, long completion, long cached) {
return input.multiply(BigDecimal.valueOf(prompt-cached))
.add(cachedInput.multiply(BigDecimal.valueOf(cached)))
.add(output.multiply(BigDecimal.valueOf(completion)))
.divide(new BigDecimal("1000000"),8,RoundingMode.HALF_UP);
}
}
public record Tokens(long prompt, long completion, long cached) {}
public record Request(Map<String,Object> body, long reservedInputTokens, int maxOutputTokens) {}
public static Rates rates(Object value, boolean mock) {
if (mock) return new Rates(BigDecimal.ZERO,BigDecimal.ZERO,BigDecimal.ZERO);
if (!(value instanceof Map<?,?> prices) || !"CNY / 1M tokens".equals(prices.get("unit")))
throw new ApiException("MODEL_NOT_PRICED","该分组尚未配置人民币 Token 售价",503);
BigDecimal input=price(prices.get("input")), output=price(prices.get("output"));
BigDecimal cached=prices.get("cachedInput")==null?input:price(prices.get("cachedInput"));
return new Rates(input,output,cached);
}
private static BigDecimal price(Object value) {
try {
BigDecimal price=new BigDecimal(String.valueOf(value));
if(price.signum()<0 || price.compareTo(new BigDecimal("1000000"))>0 || price.scale()>8) throw new NumberFormatException();
return price;
} catch(Exception e) { throw new ApiException("MODEL_NOT_PRICED","请配置有效的输入、输出人民币售价(最多 8 位小数)",503); }
}
public static Tokens tokens(Map<String,Object> response) {
if(response.containsKey("error") || !(response.get("usage") instanceof Map<?,?> usage)) throw invalidUsage();
long prompt=count(usage.get("prompt_tokens")), completion=count(usage.get("completion_tokens"));
Object cached=usage.get("prompt_cache_hit_tokens");
if(usage.get("prompt_tokens_details") instanceof Map<?,?> details && details.get("cached_tokens")!=null) cached=details.get("cached_tokens");
long hits=cached==null?0:count(cached);
if(hits>prompt) throw invalidUsage();
return new Tokens(prompt,completion,hits);
}
private static long count(Object value) {
try {
if(!(value instanceof Number)) throw new NumberFormatException();
long number=new BigDecimal(value.toString()).longValueExact();
if(number<0 || number>1_000_000_000L) throw new NumberFormatException();
return number;
} catch(Exception e) { throw invalidUsage(); }
}
private static ApiException invalidUsage() { return new ApiException("UPSTREAM_USAGE_MISSING","上游未返回有效 Token 用量,本次未扣费",502); }
public static Request request(Map<String,Object> input, ObjectMapper json) {
if(!(input.get("messages") instanceof List<?> messages) || messages.isEmpty() || messages.size()>256)
throw new ApiException("INVALID_MESSAGES","messages 不能为空,且不能超过 256 条");
for(Object item:messages) {
if(!(item instanceof Map<?,?> message) || !(message.get("role") instanceof String)) throw new ApiException("INVALID_MESSAGES","消息格式无效");
Object content=message.get("content");
if(content instanceof List<?> parts) {
for(Object part:parts) if(!(part instanceof Map<?,?> p) || !"text".equals(p.get("type")) || !(p.get("text") instanceof String))
throw new ApiException("UNSUPPORTED_CONTENT","当前聊天计费支持文本和工具调用,图片、音频请使用对应接口");
} else if(content!=null && !(content instanceof String)) throw new ApiException("INVALID_MESSAGES","消息内容格式无效");
}
if(input.get("n")!=null && !"1".equals(input.get("n").toString())) throw new ApiException("UNSUPPORTED_N","当前仅支持 n=1");
if(input.containsKey("max_tokens") && input.containsKey("max_completion_tokens")) throw new ApiException("INVALID_MAX_TOKENS","请只设置一种输出 Token 上限");
String maxField=input.containsKey("max_completion_tokens")?"max_completion_tokens":"max_tokens";
int max=4096;
try {
if(input.containsKey(maxField)) max=new BigDecimal(input.get(maxField).toString()).intValueExact();
if(max<1 || max>32768) throw new NumberFormatException();
} catch(Exception e) { throw new ApiException("INVALID_MAX_TOKENS","输出 Token 上限须在 1–32768 之间"); }
var body=new LinkedHashMap<String,Object>();
// Do not let client credentials, alternate upstreams or multimodal billing bypass admission.
var fields=Set.of("model","messages","stream","temperature","top_p","stop","presence_penalty","frequency_penalty","seed","tools","tool_choice","parallel_tool_calls","response_format","reasoning_effort","thinking","n","max_tokens","max_completion_tokens","stream_options");
for(String field:input.keySet()) {
if(!fields.contains(field)) throw new ApiException("UNSUPPORTED_PARAMETER","当前不支持请求参数:"+field);
body.put(field,input.get(field));
}
body.put(maxField,max);
body.put("stream",Boolean.TRUE.equals(input.get("stream")));
try {
int bytes=json.writeValueAsBytes(body).length;
if(bytes>262144) throw new ApiException("REQUEST_TOO_LARGE","聊天请求不能超过 256 KiB",413);
// UTF-8 bytes plus framing is deliberately conservative; actual usage settles later.
return new Request(body,bytes+128L*messages.size()+1024,max);
} catch(java.io.IOException e) { throw new ApiException("INVALID_REQUEST","请求无法编码"); }
}
}
@@ -0,0 +1,53 @@
package com.moyi.controlplane.gateway;
import com.moyi.controlplane.common.ApiException;
import java.util.*;
/** Retain a bounded, useful history while forwarding every original SSE delta. */
final class ChatStreamCollector {
private final String model;
private final long started;
private final StringBuilder content=new StringBuilder(), reasoning=new StringBuilder();
private final Map<Integer,Map<String,Object>> tools=new TreeMap<>();
private Object usage, id, created, finish;
private Integer firstToken;
ChatStreamCollector(String model,long started) { this.model=model; this.started=started; }
void accept(Map<String,Object> chunk) {
if(chunk.get("id")!=null) id=chunk.get("id");
if(chunk.get("created")!=null) created=chunk.get("created");
if(chunk.get("usage") instanceof Map<?,?>) usage=chunk.get("usage");
if(chunk.get("choices") instanceof List<?> choices) for(Object raw:choices) {
if(!(raw instanceof Map<?,?> choice)) continue;
if(choice.get("finish_reason")!=null) finish=choice.get("finish_reason");
if(choice.get("delta") instanceof Map<?,?> delta) {
append(content,delta.get("content")); append(reasoning,delta.get("reasoning_content"));
if(delta.get("tool_calls") instanceof List<?> calls) for(Object item:calls) {
if(!(item instanceof Map<?,?> call) || !(call.get("index") instanceof Number index)) continue;
int i=index.intValue();
if(i<0 || i>=64) throw new ApiException("UPSTREAM_RESPONSE_INVALID","上游工具调用数量超过限制",502);
var tool=tools.computeIfAbsent(i,k -> new LinkedHashMap<>());
for(String field:List.of("id","type")) if(call.get(field)!=null) tool.put(field,call.get(field));
if(call.get("function") instanceof Map<?,?> function) {
@SuppressWarnings("unchecked") var target=(Map<String,Object>)tool.computeIfAbsent("function",k -> new LinkedHashMap<>());
for(String field:List.of("name","arguments")) if(function.get(field) instanceof String text) {
String value=String.valueOf(target.getOrDefault(field,""))+text;
target.put(field,value.substring(0,Math.min(8000,value.length())));
}
}
}
if(firstToken==null && ((delta.get("content") instanceof String s && !s.isEmpty()) || (delta.get("reasoning_content") instanceof String r && !r.isEmpty()) || delta.get("tool_calls")!=null)) firstToken=(int)(System.currentTimeMillis()-started);
}
}
}
private void append(StringBuilder target,Object value) { if(value instanceof String s && target.length()<8000) target.append(s,0,Math.min(s.length(),8000-target.length())); }
Integer firstTokenMs() { return firstToken; }
Map<String,Object> result() {
if(finish==null) throw new ApiException("UPSTREAM_STREAM_INCOMPLETE","上游未完成输出,本次未扣费",502);
var message=new LinkedHashMap<String,Object>(); message.put("role","assistant"); message.put("content",content.toString());
if(!reasoning.isEmpty()) message.put("reasoning_content",reasoning.toString());
if(!tools.isEmpty()) message.put("tool_calls",List.copyOf(tools.values()));
var result=new LinkedHashMap<String,Object>(); result.put("id",id); result.put("created",created); result.put("model",model); result.put("object","chat.completion");
result.put("usage",usage); result.put("choices",List.of(Map.of("index",0,"message",message,"finish_reason",finish)));
return result;
}
}
@@ -3,13 +3,15 @@ package com.moyi.controlplane.gateway;
import com.moyi.controlplane.catalog.CatalogService;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
import com.moyi.controlplane.usage.UsageAuditService;
import com.moyi.controlplane.wallet.ChatBillingService;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
@@ -24,13 +26,15 @@ public class GatewayController {
private final CatalogService catalogService;
private final OpenAiCompatibleProvider provider;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final ChatBillingService billing;
private final ObjectMapper json;
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, UsageAuditService usage) {
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, ChatBillingService billing, ObjectMapper json) {
this.catalogService = catalogService;
this.provider = provider;
this.guard = guard;
this.usage = usage;
this.billing = billing;
this.json = json;
}
@GetMapping("/models")
@@ -55,7 +59,9 @@ public class GatewayController {
@PostMapping("/chat/completions")
public ResponseEntity<Map<String, Object>> chatCompletions(
@RequestHeader(value = "Authorization", required = false) String authorization,
@RequestBody Map<String, Object> input
@RequestBody Map<String, Object> input,
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
HttpServletResponse servletResponse
) {
Map<String, Object> key = requireKey(authorization);
long startedAt = System.currentTimeMillis();
@@ -64,34 +70,89 @@ public class GatewayController {
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线", 404);
if (!catalogService.keyCanInvoke(key, modelId)) throw new ApiException("MODEL_FORBIDDEN", "当前 Key 无权调用该模型", 403);
if ("async".equals(model.get("mode"))) throw new ApiException("ASYNC_MODEL", "该模型需要使用异步任务接口", 400);
if (Boolean.TRUE.equals(input.get("stream"))) throw new ApiException("STREAM_NOT_IMPLEMENTED", "流式转发将在下一节点开放", 501);
String providerId=String.valueOf(model.get("providerId"));
boolean mock="provider-mock".equals(providerId);
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
ChatMeter.Request request=ChatMeter.request(input,json);
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
try (GatewayGuard.Slot slot = guard.acquire(model)) {
ResponseEntity<Map<String, Object>> result = invoke(key, modelId, model, input);
Map<String, Object> body = result.getBody();
usage.recordCall(UUID.fromString(String.valueOf(key.get("userId"))), UUID.fromString(String.valueOf(key.get("id"))), modelId, String.valueOf(model.get("providerId")), "succeeded", (int) (System.currentTimeMillis() - startedAt), number(body, "usage.prompt_tokens"), number(body, "usage.completion_tokens"), input, body, null, null);
return result;
} catch (ApiException exception) {
usage.recordCall(UUID.fromString(String.valueOf(key.get("userId"))), UUID.fromString(String.valueOf(key.get("id"))), modelId, String.valueOf(model.get("providerId")), "failed", (int) (System.currentTimeMillis() - startedAt), 0, 0, input, null, exception.getCode(), exception.getMessage());
throw exception;
var ticket=billing.reserve(key,model,request,idempotencyKey);
servletResponse.setHeader("X-Request-Id",ticket.id().toString());
try {
catalogService.touchKey(key);
if(Boolean.TRUE.equals(request.body().get("stream"))) {
stream(model,request.body(),ticket,servletResponse,startedAt,mock);
return null;
}
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),request.body());
body.put("model",modelId);
requireCompletion(body);
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(body);
} catch(Exception exception) {
ApiException error=exception instanceof ApiException e?e:new ApiException("CALL_INTERRUPTED","调用中断,请查看使用记录确认结算状态",502);
billing.fail(ticket,error.getCode(),error.getMessage(),elapsed(startedAt));
if(servletResponse.isCommitted()) {
try { event(servletResponse,Map.of("error",Map.of("code",error.getCode(),"message",error.getMessage()))); done(servletResponse); } catch(IOException ignored) { }
return null;
}
servletResponse.resetBuffer();
servletResponse.setContentType("application/json");
throw error;
}
}
}
private ResponseEntity<Map<String, Object>> invoke(Map<String, Object> key, String modelId, Map<String, Object> model, Map<String, Object> input) {
String providerId = String.valueOf(model.get("providerId"));
if (provider.isConfigured(providerId)) {
List<?> messages = input.get("messages") instanceof List<?> values ? values : List.of();
Map<String, Object> upstream = provider.chat(providerId, String.valueOf(model.get("upstreamModel")), messages);
upstream.put("model", modelId);
catalogService.touchKey(key);
return ResponseEntity.ok(upstream);
private void stream(Map<String,Object> model, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
String modelId=model.get("id").toString();
if(mock) {
var body=mockResponse(modelId,input);
prepareStream(response);
var choices=(List<?>)body.get("choices");
var message=((Map<?,?>)choices.getFirst()).get("message");
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",message))));
billing.settle(ticket,body,elapsed(started),elapsed(started));
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",Map.of(),"finish_reason","stop"))));
event(response,Map.of("choices",List.of(),"usage",body.get("usage"))); done(response); return;
}
if (!"provider-mock".equals(providerId)) {
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该供应商尚未配置密钥,不能执行真实调用", 503);
// Keep admission, the concurrency slot and the upstream stream alive on this request thread.
// Upstream HTTP errors are inspected before committing a 200/SSE response.
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),input)) {
var collector=new ChatStreamCollector(modelId,started);
prepareStream(response);
boolean[] disconnected={false};
upstream.events(chunk -> {
chunk.put("model",modelId);
collector.accept(chunk);
// A closed browser must not turn an otherwise completed provider call into free usage.
// Continue within the same hard deadline to obtain the final meter and settle once.
if(!disconnected[0]) try { event(response,chunk); } catch(IOException clientClosed) { disconnected[0]=true; }
});
var body=collector.result();
billing.settle(ticket,body,elapsed(started),collector.firstTokenMs());
if(!disconnected[0]) done(response);
}
}
private void prepareStream(HttpServletResponse response) {
response.setContentType("text/event-stream"); response.setCharacterEncoding("UTF-8");
response.setHeader("Cache-Control","no-cache"); response.setHeader("X-Accel-Buffering","no");
}
private void event(HttpServletResponse response,Object data) throws IOException {
response.getOutputStream().write(("data: "+json.writeValueAsString(data)+"\n\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
response.flushBuffer();
}
private void done(HttpServletResponse response) throws IOException {
response.getOutputStream().write("data: [DONE]\n\n".getBytes(java.nio.charset.StandardCharsets.UTF_8)); response.flushBuffer();
}
private int elapsed(long started) { return (int)Math.min(Integer.MAX_VALUE,System.currentTimeMillis()-started); }
private void requireCompletion(Map<String,Object> body) {
if(body.containsKey("error") || !(body.get("choices") instanceof List<?> choices) || choices.isEmpty()
|| !(choices.getFirst() instanceof Map<?,?> choice) || choice.get("finish_reason")==null)
throw new ApiException("UPSTREAM_RESPONSE_INVALID","上游未返回完整的聊天结果,本次未扣费",502);
}
private Map<String,Object> mockResponse(String modelId,Map<String,Object> input) {
String prompt = extractPrompt(input.get("messages"));
String content = "MOYI Java Mock 已收到:" + (prompt.isBlank() ? "(空消息)" : prompt);
catalogService.touchKey(key);
Map<String, Object> response = new LinkedHashMap<>();
response.put("id", "chatcmpl-" + UUID.randomUUID().toString().replace("-", "").substring(0, 12));
response.put("object", "chat.completion");
@@ -99,7 +160,7 @@ public class GatewayController {
response.put("model", modelId);
response.put("choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", content), "finish_reason", "stop")));
response.put("usage", Map.of("prompt_tokens", prompt.length(), "completion_tokens", content.length(), "total_tokens", prompt.length() + content.length()));
return ResponseEntity.status(HttpStatus.OK).body(response);
return response;
}
private Map<String, Object> requireKey(String authorization) {
@@ -117,12 +178,4 @@ public class GatewayController {
}).reduce((left, right) -> left + "\n" + right).orElse("");
}
private long number(Map<String, Object> body, String path) {
if (body == null) return 0;
Object usageValue = body.get("usage");
if (!(usageValue instanceof Map<?, ?> map)) return 0;
String key = path.substring(path.indexOf('.') + 1);
Object value = map.get(key);
return value instanceof Number number ? number.longValue() : 0;
}
}
@@ -8,10 +8,14 @@ import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import jakarta.annotation.PreDestroy;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
@@ -23,6 +27,8 @@ public class OpenAiCompatibleProvider {
private final JdbcTemplate jdbc;
private final SecretCipher secretCipher;
private final Map<String, ProviderConfig> providers;
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
private final int timeoutSeconds;
public OpenAiCompatibleProvider(
ObjectMapper objectMapper,
@@ -34,6 +40,7 @@ public class OpenAiCompatibleProvider {
this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
this.jdbc = jdbc;
this.secretCipher = secretCipher;
this.timeoutSeconds = Math.max(1,Math.min(90,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
this.providers = Map.of(
"provider-deepseek", new ProviderConfig(environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com")),
"provider-kimi", new ProviderConfig(environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1")),
@@ -46,29 +53,49 @@ public class OpenAiCompatibleProvider {
return provider != null && !provider.apiKey().isBlank();
}
public Map<String, Object> chat(String providerId, String upstreamModel, List<?> messages) {
public Map<String, Object> chat(String providerId, String upstreamModel, Map<String,Object> input) {
try(var response=open(providerId,upstreamModel,input,false)) {
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
return objectMapper.readValue(bytes, new com.fasterxml.jackson.core.type.TypeReference<>() {});
} catch(ApiException e) { throw e; }
catch(Exception e) { throw upstreamError(); }
}
public StreamResponse stream(String providerId, String upstreamModel, Map<String,Object> input) {
return open(providerId,upstreamModel,input,true);
}
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
ProviderConfig provider = resolve(providerId);
if (provider == null || provider.apiKey().isBlank()) {
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
}
Map<String, Object> body = new LinkedHashMap<>();
Map<String, Object> body = new LinkedHashMap<>(input);
body.put("model", upstreamModel);
body.put("messages", messages == null ? List.of() : messages);
body.put("stream", false);
body.put("stream", streaming);
if(streaming) body.put("stream_options",Map.of("include_usage",true));
else body.remove("stream_options");
String json = writeJson(body);
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(normalizeUrl(provider.baseUrl()) + "/chat/completions"))
.timeout(Duration.ofSeconds(90))
.timeout(Duration.ofSeconds(timeoutSeconds))
.header("Authorization", "Bearer " + provider.apiKey())
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(json))
.build();
try {
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
long started=System.nanoTime();
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502);
response.body().close();
throw upstreamError();
}
return objectMapper.readValue(response.body(), new com.fasterxml.jackson.core.type.TypeReference<>() {});
if(streaming && !response.headers().firstValue("Content-Type").orElse("").toLowerCase().startsWith("text/event-stream")) {
response.body().close();
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
}
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)));
} catch (ApiException exception) {
throw exception;
} catch (InterruptedException exception) {
@@ -79,6 +106,46 @@ public class OpenAiCompatibleProvider {
}
}
/** A hard body deadline is required: HttpRequest.timeout alone does not cover InputStream reads. */
public final class StreamResponse implements AutoCloseable {
private final InputStream body;
private final AtomicBoolean expired=new AtomicBoolean();
private final ScheduledFuture<?> deadline;
private StreamResponse(InputStream body,long remainingMs) {
this.body=body;
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
}
public void events(EventConsumer consumer) throws IOException {
var reader=new BufferedReader(new InputStreamReader(body,StandardCharsets.UTF_8));
StringBuilder event=new StringBuilder(), line=new StringBuilder();
long total=0;
int c;
// Bound both individual frames and the whole response before allocating unbounded strings.
while((c=reader.read())!=-1) {
if(expired.get()) throw new ApiException("UPSTREAM_PROVIDER_TIMEOUT","上游响应超时,本次未扣费",504);
if(++total>4*1024*1024 || line.length()>262144 || event.length()>262144) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
if(c=='\n') {
String text=line.toString(); line.setLength(0);
if(text.endsWith("\r")) text=text.substring(0,text.length()-1);
if(text.isEmpty()) {
if(!event.isEmpty()) {
String data=event.substring(0,event.length()-1); event.setLength(0);
if("[DONE]".equals(data)) return;
Map<String,Object> chunk=objectMapper.readValue(data,new com.fasterxml.jackson.core.type.TypeReference<>(){});
if(chunk.containsKey("error")) throw upstreamError();
consumer.accept(chunk);
}
} else if(text.startsWith("data:")) event.append(text.substring(5).stripLeading()).append('\n');
} else line.append((char)c);
}
throw new ApiException(expired.get()?"UPSTREAM_PROVIDER_TIMEOUT":"UPSTREAM_STREAM_INCOMPLETE","上游流式响应中断,本次未扣费",502);
}
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} }
}
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
@PreDestroy public void close() { deadlines.shutdownNow(); }
private ApiException upstreamError() { return new ApiException("UPSTREAM_PROVIDER_ERROR","上游模型供应商请求失败",502); }
private String normalizeUrl(String value) {
return value == null ? "" : value.replaceAll("/+$", "");
}
@@ -33,6 +33,24 @@ public class UsageAuditService {
this.objectMapper = objectMapper;
}
public void startChat(UUID id, UUID userId, UUID keyId, Map<String,Object> model, Map<String,Object> input) {
String agent=null, source="api";
var attributes=org.springframework.web.context.request.RequestContextHolder.getRequestAttributes();
if(attributes instanceof org.springframework.web.context.request.ServletRequestAttributes servlet) {
agent=servlet.getRequest().getHeader("User-Agent");
if("playground".equals(servlet.getRequest().getHeader("X-Moyi-Source"))) source="playground";
}
jdbc.update("INSERT INTO usage_events(id,request_id,user_id,api_key_id,model_id,provider_id,event_kind,status,input_payload,endpoint,call_mode,request_source,user_agent,reasoning_effort) VALUES (?,?,?,?,?,?,'call','running',?::jsonb,'/v1/chat/completions',?,?,?,?)",
id,id.toString(),userId,keyId,model.get("id"),model.get("providerId"),json(input),Boolean.TRUE.equals(input.get("stream"))?"stream":"sync",source,
agent==null?null:agent.substring(0,Math.min(1000,agent.length())),input.get("reasoning_effort")==null?null:String.valueOf(input.get("reasoning_effort")).substring(0,Math.min(32,String.valueOf(input.get("reasoning_effort")).length())));
}
public void finishChat(UUID id, String status, com.moyi.controlplane.gateway.ChatMeter.Tokens tokens,
int latency, Integer firstTokenMs, java.math.BigDecimal cost, Object output, String code, String message) {
jdbc.update("UPDATE usage_events SET status=?,input_units=?,output_units=?,cached_input_units=?,latency_ms=?,first_token_ms=?,cost_amount=?,output_payload=?::jsonb,error_code=?,error_message=?,updated_at=now() WHERE id=?",
status,tokens.prompt(),tokens.completion(),tokens.cached(),latency,firstTokenMs,cost,json(output),code,message,id);
}
public void recordCall(
UUID userId, UUID apiKeyId, String modelId, String providerId, String status,
int latencyMs, long inputUnits, long outputUnits, Object input, Object output,
@@ -263,7 +281,9 @@ public class UsageAuditService {
ObjectNode result = objectMapper.createObjectNode();
node.fields().forEachRemaining(entry -> {
String key = entry.getKey().toLowerCase();
if (key.contains("authorization") || key.contains("api_key") || key.contains("apikey") || key.contains("token") || key.contains("secret") || key.contains("password")) result.put(entry.getKey(), "[REDACTED]");
boolean usageField = java.util.Set.of("prompt_tokens", "completion_tokens", "total_tokens", "cached_tokens", "reasoning_tokens", "audio_tokens", "accepted_prediction_tokens", "rejected_prediction_tokens", "prompt_cache_hit_tokens", "prompt_cache_miss_tokens", "max_tokens", "max_completion_tokens").contains(key) && entry.getValue().isNumber();
boolean usageDetails = java.util.Set.of("prompt_tokens_details", "completion_tokens_details").contains(key) && entry.getValue().isObject();
if (key.contains("authorization") || key.contains("api_key") || key.contains("apikey") || (key.contains("token") && !usageField && !usageDetails) || key.contains("secret") || key.contains("password")) result.put(entry.getKey(), "[REDACTED]");
else result.set(entry.getKey(), sanitize(entry.getValue()));
});
return result;
@@ -0,0 +1,90 @@
package com.moyi.controlplane.wallet;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.gateway.ChatMeter;
import com.moyi.controlplane.usage.UsageAuditService;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.*;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
public class ChatBillingService {
private final JdbcTemplate jdbc;
private final ObjectMapper json;
private final UsageAuditService usage;
public ChatBillingService(JdbcTemplate jdbc, ObjectMapper json, UsageAuditService usage) { this.jdbc=jdbc; this.json=json; this.usage=usage; }
public record Ticket(UUID id, UUID userId, ChatMeter.Rates rates) {}
@Transactional
public Ticket reserve(Map<String,Object> key, Map<String,Object> model, ChatMeter.Request request, String idempotencyKey) {
UUID userId=UUID.fromString(key.get("userId").toString());
ChatMeter.Rates rates=ChatMeter.rates(model.get("pricing"),"provider-mock".equals(model.get("providerId")));
String requestHash=hash(encode(new TreeMap<>(request.body())));
String idem=null;
if(idempotencyKey!=null) {
if(idempotencyKey.isBlank() || idempotencyKey.length()>128) throw new ApiException("INVALID_IDEMPOTENCY_KEY","Idempotency-Key 长度须为 1–128");
idem=hash(idempotencyKey);
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))",userId+":"+idem);
var previous=jdbc.queryForList("SELECT id, request_hash FROM chat_billing_requests WHERE user_id=? AND idempotency_hash=?",userId,idem);
if(!previous.isEmpty()) throw new ApiException(requestHash.equals(previous.getFirst().get("request_hash"))?"DUPLICATE_REQUEST":"IDEMPOTENCY_CONFLICT","该请求编号已受理,请在使用记录中查看结果;不会重复调用或扣费",409);
}
BigDecimal hold=rates.input().max(rates.cachedInput()).multiply(BigDecimal.valueOf(request.reservedInputTokens()))
.add(rates.output().multiply(BigDecimal.valueOf(request.maxOutputTokens())))
.divide(new BigDecimal("1000000"),8,java.math.RoundingMode.CEILING);
jdbc.update("INSERT INTO user_wallets(user_id) VALUES (?) ON CONFLICT DO NOTHING",userId);
var wallet=jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE",userId);
BigDecimal available=((BigDecimal)wallet.get("balance")).subtract((BigDecimal)wallet.get("reserved_balance"));
if(available.compareTo(hold)<0) throw new ApiException("INSUFFICIENT_BALANCE","可用余额不足以预留本次调用费用,请充值或调低输出 Token 上限",402);
UUID id=UUID.randomUUID();
usage.startChat(id,userId,UUID.fromString(key.get("id").toString()),model,request.body());
// Longer than the upstream hard deadline and the concurrency lease, including cleanup margin.
jdbc.update("INSERT INTO chat_billing_requests(id,user_id,idempotency_hash,request_hash,reserved_amount,pricing,expires_at) VALUES (?,?,?,?,?,?::jsonb,now()+interval '5 minutes')",id,userId,idem,requestHash,hold,encode(rates.snapshot()));
jdbc.update("UPDATE user_wallets SET reserved_balance=reserved_balance+?,updated_at=now() WHERE user_id=?",hold,userId);
return new Ticket(id,userId,rates);
}
@Transactional
public BigDecimal settle(Ticket ticket, Map<String,Object> response, int latency, Integer firstTokenMs) {
var call=jdbc.queryForMap("SELECT * FROM chat_billing_requests WHERE id=? FOR UPDATE",ticket.id());
if("succeeded".equals(call.get("status"))) return (BigDecimal)call.get("cost_amount");
if(!"pending".equals(call.get("status"))) throw new ApiException("BILLING_REQUEST_CLOSED","调用已结束,不能重复结算",409);
var tokens=ChatMeter.tokens(response);
// Read the persisted snapshot, not editable catalog prices or caller-supplied rates.
ChatMeter.Rates rates=ChatMeter.rates(decode(call.get("pricing").toString()),false);
BigDecimal cost=rates.cost(tokens.prompt(),tokens.completion(),tokens.cached());
BigDecimal before=jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=? FOR UPDATE",BigDecimal.class,ticket.userId());
BigDecimal after=before.subtract(cost);
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=reserved_balance-?,updated_at=now() WHERE user_id=?",after,call.get("reserved_amount"),ticket.userId());
if(cost.signum()>0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_before,balance_after,reason,operation,requested_amount) VALUES (?,?,?,?,?,?,?,'usage',?)",UUID.randomUUID(),ticket.userId(),ticket.id(),cost.negate(),before,after,"模型调用 · "+ticket.id(),cost);
usage.finishChat(ticket.id(),"succeeded",tokens,latency,firstTokenMs,cost,response,null,null);
jdbc.update("UPDATE chat_billing_requests SET status='succeeded',cost_amount=? WHERE id=?",cost,ticket.id());
return cost;
}
@Transactional
public void fail(Ticket ticket, String code, String message, int latency) { release(ticket.id(),code,message,latency); }
private void release(UUID id, String code, String message, int latency) {
var call=jdbc.queryForMap("SELECT * FROM chat_billing_requests WHERE id=? FOR UPDATE",id);
if(!"pending".equals(call.get("status"))) return;
jdbc.update("UPDATE user_wallets SET reserved_balance=reserved_balance-?,updated_at=now() WHERE user_id=?",call.get("reserved_amount"),call.get("user_id"));
jdbc.update("UPDATE chat_billing_requests SET status='failed',cost_amount=0 WHERE id=?",id);
usage.finishChat(id,"failed",new ChatMeter.Tokens(0,0,0),latency,null,BigDecimal.ZERO,null,code,message);
}
@Scheduled(fixedDelayString="${moyi.billing.recovery-interval-ms:60000}")
@Transactional
public void recoverExpired() {
var ids=jdbc.queryForList("SELECT id FROM chat_billing_requests WHERE status='pending' AND expires_at<now() ORDER BY expires_at LIMIT 100 FOR UPDATE SKIP LOCKED",UUID.class);
for(UUID id:ids) release(id,"CALL_INTERRUPTED","调用中断,未取得可结算用量,预留额度已释放",0);
}
private String encode(Object value) { try { return json.writeValueAsString(value); } catch(Exception e) { throw new ApiException("INVALID_REQUEST","请求无法编码"); } }
private Map<String,Object> decode(String value) { try { return json.readValue(value,new com.fasterxml.jackson.core.type.TypeReference<>(){}); } catch(Exception e) { throw new IllegalStateException("Invalid billing snapshot",e); } }
private String hash(String value) { try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); } catch(Exception e) { throw new IllegalStateException(e); } }
}
@@ -25,7 +25,7 @@ public class WalletService {
public List<Map<String,Object>> entries(String token, UUID userId) {
auth.requireAdmin(token);
return jdbc.queryForList("SELECT e.id, e.amount, e.operation, e.balance_before AS \"balanceBefore\", e.balance_after AS \"balanceAfter\", e.currency, e.reason, e.created_at AS \"createdAt\", u.display_name AS actor FROM wallet_entries e JOIN users u ON u.id=e.actor_user_id WHERE e.user_id=? ORDER BY e.created_at DESC LIMIT 100",userId);
return jdbc.queryForList("SELECT e.id, e.request_id AS \"requestId\", e.amount, e.operation, e.balance_before AS \"balanceBefore\", e.balance_after AS \"balanceAfter\", e.currency, e.reason, e.created_at AS \"createdAt\", COALESCE(u.display_name,'系统计费') AS actor FROM wallet_entries e LEFT JOIN users u ON u.id=e.actor_user_id WHERE e.user_id=? ORDER BY e.created_at DESC LIMIT 100",userId);
}
@Transactional
@@ -64,7 +64,8 @@ public class WalletService {
BigDecimal balance=jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=? FOR UPDATE",BigDecimal.class,userId);
if(expected!=null && balance.compareTo(expected)!=0) throw new ApiException("BALANCE_CHANGED","余额已变化,请刷新后重新确认调整",409);
BigDecimal next=switch(operation) { case "decrease" -> balance.subtract(amount); case "set" -> amount; default -> balance.add(amount); };
if(next.signum()<0) throw new ApiException("INSUFFICIENT_BALANCE","减少金额不能超过当前余额",409);
BigDecimal reserved=jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,userId);
if(!"increase".equals(operation) && (next.signum()<0 || next.compareTo(reserved)<0)) throw new ApiException("INSUFFICIENT_BALANCE","减少后的余额不能低于 0 或正在调用的预留额度",409);
BigDecimal delta=next.subtract(balance);
UUID id=UUID.randomUUID();
jdbc.update("UPDATE user_wallets SET balance=?, updated_at=now() WHERE user_id=?",next,userId);
@@ -0,0 +1,26 @@
-- A hold does not consume credit. Only a completed, metered call creates a debit.
ALTER TABLE user_wallets ADD COLUMN reserved_balance numeric(20,8) NOT NULL DEFAULT 0 CHECK (reserved_balance >= 0);
-- If a provider exceeds the token reservation, retain the actual charge as debt.
-- New calls and administrative withdrawals cannot spend this debt.
ALTER TABLE user_wallets DROP CONSTRAINT user_wallets_balance_check;
ALTER TABLE wallet_entries DROP CONSTRAINT wallet_entries_balance_check;
ALTER TABLE wallet_entries ADD CONSTRAINT wallet_entries_balance_check CHECK (balance_after=balance_before+amount);
ALTER TABLE wallet_entries DROP CONSTRAINT wallet_entries_operation_check;
ALTER TABLE wallet_entries ADD CONSTRAINT wallet_entries_operation_check CHECK (operation IN ('increase','decrease','set','usage'));
ALTER TABLE wallet_entries ALTER COLUMN actor_user_id DROP NOT NULL;
ALTER TABLE usage_events ALTER COLUMN cost_amount TYPE numeric(20,8);
CREATE TABLE chat_billing_requests (
id uuid PRIMARY KEY REFERENCES usage_events(id),
user_id uuid NOT NULL REFERENCES users(id),
idempotency_hash varchar(64),
request_hash varchar(64) NOT NULL,
status varchar(16) NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','succeeded','failed')),
reserved_amount numeric(20,8) NOT NULL CHECK (reserved_amount >= 0),
pricing jsonb NOT NULL,
cost_amount numeric(20,8),
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(user_id,idempotency_hash)
);
CREATE INDEX chat_billing_expiration_idx ON chat_billing_requests(expires_at) WHERE status='pending';
@@ -7,6 +7,8 @@ import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.common.ApiExceptionHandler;
import com.moyi.controlplane.security.SecretCipher;
import com.moyi.controlplane.wallet.WalletService;
import com.moyi.controlplane.wallet.ChatBillingService;
import com.moyi.controlplane.gateway.ChatMeter;
import com.moyi.controlplane.catalog.CatalogController;
import com.moyi.controlplane.usage.*;
import jakarta.servlet.http.Cookie;
@@ -78,6 +80,8 @@ class PlatformAdminPostgresTest {
context.registerBean(AuthService.class, () -> new AuthService(jdbc, new PasswordHasher("test-pepper"), json, 168));
context.registerBean(CatalogService.class, () -> new CatalogService(jdbc, json, new SecretCipher("test-encryption")));
context.registerBean(WalletService.class, () -> new WalletService(jdbc, context.getBean(AuthService.class)));
context.registerBean(UsageAuditService.class, () -> new UsageAuditService(jdbc,json));
context.registerBean(ChatBillingService.class, () -> new ChatBillingService(jdbc,json,context.getBean(UsageAuditService.class)));
context.registerBean(VerificationService.class, () -> new VerificationService(jdbc, sender, "test-pepper"));
context.refresh();
auth = context.getBean(AuthService.class);
@@ -268,7 +272,7 @@ class PlatformAdminPostgresTest {
org.springframework.web.context.request.RequestContextHolder.setRequestAttributes(new org.springframework.web.context.request.ServletRequestAttributes(request));
try {
audit.recordCall(memberId,null,groupId,providerId,"succeeded",200,100,20,
Map.of("prompt","hello","authorization","hidden"),
Map.of("prompt","hello","authorization","hidden","accessToken","hidden","refresh_token","hidden","prompt_tokens","hidden"),
Map.of("usage",Map.of("prompt_tokens_details",Map.of("cached_tokens",80),"cache_creation_input_tokens",5)),null,null);
} finally { org.springframework.web.context.request.RequestContextHolder.resetRequestAttributes(); }
var saved=jdbc.queryForMap("SELECT * FROM usage_events WHERE user_id=?",memberId);
@@ -280,6 +284,7 @@ class PlatformAdminPostgresTest {
assertNull(saved.get("cost_amount"));
assertNull(saved.get("first_token_ms"));
assertFalse(saved.get("input_payload").toString().contains("hidden"));
assertEquals(80,json.readTree(saved.get("output_payload").toString()).path("usage").path("prompt_tokens_details").path("cached_tokens").asInt());
audit.recordTask(memberId,null,groupId,providerId,"local-timing-task","queued",Map.of("prompt","hello"),null,null,null);
jdbc.update("UPDATE usage_events SET created_at=now()-interval '2 seconds' WHERE provider_task_id='local-timing-task'");
audit.updateTask("local-timing-task","succeeded",Map.of("url","https://example.test/result"),null,null);
@@ -329,4 +334,166 @@ class PlatformAdminPostgresTest {
assertFalse(verification.capabilities().containsValue(true));
assertThrows(ApiException.class, () -> verification.send("unavailable@example.test",invite,"127.0.0.1"));
}
record ChatFixture(ChatBillingService billing, Map<String,Object> key, Map<String,Object> model, ChatMeter.Request request) {
ChatBillingService.Ticket reserve(String idem) { return billing.reserve(key,model,request,idem); }
}
ChatFixture chatFixture() {
var model=new LinkedHashMap<>(catalog.listModels(false).stream().filter(m -> "provider-deepseek".equals(m.get("providerId"))).findFirst().orElseThrow());
model.put("pricing",Map.of("input",2,"output",4,"cachedInput",1,"unit","CNY / 1M tokens"));
var created=catalog.createKey(memberToken,Map.of("name","Local billing test","modelIds",List.of(model.get("id"))),auth);
var key=catalog.authenticateKey(created.get("key").toString());
var input=ChatMeter.request(Map.of("model",model.get("id"),"messages",List.of(Map.of("role","user","content","你好")),"max_tokens",16),json);
return new ChatFixture(context.getBean(ChatBillingService.class),key,model,input);
}
Map<String,Object> meteredResult() {
return Map.of("choices",List.of(Map.of("message",Map.of("content","OK"),"finish_reason","stop")),"usage",Map.of("prompt_tokens",100,"completion_tokens",50,"prompt_tokens_details",Map.of("cached_tokens",60)));
}
@Test void actualUsageSettlesOnceWithSnapshotAndAtomicHistory() throws Exception {
var f=chatFixture();
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
var ticket=f.reserve("same-call");
var held=jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId);
assertTrue(held.signum()>0);
assertEquals("DUPLICATE_REQUEST",assertThrows(ApiException.class,()->f.reserve("same-call")).getCode());
f.model().put("pricing",Map.of("input",999,"output",999,"unit","CNY / 1M tokens"));
try(var executor=Executors.newFixedThreadPool(2)) {
var a=executor.submit(()->f.billing().settle(ticket,meteredResult(),20,5));
var b=executor.submit(()->f.billing().settle(ticket,meteredResult(),20,5));
assertEquals(new BigDecimal("0.00034000"),a.get()); assertEquals(a.get(),b.get());
}
f.billing().fail(ticket,"LATE_FAILURE","Client closed after settlement",20);
assertEquals(new BigDecimal("0.99966000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE request_id=?",Integer.class,ticket.id()));
assertEquals(new BigDecimal("0.00034000"),jdbc.queryForObject("SELECT cost_amount FROM usage_events WHERE id=?",BigDecimal.class,ticket.id()));
assertEquals(ticket.id().toString(),jdbc.queryForObject("SELECT request_id FROM usage_events WHERE id=?",String.class,ticket.id()));
assertEquals(100,json.readTree(jdbc.queryForObject("SELECT output_payload::text FROM usage_events WHERE id=?",String.class,ticket.id())).at("/usage/prompt_tokens").intValue());
assertEquals(2,wallet.entries(ownerToken,memberId).size());
}
@Test void reservationBlocksConcurrentSpendingAndAdminWithdrawals() throws Exception {
var f=chatFixture();
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
jdbc.update("UPDATE user_wallets SET balance=0.004 WHERE user_id=?",memberId);
try(var executor=Executors.newFixedThreadPool(2)) {
var ready=new CountDownLatch(1);
Callable<Object> attempt=()->{ ready.await(); try { return f.reserve(null); } catch(ApiException e) { return e.getCode(); } };
var a=executor.submit(attempt); var b=executor.submit(attempt); ready.countDown();
var results=List.of(a.get(),b.get());
assertEquals(1,results.stream().filter(x -> x instanceof ChatBillingService.Ticket).count());
assertTrue(results.contains("INSUFFICIENT_BALANCE"));
}
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->wallet.adjust(ownerToken,memberId,UUID.randomUUID(),"set","0","reserved balance test","0.004",meta)).getCode());
}
@Test void invalidUsageAndInterruptedCallsReleaseWithoutCharging() {
var f=chatFixture();
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
var ticket=f.reserve(null);
assertEquals("UPSTREAM_USAGE_MISSING",assertThrows(ApiException.class,()->f.billing().settle(ticket,Map.of(),1,null)).getCode());
f.billing().fail(ticket,"UPSTREAM_USAGE_MISSING","No actual usage",1);
f.billing().fail(ticket,"RETRY","Already released",1);
var abandoned=f.reserve(null);
jdbc.update("UPDATE chat_billing_requests SET expires_at=now()-interval '1 second' WHERE id=?",abandoned.id());
f.billing().recoverExpired(); f.billing().recoverExpired();
assertEquals(new BigDecimal("1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
assertEquals(2,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE status='failed' AND cost_amount=0",Integer.class));
}
@Test void unexpectedProviderOverrunIsRecordedAsDebtAndBlocksNewCalls() {
var f=chatFixture();
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
var ticket=f.reserve(null);
f.billing().settle(ticket,Map.of("usage",Map.of("prompt_tokens",1_000_000,"completion_tokens",0)),10,null);
assertEquals(new BigDecimal("-1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"0.50","partial debt repayment",meta);
assertEquals(new BigDecimal("-0.50000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
}
@Test void unpricedRoutesAndEmptyBalancesFailBeforeCreatingHolds() {
var f=chatFixture();
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
f.model().remove("pricing");
assertEquals("MODEL_NOT_PRICED",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
f.model().put("pricing",Map.of("input",1,"output",2,"unit","USD / 1M tokens"));
assertEquals("MODEL_NOT_PRICED",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM chat_billing_requests",Integer.class));
}
@Test void gatewaySyncAndStreamRecordTheSameChargeAndPropagateFailures() throws Exception {
var f=chatFixture();
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","gateway acceptance",meta);
jdbc.update("UPDATE model_catalog SET pricing=?::jsonb WHERE id=?",json.writeValueAsString(f.model().get("pricing")),f.model().get("id"));
String key=catalog.createKey(memberToken,Map.of("name","Local gateway test","modelIds",List.of(f.model().get("id"))),auth).get("key").toString();
var upstream=com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress("127.0.0.1",0),0);
var mode=new java.util.concurrent.atomic.AtomicReference<>("ok");
var count=new java.util.concurrent.atomic.AtomicInteger();
upstream.createContext("/v1/chat/completions",exchange->{
count.incrementAndGet();
var input=json.readTree(exchange.getRequestBody());
if(mode.get().equals("http-error")) { exchange.sendResponseHeaders(500,-1); exchange.close(); return; }
if(input.path("stream").asBoolean()) {
exchange.getResponseHeaders().set("Content-Type","text/event-stream"); exchange.sendResponseHeaders(200,0);
try(var out=exchange.getResponseBody()) {
String frames="data: {\"model\":\"private-alias\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\n";
if(!mode.get().equals("no-usage")) frames+="data: "+json.writeValueAsString(Map.of("choices",List.of(),"usage",meteredResult().get("usage")))+"\n\n";
out.write((frames+"data: [DONE]\n\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
}
} else {
byte[] body=json.writeValueAsBytes(meteredResult()); exchange.getResponseHeaders().set("Content-Type","application/json"); exchange.sendResponseHeaders(200,body.length);
try(var out=exchange.getResponseBody()) { out.write(body); }
}
}); upstream.start();
var environment=new org.springframework.mock.env.MockEnvironment().withProperty("moyi.providers.deepseek.api-key","local-fake")
.withProperty("moyi.providers.deepseek.base-url","http://127.0.0.1:"+upstream.getAddress().getPort()+"/v1");
var provider=new com.moyi.controlplane.provider.OpenAiCompatibleProvider(json,environment,jdbc,new SecretCipher("local-test"));
try {
var controller=new com.moyi.controlplane.gateway.GatewayController(catalog,provider,org.mockito.Mockito.mock(com.moyi.controlplane.gateway.GatewayGuard.class),f.billing(),json);
var http=MockMvcBuilders.standaloneSetup(controller).setControllerAdvice(new ApiExceptionHandler()).build();
var input=new LinkedHashMap<>(f.request().body());
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("Idempotency-Key","retry-test").contentType("application/json").content(json.writeValueAsBytes(input)))
.andExpect(status().isOk()).andExpect(header().string("X-Moyi-Cost-CNY","0.00034000")).andExpect(jsonPath("$.model").value(f.model().get("id")));
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("Idempotency-Key","retry-test").contentType("application/json").content(json.writeValueAsBytes(input))).andExpect(status().isConflict());
assertEquals(1,count.get());
input.put("stream",true);
var streamed=http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("X-Moyi-Source","playground").contentType("application/json").content(json.writeValueAsBytes(input)))
.andExpect(status().isOk()).andExpect(content().contentTypeCompatibleWith("text/event-stream")).andReturn().getResponse().getContentAsString();
assertTrue(streamed.endsWith("data: [DONE]\n\n")); assertFalse(streamed.contains("private-alias"));
assertEquals(new BigDecimal("0.99932000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE call_mode='stream' AND request_source='playground' AND cost_amount=0.00034 AND first_token_ms IS NOT NULL",Integer.class));
mode.set("no-usage");
var failed=http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).contentType("application/json").content(json.writeValueAsBytes(input))).andReturn().getResponse().getContentAsString();
assertTrue(failed.contains("UPSTREAM_USAGE_MISSING"));
mode.set("http-error");
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).contentType("application/json").content(json.writeValueAsBytes(input))).andExpect(status().isBadGateway());
assertEquals(new BigDecimal("0.99932000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
mode.set("ok");
var disconnected=new org.springframework.mock.web.MockHttpServletResponse() {
@Override public jakarta.servlet.ServletOutputStream getOutputStream() {
return new jakarta.servlet.ServletOutputStream() {
@Override public boolean isReady() { return true; }
@Override public void setWriteListener(jakarta.servlet.WriteListener listener) { }
@Override public void write(int b) throws java.io.IOException { throw new java.io.IOException("Browser closed"); }
};
}
};
controller.chatCompletions("Bearer "+key,input,"closed-browser",disconnected);
assertEquals(new BigDecimal("0.99898000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
assertEquals(3,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=? AND operation='usage'",Integer.class,memberId));
assertEquals(new BigDecimal("0.00102000"),catalog.overview(memberToken,auth).get("costThisMonth"));
} finally { provider.close(); upstream.stop(0); }
}
@Test void failedLedgerTransactionCannotPartiallyDebitTheWallet() {
var f=chatFixture(); wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","atomicity test",meta);
var ticket=f.reserve(null);
jdbc.execute("ALTER TABLE usage_events ADD CONSTRAINT test_reject_cost CHECK (cost_amount IS NULL OR cost_amount=0)");
assertThrows(org.springframework.dao.DataIntegrityViolationException.class,()->f.billing().settle(ticket,meteredResult(),5,null));
assertEquals(new BigDecimal("1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
assertEquals("pending",jdbc.queryForObject("SELECT status FROM chat_billing_requests WHERE id=?",String.class,ticket.id()));
f.billing().fail(ticket,"SETTLEMENT_FAILED","Test rollback",5);
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
}
}
@@ -0,0 +1,92 @@
package com.moyi.controlplane.gateway;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
import com.moyi.controlplane.security.SecretCipher;
import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.*;
import org.junit.jupiter.api.*;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.mock.env.MockEnvironment;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
class ChatStreamingTest {
HttpServer server;
OpenAiCompatibleProvider provider;
ObjectMapper json=new ObjectMapper();
Map<String,Object> request=Map.of("messages",List.of(Map.of("role","user","content","测试")),"max_tokens",16,"temperature",0.2,"stream",true);
@BeforeEach void setup() throws Exception {
server=HttpServer.create(new InetSocketAddress("127.0.0.1",0),0);
var env=new MockEnvironment().withProperty("moyi.providers.deepseek.api-key","local-fake-key")
.withProperty("moyi.providers.deepseek.base-url","http://127.0.0.1:"+server.getAddress().getPort()+"/v1")
.withProperty("moyi.gateway.upstream-timeout-seconds","2");
provider=new OpenAiCompatibleProvider(json,env,mock(JdbcTemplate.class),new SecretCipher("local-test-only"));
}
@AfterEach void close() { provider.close(); server.stop(0); }
@Test void fragmentedUtf8StreamsBeforeCompletionAndCarriesActualUsage() throws Exception {
var first=new CountDownLatch(1); var finish=new CountDownLatch(1);
var received=new CompletableFuture<Map<?,?>>();
server.createContext("/v1/chat/completions", exchange -> {
received.complete(json.readValue(exchange.getRequestBody(),Map.class));
exchange.getResponseHeaders().set("Content-Type","text/event-stream"); exchange.sendResponseHeaders(200,0);
try(var out=exchange.getResponseBody()) {
byte[] initial="data: {\"id\":\"test\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"你好\"}}]}\r\n\r\n".getBytes(StandardCharsets.UTF_8);
for(byte b:initial) { out.write(b); out.flush(); }
try { if(!finish.await(1,TimeUnit.SECONDS)) throw new AssertionError("first delta was buffered"); } catch(InterruptedException e) { Thread.currentThread().interrupt(); }
out.write(("data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n"
+"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":20,\"completion_tokens\":2,\"prompt_tokens_details\":{\"cached_tokens\":10}}}\n\n"
+"data: [DONE]\n\n").getBytes(StandardCharsets.UTF_8));
}
}); server.start();
try(var pool=Executors.newSingleThreadExecutor()) {
var result=pool.submit(()->{
var collector=new ChatStreamCollector("public-route",System.currentTimeMillis());
try(var stream=provider.stream("provider-deepseek","upstream-alias",request)) {
stream.events(chunk->{ collector.accept(chunk); first.countDown(); });
}
assertNotNull(collector.firstTokenMs()); return collector.result();
});
assertTrue(first.await(1500,TimeUnit.MILLISECONDS)); assertFalse(result.isDone()); finish.countDown();
var body=result.get(2,TimeUnit.SECONDS);
assertEquals("你好",((Map<?,?>)((Map<?,?>)((List<?>)body.get("choices")).getFirst()).get("message")).get("content"));
assertEquals(new ChatMeter.Tokens(20,2,10),ChatMeter.tokens(body));
assertEquals("public-route",body.get("model"));
}
assertEquals("upstream-alias",received.get().get("model"));
assertEquals(16,received.get().get("max_tokens"));
assertEquals(Map.of("include_usage",true),received.get().get("stream_options"));
}
@Test void deadlineClosesAStreamThatNeverFinishes() {
server.createContext("/v1/chat/completions", exchange->{
exchange.getResponseHeaders().set("Content-Type","text/event-stream"); exchange.sendResponseHeaders(200,0);
exchange.getResponseBody().write(": heartbeat\n\n".getBytes()); exchange.getResponseBody().flush();
// Leave the body open: the client hard deadline must terminate the read.
}); server.start();
long started=System.nanoTime();
assertThrows(Exception.class,()->{ try(var stream=provider.stream("provider-deepseek","alias",request)) { stream.events(chunk->{}); } });
assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)<4000);
}
@Test void upstreamHttpFailureNeverBecomesSuccessfulStream() {
server.createContext("/v1/chat/completions",e->{ e.sendResponseHeaders(401,-1); e.close(); }); server.start();
assertEquals(502,assertThrows(ApiException.class,()->provider.stream("provider-deepseek","alias",request)).getStatus());
}
@Test void eofWithoutDoneAndMissingUsageAreNotBillable() {
server.createContext("/v1/chat/completions",e->{
e.getResponseHeaders().set("Content-Type","text/event-stream"); e.sendResponseHeaders(200,0);
e.getResponseBody().write("data: {\"choices\":[]}\n\n".getBytes()); e.close();
}); server.start();
assertEquals("UPSTREAM_STREAM_INCOMPLETE",assertThrows(ApiException.class,()->{ try(var stream=provider.stream("provider-deepseek","alias",request)) { stream.events(chunk->{}); } }).getCode());
assertThrows(ApiException.class,()->ChatMeter.tokens(Map.of("usage",Map.of("prompt_tokens",1.5,"completion_tokens",2))));
assertThrows(ApiException.class,()->ChatMeter.tokens(Map.of("usage",Map.of("prompt_tokens",1,"completion_tokens",2,"prompt_cache_hit_tokens",3))));
}
@Test void tokenBoundsAndUnsupportedMultimodalAreRejectedBeforeUpstream() {
assertThrows(ApiException.class,()->ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content",List.of(Map.of("type","image_url"))))),json));
assertThrows(ApiException.class,()->ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content","test")),"max_tokens",0),json));
assertEquals(4096,ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content","test"))),json).maxOutputTokens());
}
}
@@ -6,6 +6,8 @@ import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
import com.moyi.controlplane.task.AsyncTaskService;
import com.moyi.controlplane.usage.UsageAuditService;
import com.moyi.controlplane.wallet.ChatBillingService;
import org.springframework.mock.web.MockHttpServletResponse;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -42,11 +44,11 @@ class GatewayDeploymentTest {
when(catalog.authenticateKey("test-key")).thenReturn(key);
when(catalog.activeModel("model-real")).thenReturn(Map.of("id", "model-real", "providerId", "provider-deepseek", "mode", "sync"));
when(catalog.keyCanInvoke(key, "model-real")).thenReturn(true);
GatewayController controller = new GatewayController(catalog, provider, guard, usage);
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "model-real")));
GatewayController controller = new GatewayController(catalog, provider, guard, mock(ChatBillingService.class),new ObjectMapper());
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "model-real"),null,new MockHttpServletResponse()));
assertEquals("PROVIDER_NOT_CONFIGURED", error.getCode());
assertEquals(503, error.getStatus());
verify(provider, never()).chat(anyString(), anyString(), anyList());
verify(provider, never()).chat(anyString(), anyString(), anyMap());
}
@Test
@@ -56,8 +58,11 @@ class GatewayDeploymentTest {
when(catalog.authenticateKey("test-key")).thenReturn(key);
when(catalog.activeModel("mock")).thenReturn(Map.of("id", "mock", "providerId", "provider-mock", "mode", "sync"));
when(catalog.keyCanInvoke(key, "mock")).thenReturn(true);
GatewayController controller = new GatewayController(catalog, mock(OpenAiCompatibleProvider.class), mock(GatewayGuard.class), mock(UsageAuditService.class));
assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock")).getStatusCode().value());
var billing=mock(ChatBillingService.class);
when(billing.reserve(anyMap(),anyMap(),any(),any())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(),UUID.fromString(key.get("userId").toString()),ChatMeter.rates(null,true)));
when(billing.settle(any(),anyMap(),anyInt(),any())).thenReturn(java.math.BigDecimal.ZERO);
GatewayController controller = new GatewayController(catalog, mock(OpenAiCompatibleProvider.class), mock(GatewayGuard.class), billing,new ObjectMapper());
assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock","messages",List.of(Map.of("role","user","content","test"))),null,new MockHttpServletResponse()).getStatusCode().value());
}
@Test