feat: stream chat and settle metered CNY usage atomically
This commit is contained in:
1 parent
7360cf3ebe
commit
b46ee72f2d
28 files changed
+1098
-86
No files matched your search
@@ -221,11 +221,11 @@ public class CatalogService {
|
||||
|
||||
private void validatePricing(Object input) {
|
||||
if (!(input instanceof Map<?,?> prices)) return;
|
||||
for(String field: List.of("input","output")) {
|
||||
for(String field: List.of("input","output","cachedInput")) {
|
||||
Object amount=prices.get(field);
|
||||
if(amount==null) continue;
|
||||
try { if(new java.math.BigDecimal(String.valueOf(amount)).signum()<0) throw new NumberFormatException(); }
|
||||
catch(NumberFormatException e) { throw new ApiException("INVALID_PRICE","价格必须是非负数"); }
|
||||
try { var price=new java.math.BigDecimal(String.valueOf(amount)); if(price.signum()<0 || price.scale()>8 || price.compareTo(new java.math.BigDecimal("1000000"))>0) throw new NumberFormatException(); }
|
||||
catch(NumberFormatException e) { throw new ApiException("INVALID_PRICE","价格须在 0–100 万之间,最多 8 位小数"); }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,8 +351,13 @@ public class CatalogService {
|
||||
result.put("tasks", Map.of("queued", 0, "running", 0, "succeeded", 0, "failed", 0));
|
||||
result.put("series", List.of());
|
||||
result.put("ledger", List.of());
|
||||
List<java.math.BigDecimal> balances=jdbc.queryForList("SELECT balance FROM user_wallets WHERE user_id=?",java.math.BigDecimal.class,user.id());
|
||||
result.put("availableCredit",balances.isEmpty()?java.math.BigDecimal.ZERO:balances.getFirst());
|
||||
var balances=jdbc.queryForList("SELECT balance, reserved_balance FROM user_wallets WHERE user_id=?",user.id());
|
||||
var balance=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("balance");
|
||||
var reserved=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("reserved_balance");
|
||||
result.put("availableCredit",balance.subtract(reserved));
|
||||
result.put("balance",balance);
|
||||
result.put("reservedCredit",reserved);
|
||||
result.put("costThisMonth",jdbc.queryForObject("SELECT COALESCE(-SUM(amount),0) FROM wallet_entries WHERE user_id=? AND operation='usage' AND created_at >= date_trunc('month',now())",java.math.BigDecimal.class,user.id()));
|
||||
result.put("ledger",jdbc.query("SELECT id, reason, amount, created_at FROM wallet_entries WHERE user_id=? ORDER BY created_at DESC LIMIT 100", (rs,n)->Map.of("id",rs.getString("id"),"title",rs.getString("reason"),"amount",rs.getBigDecimal("amount"),"createdAt",rs.getObject("created_at",OffsetDateTime.class)),user.id()));
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -22,7 +22,8 @@ public class WebConfig implements WebMvcConfigurer {
|
||||
registry.addMapping("/**")
|
||||
.allowedOrigins(allowedOrigins)
|
||||
.allowedMethods("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS")
|
||||
.allowedHeaders("Content-Type", "Authorization", "X-Moyi-Source")
|
||||
.allowedHeaders("Content-Type", "Authorization", "X-Moyi-Source", "Idempotency-Key")
|
||||
.exposedHeaders("X-Request-Id", "X-Moyi-Cost-CNY")
|
||||
.allowCredentials(true)
|
||||
.maxAge(3600);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package com.moyi.controlplane.gateway;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.util.*;
|
||||
|
||||
/** Monetary calculations never use floating point or locally estimated completion usage. */
|
||||
public final class ChatMeter {
|
||||
private ChatMeter() {}
|
||||
public record Rates(BigDecimal input, BigDecimal output, BigDecimal cachedInput) {
|
||||
public Map<String,Object> snapshot() {
|
||||
return Map.of("input",input,"output",output,"cachedInput",cachedInput,"unit","CNY / 1M tokens");
|
||||
}
|
||||
public BigDecimal cost(long prompt, long completion, long cached) {
|
||||
return input.multiply(BigDecimal.valueOf(prompt-cached))
|
||||
.add(cachedInput.multiply(BigDecimal.valueOf(cached)))
|
||||
.add(output.multiply(BigDecimal.valueOf(completion)))
|
||||
.divide(new BigDecimal("1000000"),8,RoundingMode.HALF_UP);
|
||||
}
|
||||
}
|
||||
public record Tokens(long prompt, long completion, long cached) {}
|
||||
public record Request(Map<String,Object> body, long reservedInputTokens, int maxOutputTokens) {}
|
||||
|
||||
public static Rates rates(Object value, boolean mock) {
|
||||
if (mock) return new Rates(BigDecimal.ZERO,BigDecimal.ZERO,BigDecimal.ZERO);
|
||||
if (!(value instanceof Map<?,?> prices) || !"CNY / 1M tokens".equals(prices.get("unit")))
|
||||
throw new ApiException("MODEL_NOT_PRICED","该分组尚未配置人民币 Token 售价",503);
|
||||
BigDecimal input=price(prices.get("input")), output=price(prices.get("output"));
|
||||
BigDecimal cached=prices.get("cachedInput")==null?input:price(prices.get("cachedInput"));
|
||||
return new Rates(input,output,cached);
|
||||
}
|
||||
private static BigDecimal price(Object value) {
|
||||
try {
|
||||
BigDecimal price=new BigDecimal(String.valueOf(value));
|
||||
if(price.signum()<0 || price.compareTo(new BigDecimal("1000000"))>0 || price.scale()>8) throw new NumberFormatException();
|
||||
return price;
|
||||
} catch(Exception e) { throw new ApiException("MODEL_NOT_PRICED","请配置有效的输入、输出人民币售价(最多 8 位小数)",503); }
|
||||
}
|
||||
public static Tokens tokens(Map<String,Object> response) {
|
||||
if(response.containsKey("error") || !(response.get("usage") instanceof Map<?,?> usage)) throw invalidUsage();
|
||||
long prompt=count(usage.get("prompt_tokens")), completion=count(usage.get("completion_tokens"));
|
||||
Object cached=usage.get("prompt_cache_hit_tokens");
|
||||
if(usage.get("prompt_tokens_details") instanceof Map<?,?> details && details.get("cached_tokens")!=null) cached=details.get("cached_tokens");
|
||||
long hits=cached==null?0:count(cached);
|
||||
if(hits>prompt) throw invalidUsage();
|
||||
return new Tokens(prompt,completion,hits);
|
||||
}
|
||||
private static long count(Object value) {
|
||||
try {
|
||||
if(!(value instanceof Number)) throw new NumberFormatException();
|
||||
long number=new BigDecimal(value.toString()).longValueExact();
|
||||
if(number<0 || number>1_000_000_000L) throw new NumberFormatException();
|
||||
return number;
|
||||
} catch(Exception e) { throw invalidUsage(); }
|
||||
}
|
||||
private static ApiException invalidUsage() { return new ApiException("UPSTREAM_USAGE_MISSING","上游未返回有效 Token 用量,本次未扣费",502); }
|
||||
|
||||
public static Request request(Map<String,Object> input, ObjectMapper json) {
|
||||
if(!(input.get("messages") instanceof List<?> messages) || messages.isEmpty() || messages.size()>256)
|
||||
throw new ApiException("INVALID_MESSAGES","messages 不能为空,且不能超过 256 条");
|
||||
for(Object item:messages) {
|
||||
if(!(item instanceof Map<?,?> message) || !(message.get("role") instanceof String)) throw new ApiException("INVALID_MESSAGES","消息格式无效");
|
||||
Object content=message.get("content");
|
||||
if(content instanceof List<?> parts) {
|
||||
for(Object part:parts) if(!(part instanceof Map<?,?> p) || !"text".equals(p.get("type")) || !(p.get("text") instanceof String))
|
||||
throw new ApiException("UNSUPPORTED_CONTENT","当前聊天计费支持文本和工具调用,图片、音频请使用对应接口");
|
||||
} else if(content!=null && !(content instanceof String)) throw new ApiException("INVALID_MESSAGES","消息内容格式无效");
|
||||
}
|
||||
if(input.get("n")!=null && !"1".equals(input.get("n").toString())) throw new ApiException("UNSUPPORTED_N","当前仅支持 n=1");
|
||||
if(input.containsKey("max_tokens") && input.containsKey("max_completion_tokens")) throw new ApiException("INVALID_MAX_TOKENS","请只设置一种输出 Token 上限");
|
||||
String maxField=input.containsKey("max_completion_tokens")?"max_completion_tokens":"max_tokens";
|
||||
int max=4096;
|
||||
try {
|
||||
if(input.containsKey(maxField)) max=new BigDecimal(input.get(maxField).toString()).intValueExact();
|
||||
if(max<1 || max>32768) throw new NumberFormatException();
|
||||
} catch(Exception e) { throw new ApiException("INVALID_MAX_TOKENS","输出 Token 上限须在 1–32768 之间"); }
|
||||
var body=new LinkedHashMap<String,Object>();
|
||||
// Do not let client credentials, alternate upstreams or multimodal billing bypass admission.
|
||||
var fields=Set.of("model","messages","stream","temperature","top_p","stop","presence_penalty","frequency_penalty","seed","tools","tool_choice","parallel_tool_calls","response_format","reasoning_effort","thinking","n","max_tokens","max_completion_tokens","stream_options");
|
||||
for(String field:input.keySet()) {
|
||||
if(!fields.contains(field)) throw new ApiException("UNSUPPORTED_PARAMETER","当前不支持请求参数:"+field);
|
||||
body.put(field,input.get(field));
|
||||
}
|
||||
body.put(maxField,max);
|
||||
body.put("stream",Boolean.TRUE.equals(input.get("stream")));
|
||||
try {
|
||||
int bytes=json.writeValueAsBytes(body).length;
|
||||
if(bytes>262144) throw new ApiException("REQUEST_TOO_LARGE","聊天请求不能超过 256 KiB",413);
|
||||
// UTF-8 bytes plus framing is deliberately conservative; actual usage settles later.
|
||||
return new Request(body,bytes+128L*messages.size()+1024,max);
|
||||
} catch(java.io.IOException e) { throw new ApiException("INVALID_REQUEST","请求无法编码"); }
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.moyi.controlplane.gateway;
|
||||
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import java.util.*;
|
||||
|
||||
/** Retain a bounded, useful history while forwarding every original SSE delta. */
|
||||
final class ChatStreamCollector {
|
||||
private final String model;
|
||||
private final long started;
|
||||
private final StringBuilder content=new StringBuilder(), reasoning=new StringBuilder();
|
||||
private final Map<Integer,Map<String,Object>> tools=new TreeMap<>();
|
||||
private Object usage, id, created, finish;
|
||||
private Integer firstToken;
|
||||
ChatStreamCollector(String model,long started) { this.model=model; this.started=started; }
|
||||
void accept(Map<String,Object> chunk) {
|
||||
if(chunk.get("id")!=null) id=chunk.get("id");
|
||||
if(chunk.get("created")!=null) created=chunk.get("created");
|
||||
if(chunk.get("usage") instanceof Map<?,?>) usage=chunk.get("usage");
|
||||
if(chunk.get("choices") instanceof List<?> choices) for(Object raw:choices) {
|
||||
if(!(raw instanceof Map<?,?> choice)) continue;
|
||||
if(choice.get("finish_reason")!=null) finish=choice.get("finish_reason");
|
||||
if(choice.get("delta") instanceof Map<?,?> delta) {
|
||||
append(content,delta.get("content")); append(reasoning,delta.get("reasoning_content"));
|
||||
if(delta.get("tool_calls") instanceof List<?> calls) for(Object item:calls) {
|
||||
if(!(item instanceof Map<?,?> call) || !(call.get("index") instanceof Number index)) continue;
|
||||
int i=index.intValue();
|
||||
if(i<0 || i>=64) throw new ApiException("UPSTREAM_RESPONSE_INVALID","上游工具调用数量超过限制",502);
|
||||
var tool=tools.computeIfAbsent(i,k -> new LinkedHashMap<>());
|
||||
for(String field:List.of("id","type")) if(call.get(field)!=null) tool.put(field,call.get(field));
|
||||
if(call.get("function") instanceof Map<?,?> function) {
|
||||
@SuppressWarnings("unchecked") var target=(Map<String,Object>)tool.computeIfAbsent("function",k -> new LinkedHashMap<>());
|
||||
for(String field:List.of("name","arguments")) if(function.get(field) instanceof String text) {
|
||||
String value=String.valueOf(target.getOrDefault(field,""))+text;
|
||||
target.put(field,value.substring(0,Math.min(8000,value.length())));
|
||||
}
|
||||
}
|
||||
}
|
||||
if(firstToken==null && ((delta.get("content") instanceof String s && !s.isEmpty()) || (delta.get("reasoning_content") instanceof String r && !r.isEmpty()) || delta.get("tool_calls")!=null)) firstToken=(int)(System.currentTimeMillis()-started);
|
||||
}
|
||||
}
|
||||
}
|
||||
private void append(StringBuilder target,Object value) { if(value instanceof String s && target.length()<8000) target.append(s,0,Math.min(s.length(),8000-target.length())); }
|
||||
Integer firstTokenMs() { return firstToken; }
|
||||
Map<String,Object> result() {
|
||||
if(finish==null) throw new ApiException("UPSTREAM_STREAM_INCOMPLETE","上游未完成输出,本次未扣费",502);
|
||||
var message=new LinkedHashMap<String,Object>(); message.put("role","assistant"); message.put("content",content.toString());
|
||||
if(!reasoning.isEmpty()) message.put("reasoning_content",reasoning.toString());
|
||||
if(!tools.isEmpty()) message.put("tool_calls",List.copyOf(tools.values()));
|
||||
var result=new LinkedHashMap<String,Object>(); result.put("id",id); result.put("created",created); result.put("model",model); result.put("object","chat.completion");
|
||||
result.put("usage",usage); result.put("choices",List.of(Map.of("index",0,"message",message,"finish_reason",finish)));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
+87
-34
@@ -3,13 +3,15 @@ package com.moyi.controlplane.gateway;
|
||||
import com.moyi.controlplane.catalog.CatalogService;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import com.moyi.controlplane.wallet.ChatBillingService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
@@ -24,13 +26,15 @@ public class GatewayController {
|
||||
private final CatalogService catalogService;
|
||||
private final OpenAiCompatibleProvider provider;
|
||||
private final GatewayGuard guard;
|
||||
private final UsageAuditService usage;
|
||||
private final ChatBillingService billing;
|
||||
private final ObjectMapper json;
|
||||
|
||||
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, UsageAuditService usage) {
|
||||
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, ChatBillingService billing, ObjectMapper json) {
|
||||
this.catalogService = catalogService;
|
||||
this.provider = provider;
|
||||
this.guard = guard;
|
||||
this.usage = usage;
|
||||
this.billing = billing;
|
||||
this.json = json;
|
||||
}
|
||||
|
||||
@GetMapping("/models")
|
||||
@@ -55,7 +59,9 @@ public class GatewayController {
|
||||
@PostMapping("/chat/completions")
|
||||
public ResponseEntity<Map<String, Object>> chatCompletions(
|
||||
@RequestHeader(value = "Authorization", required = false) String authorization,
|
||||
@RequestBody Map<String, Object> input
|
||||
@RequestBody Map<String, Object> input,
|
||||
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
|
||||
HttpServletResponse servletResponse
|
||||
) {
|
||||
Map<String, Object> key = requireKey(authorization);
|
||||
long startedAt = System.currentTimeMillis();
|
||||
@@ -64,34 +70,89 @@ public class GatewayController {
|
||||
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线", 404);
|
||||
if (!catalogService.keyCanInvoke(key, modelId)) throw new ApiException("MODEL_FORBIDDEN", "当前 Key 无权调用该模型", 403);
|
||||
if ("async".equals(model.get("mode"))) throw new ApiException("ASYNC_MODEL", "该模型需要使用异步任务接口", 400);
|
||||
if (Boolean.TRUE.equals(input.get("stream"))) throw new ApiException("STREAM_NOT_IMPLEMENTED", "流式转发将在下一节点开放", 501);
|
||||
String providerId=String.valueOf(model.get("providerId"));
|
||||
boolean mock="provider-mock".equals(providerId);
|
||||
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||
try (GatewayGuard.Slot slot = guard.acquire(model)) {
|
||||
ResponseEntity<Map<String, Object>> result = invoke(key, modelId, model, input);
|
||||
Map<String, Object> body = result.getBody();
|
||||
usage.recordCall(UUID.fromString(String.valueOf(key.get("userId"))), UUID.fromString(String.valueOf(key.get("id"))), modelId, String.valueOf(model.get("providerId")), "succeeded", (int) (System.currentTimeMillis() - startedAt), number(body, "usage.prompt_tokens"), number(body, "usage.completion_tokens"), input, body, null, null);
|
||||
return result;
|
||||
} catch (ApiException exception) {
|
||||
usage.recordCall(UUID.fromString(String.valueOf(key.get("userId"))), UUID.fromString(String.valueOf(key.get("id"))), modelId, String.valueOf(model.get("providerId")), "failed", (int) (System.currentTimeMillis() - startedAt), 0, 0, input, null, exception.getCode(), exception.getMessage());
|
||||
throw exception;
|
||||
var ticket=billing.reserve(key,model,request,idempotencyKey);
|
||||
servletResponse.setHeader("X-Request-Id",ticket.id().toString());
|
||||
try {
|
||||
catalogService.touchKey(key);
|
||||
if(Boolean.TRUE.equals(request.body().get("stream"))) {
|
||||
stream(model,request.body(),ticket,servletResponse,startedAt,mock);
|
||||
return null;
|
||||
}
|
||||
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),request.body());
|
||||
body.put("model",modelId);
|
||||
requireCompletion(body);
|
||||
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
|
||||
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(body);
|
||||
} catch(Exception exception) {
|
||||
ApiException error=exception instanceof ApiException e?e:new ApiException("CALL_INTERRUPTED","调用中断,请查看使用记录确认结算状态",502);
|
||||
billing.fail(ticket,error.getCode(),error.getMessage(),elapsed(startedAt));
|
||||
if(servletResponse.isCommitted()) {
|
||||
try { event(servletResponse,Map.of("error",Map.of("code",error.getCode(),"message",error.getMessage()))); done(servletResponse); } catch(IOException ignored) { }
|
||||
return null;
|
||||
}
|
||||
servletResponse.resetBuffer();
|
||||
servletResponse.setContentType("application/json");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ResponseEntity<Map<String, Object>> invoke(Map<String, Object> key, String modelId, Map<String, Object> model, Map<String, Object> input) {
|
||||
String providerId = String.valueOf(model.get("providerId"));
|
||||
if (provider.isConfigured(providerId)) {
|
||||
List<?> messages = input.get("messages") instanceof List<?> values ? values : List.of();
|
||||
Map<String, Object> upstream = provider.chat(providerId, String.valueOf(model.get("upstreamModel")), messages);
|
||||
upstream.put("model", modelId);
|
||||
catalogService.touchKey(key);
|
||||
return ResponseEntity.ok(upstream);
|
||||
private void stream(Map<String,Object> model, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
||||
String modelId=model.get("id").toString();
|
||||
if(mock) {
|
||||
var body=mockResponse(modelId,input);
|
||||
prepareStream(response);
|
||||
var choices=(List<?>)body.get("choices");
|
||||
var message=((Map<?,?>)choices.getFirst()).get("message");
|
||||
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",message))));
|
||||
billing.settle(ticket,body,elapsed(started),elapsed(started));
|
||||
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",Map.of(),"finish_reason","stop"))));
|
||||
event(response,Map.of("choices",List.of(),"usage",body.get("usage"))); done(response); return;
|
||||
}
|
||||
if (!"provider-mock".equals(providerId)) {
|
||||
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该供应商尚未配置密钥,不能执行真实调用", 503);
|
||||
// Keep admission, the concurrency slot and the upstream stream alive on this request thread.
|
||||
// Upstream HTTP errors are inspected before committing a 200/SSE response.
|
||||
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),input)) {
|
||||
var collector=new ChatStreamCollector(modelId,started);
|
||||
prepareStream(response);
|
||||
boolean[] disconnected={false};
|
||||
upstream.events(chunk -> {
|
||||
chunk.put("model",modelId);
|
||||
collector.accept(chunk);
|
||||
// A closed browser must not turn an otherwise completed provider call into free usage.
|
||||
// Continue within the same hard deadline to obtain the final meter and settle once.
|
||||
if(!disconnected[0]) try { event(response,chunk); } catch(IOException clientClosed) { disconnected[0]=true; }
|
||||
});
|
||||
var body=collector.result();
|
||||
billing.settle(ticket,body,elapsed(started),collector.firstTokenMs());
|
||||
if(!disconnected[0]) done(response);
|
||||
}
|
||||
}
|
||||
private void prepareStream(HttpServletResponse response) {
|
||||
response.setContentType("text/event-stream"); response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Cache-Control","no-cache"); response.setHeader("X-Accel-Buffering","no");
|
||||
}
|
||||
private void event(HttpServletResponse response,Object data) throws IOException {
|
||||
response.getOutputStream().write(("data: "+json.writeValueAsString(data)+"\n\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
response.flushBuffer();
|
||||
}
|
||||
private void done(HttpServletResponse response) throws IOException {
|
||||
response.getOutputStream().write("data: [DONE]\n\n".getBytes(java.nio.charset.StandardCharsets.UTF_8)); response.flushBuffer();
|
||||
}
|
||||
private int elapsed(long started) { return (int)Math.min(Integer.MAX_VALUE,System.currentTimeMillis()-started); }
|
||||
private void requireCompletion(Map<String,Object> body) {
|
||||
if(body.containsKey("error") || !(body.get("choices") instanceof List<?> choices) || choices.isEmpty()
|
||||
|| !(choices.getFirst() instanceof Map<?,?> choice) || choice.get("finish_reason")==null)
|
||||
throw new ApiException("UPSTREAM_RESPONSE_INVALID","上游未返回完整的聊天结果,本次未扣费",502);
|
||||
}
|
||||
private Map<String,Object> mockResponse(String modelId,Map<String,Object> input) {
|
||||
String prompt = extractPrompt(input.get("messages"));
|
||||
String content = "MOYI Java Mock 已收到:" + (prompt.isBlank() ? "(空消息)" : prompt);
|
||||
catalogService.touchKey(key);
|
||||
Map<String, Object> response = new LinkedHashMap<>();
|
||||
response.put("id", "chatcmpl-" + UUID.randomUUID().toString().replace("-", "").substring(0, 12));
|
||||
response.put("object", "chat.completion");
|
||||
@@ -99,7 +160,7 @@ public class GatewayController {
|
||||
response.put("model", modelId);
|
||||
response.put("choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", content), "finish_reason", "stop")));
|
||||
response.put("usage", Map.of("prompt_tokens", prompt.length(), "completion_tokens", content.length(), "total_tokens", prompt.length() + content.length()));
|
||||
return ResponseEntity.status(HttpStatus.OK).body(response);
|
||||
return response;
|
||||
}
|
||||
|
||||
private Map<String, Object> requireKey(String authorization) {
|
||||
@@ -117,12 +178,4 @@ public class GatewayController {
|
||||
}).reduce((left, right) -> left + "\n" + right).orElse("");
|
||||
}
|
||||
|
||||
private long number(Map<String, Object> body, String path) {
|
||||
if (body == null) return 0;
|
||||
Object usageValue = body.get("usage");
|
||||
if (!(usageValue instanceof Map<?, ?> map)) return 0;
|
||||
String key = path.substring(path.indexOf('.') + 1);
|
||||
Object value = map.get(key);
|
||||
return value instanceof Number number ? number.longValue() : 0;
|
||||
}
|
||||
}
|
||||
+76
-9
@@ -8,10 +8,14 @@ import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.springframework.dao.EmptyResultDataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -23,6 +27,8 @@ public class OpenAiCompatibleProvider {
|
||||
private final JdbcTemplate jdbc;
|
||||
private final SecretCipher secretCipher;
|
||||
private final Map<String, ProviderConfig> providers;
|
||||
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
||||
private final int timeoutSeconds;
|
||||
|
||||
public OpenAiCompatibleProvider(
|
||||
ObjectMapper objectMapper,
|
||||
@@ -34,6 +40,7 @@ public class OpenAiCompatibleProvider {
|
||||
this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
|
||||
this.jdbc = jdbc;
|
||||
this.secretCipher = secretCipher;
|
||||
this.timeoutSeconds = Math.max(1,Math.min(90,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
||||
this.providers = Map.of(
|
||||
"provider-deepseek", new ProviderConfig(environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com")),
|
||||
"provider-kimi", new ProviderConfig(environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1")),
|
||||
@@ -46,29 +53,49 @@ public class OpenAiCompatibleProvider {
|
||||
return provider != null && !provider.apiKey().isBlank();
|
||||
}
|
||||
|
||||
public Map<String, Object> chat(String providerId, String upstreamModel, List<?> messages) {
|
||||
public Map<String, Object> chat(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||
try(var response=open(providerId,upstreamModel,input,false)) {
|
||||
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
|
||||
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
||||
return objectMapper.readValue(bytes, new com.fasterxml.jackson.core.type.TypeReference<>() {});
|
||||
} catch(ApiException e) { throw e; }
|
||||
catch(Exception e) { throw upstreamError(); }
|
||||
}
|
||||
|
||||
public StreamResponse stream(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||
return open(providerId,upstreamModel,input,true);
|
||||
}
|
||||
|
||||
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
|
||||
ProviderConfig provider = resolve(providerId);
|
||||
if (provider == null || provider.apiKey().isBlank()) {
|
||||
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
|
||||
}
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
Map<String, Object> body = new LinkedHashMap<>(input);
|
||||
body.put("model", upstreamModel);
|
||||
body.put("messages", messages == null ? List.of() : messages);
|
||||
body.put("stream", false);
|
||||
body.put("stream", streaming);
|
||||
if(streaming) body.put("stream_options",Map.of("include_usage",true));
|
||||
else body.remove("stream_options");
|
||||
String json = writeJson(body);
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(normalizeUrl(provider.baseUrl()) + "/chat/completions"))
|
||||
.timeout(Duration.ofSeconds(90))
|
||||
.timeout(Duration.ofSeconds(timeoutSeconds))
|
||||
.header("Authorization", "Bearer " + provider.apiKey())
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(json))
|
||||
.build();
|
||||
try {
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
long started=System.nanoTime();
|
||||
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502);
|
||||
response.body().close();
|
||||
throw upstreamError();
|
||||
}
|
||||
return objectMapper.readValue(response.body(), new com.fasterxml.jackson.core.type.TypeReference<>() {});
|
||||
if(streaming && !response.headers().firstValue("Content-Type").orElse("").toLowerCase().startsWith("text/event-stream")) {
|
||||
response.body().close();
|
||||
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
|
||||
}
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)));
|
||||
} catch (ApiException exception) {
|
||||
throw exception;
|
||||
} catch (InterruptedException exception) {
|
||||
@@ -79,6 +106,46 @@ public class OpenAiCompatibleProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/** A hard body deadline is required: HttpRequest.timeout alone does not cover InputStream reads. */
|
||||
public final class StreamResponse implements AutoCloseable {
|
||||
private final InputStream body;
|
||||
private final AtomicBoolean expired=new AtomicBoolean();
|
||||
private final ScheduledFuture<?> deadline;
|
||||
private StreamResponse(InputStream body,long remainingMs) {
|
||||
this.body=body;
|
||||
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
|
||||
}
|
||||
public void events(EventConsumer consumer) throws IOException {
|
||||
var reader=new BufferedReader(new InputStreamReader(body,StandardCharsets.UTF_8));
|
||||
StringBuilder event=new StringBuilder(), line=new StringBuilder();
|
||||
long total=0;
|
||||
int c;
|
||||
// Bound both individual frames and the whole response before allocating unbounded strings.
|
||||
while((c=reader.read())!=-1) {
|
||||
if(expired.get()) throw new ApiException("UPSTREAM_PROVIDER_TIMEOUT","上游响应超时,本次未扣费",504);
|
||||
if(++total>4*1024*1024 || line.length()>262144 || event.length()>262144) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
||||
if(c=='\n') {
|
||||
String text=line.toString(); line.setLength(0);
|
||||
if(text.endsWith("\r")) text=text.substring(0,text.length()-1);
|
||||
if(text.isEmpty()) {
|
||||
if(!event.isEmpty()) {
|
||||
String data=event.substring(0,event.length()-1); event.setLength(0);
|
||||
if("[DONE]".equals(data)) return;
|
||||
Map<String,Object> chunk=objectMapper.readValue(data,new com.fasterxml.jackson.core.type.TypeReference<>(){});
|
||||
if(chunk.containsKey("error")) throw upstreamError();
|
||||
consumer.accept(chunk);
|
||||
}
|
||||
} else if(text.startsWith("data:")) event.append(text.substring(5).stripLeading()).append('\n');
|
||||
} else line.append((char)c);
|
||||
}
|
||||
throw new ApiException(expired.get()?"UPSTREAM_PROVIDER_TIMEOUT":"UPSTREAM_STREAM_INCOMPLETE","上游流式响应中断,本次未扣费",502);
|
||||
}
|
||||
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} }
|
||||
}
|
||||
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
|
||||
@PreDestroy public void close() { deadlines.shutdownNow(); }
|
||||
private ApiException upstreamError() { return new ApiException("UPSTREAM_PROVIDER_ERROR","上游模型供应商请求失败",502); }
|
||||
|
||||
private String normalizeUrl(String value) {
|
||||
return value == null ? "" : value.replaceAll("/+$", "");
|
||||
}
|
||||
|
||||
+21
-1
@@ -33,6 +33,24 @@ public class UsageAuditService {
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public void startChat(UUID id, UUID userId, UUID keyId, Map<String,Object> model, Map<String,Object> input) {
|
||||
String agent=null, source="api";
|
||||
var attributes=org.springframework.web.context.request.RequestContextHolder.getRequestAttributes();
|
||||
if(attributes instanceof org.springframework.web.context.request.ServletRequestAttributes servlet) {
|
||||
agent=servlet.getRequest().getHeader("User-Agent");
|
||||
if("playground".equals(servlet.getRequest().getHeader("X-Moyi-Source"))) source="playground";
|
||||
}
|
||||
jdbc.update("INSERT INTO usage_events(id,request_id,user_id,api_key_id,model_id,provider_id,event_kind,status,input_payload,endpoint,call_mode,request_source,user_agent,reasoning_effort) VALUES (?,?,?,?,?,?,'call','running',?::jsonb,'/v1/chat/completions',?,?,?,?)",
|
||||
id,id.toString(),userId,keyId,model.get("id"),model.get("providerId"),json(input),Boolean.TRUE.equals(input.get("stream"))?"stream":"sync",source,
|
||||
agent==null?null:agent.substring(0,Math.min(1000,agent.length())),input.get("reasoning_effort")==null?null:String.valueOf(input.get("reasoning_effort")).substring(0,Math.min(32,String.valueOf(input.get("reasoning_effort")).length())));
|
||||
}
|
||||
|
||||
public void finishChat(UUID id, String status, com.moyi.controlplane.gateway.ChatMeter.Tokens tokens,
|
||||
int latency, Integer firstTokenMs, java.math.BigDecimal cost, Object output, String code, String message) {
|
||||
jdbc.update("UPDATE usage_events SET status=?,input_units=?,output_units=?,cached_input_units=?,latency_ms=?,first_token_ms=?,cost_amount=?,output_payload=?::jsonb,error_code=?,error_message=?,updated_at=now() WHERE id=?",
|
||||
status,tokens.prompt(),tokens.completion(),tokens.cached(),latency,firstTokenMs,cost,json(output),code,message,id);
|
||||
}
|
||||
|
||||
public void recordCall(
|
||||
UUID userId, UUID apiKeyId, String modelId, String providerId, String status,
|
||||
int latencyMs, long inputUnits, long outputUnits, Object input, Object output,
|
||||
@@ -263,7 +281,9 @@ public class UsageAuditService {
|
||||
ObjectNode result = objectMapper.createObjectNode();
|
||||
node.fields().forEachRemaining(entry -> {
|
||||
String key = entry.getKey().toLowerCase();
|
||||
if (key.contains("authorization") || key.contains("api_key") || key.contains("apikey") || key.contains("token") || key.contains("secret") || key.contains("password")) result.put(entry.getKey(), "[REDACTED]");
|
||||
boolean usageField = java.util.Set.of("prompt_tokens", "completion_tokens", "total_tokens", "cached_tokens", "reasoning_tokens", "audio_tokens", "accepted_prediction_tokens", "rejected_prediction_tokens", "prompt_cache_hit_tokens", "prompt_cache_miss_tokens", "max_tokens", "max_completion_tokens").contains(key) && entry.getValue().isNumber();
|
||||
boolean usageDetails = java.util.Set.of("prompt_tokens_details", "completion_tokens_details").contains(key) && entry.getValue().isObject();
|
||||
if (key.contains("authorization") || key.contains("api_key") || key.contains("apikey") || (key.contains("token") && !usageField && !usageDetails) || key.contains("secret") || key.contains("password")) result.put(entry.getKey(), "[REDACTED]");
|
||||
else result.set(entry.getKey(), sanitize(entry.getValue()));
|
||||
});
|
||||
return result;
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package com.moyi.controlplane.wallet;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.gateway.ChatMeter;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import java.math.BigDecimal;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.*;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Service
|
||||
public class ChatBillingService {
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper json;
|
||||
private final UsageAuditService usage;
|
||||
public ChatBillingService(JdbcTemplate jdbc, ObjectMapper json, UsageAuditService usage) { this.jdbc=jdbc; this.json=json; this.usage=usage; }
|
||||
public record Ticket(UUID id, UUID userId, ChatMeter.Rates rates) {}
|
||||
|
||||
@Transactional
|
||||
public Ticket reserve(Map<String,Object> key, Map<String,Object> model, ChatMeter.Request request, String idempotencyKey) {
|
||||
UUID userId=UUID.fromString(key.get("userId").toString());
|
||||
ChatMeter.Rates rates=ChatMeter.rates(model.get("pricing"),"provider-mock".equals(model.get("providerId")));
|
||||
String requestHash=hash(encode(new TreeMap<>(request.body())));
|
||||
String idem=null;
|
||||
if(idempotencyKey!=null) {
|
||||
if(idempotencyKey.isBlank() || idempotencyKey.length()>128) throw new ApiException("INVALID_IDEMPOTENCY_KEY","Idempotency-Key 长度须为 1–128");
|
||||
idem=hash(idempotencyKey);
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended(?,0))",userId+":"+idem);
|
||||
var previous=jdbc.queryForList("SELECT id, request_hash FROM chat_billing_requests WHERE user_id=? AND idempotency_hash=?",userId,idem);
|
||||
if(!previous.isEmpty()) throw new ApiException(requestHash.equals(previous.getFirst().get("request_hash"))?"DUPLICATE_REQUEST":"IDEMPOTENCY_CONFLICT","该请求编号已受理,请在使用记录中查看结果;不会重复调用或扣费",409);
|
||||
}
|
||||
BigDecimal hold=rates.input().max(rates.cachedInput()).multiply(BigDecimal.valueOf(request.reservedInputTokens()))
|
||||
.add(rates.output().multiply(BigDecimal.valueOf(request.maxOutputTokens())))
|
||||
.divide(new BigDecimal("1000000"),8,java.math.RoundingMode.CEILING);
|
||||
jdbc.update("INSERT INTO user_wallets(user_id) VALUES (?) ON CONFLICT DO NOTHING",userId);
|
||||
var wallet=jdbc.queryForMap("SELECT balance,reserved_balance FROM user_wallets WHERE user_id=? FOR UPDATE",userId);
|
||||
BigDecimal available=((BigDecimal)wallet.get("balance")).subtract((BigDecimal)wallet.get("reserved_balance"));
|
||||
if(available.compareTo(hold)<0) throw new ApiException("INSUFFICIENT_BALANCE","可用余额不足以预留本次调用费用,请充值或调低输出 Token 上限",402);
|
||||
UUID id=UUID.randomUUID();
|
||||
usage.startChat(id,userId,UUID.fromString(key.get("id").toString()),model,request.body());
|
||||
// Longer than the upstream hard deadline and the concurrency lease, including cleanup margin.
|
||||
jdbc.update("INSERT INTO chat_billing_requests(id,user_id,idempotency_hash,request_hash,reserved_amount,pricing,expires_at) VALUES (?,?,?,?,?,?::jsonb,now()+interval '5 minutes')",id,userId,idem,requestHash,hold,encode(rates.snapshot()));
|
||||
jdbc.update("UPDATE user_wallets SET reserved_balance=reserved_balance+?,updated_at=now() WHERE user_id=?",hold,userId);
|
||||
return new Ticket(id,userId,rates);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public BigDecimal settle(Ticket ticket, Map<String,Object> response, int latency, Integer firstTokenMs) {
|
||||
var call=jdbc.queryForMap("SELECT * FROM chat_billing_requests WHERE id=? FOR UPDATE",ticket.id());
|
||||
if("succeeded".equals(call.get("status"))) return (BigDecimal)call.get("cost_amount");
|
||||
if(!"pending".equals(call.get("status"))) throw new ApiException("BILLING_REQUEST_CLOSED","调用已结束,不能重复结算",409);
|
||||
var tokens=ChatMeter.tokens(response);
|
||||
// Read the persisted snapshot, not editable catalog prices or caller-supplied rates.
|
||||
ChatMeter.Rates rates=ChatMeter.rates(decode(call.get("pricing").toString()),false);
|
||||
BigDecimal cost=rates.cost(tokens.prompt(),tokens.completion(),tokens.cached());
|
||||
BigDecimal before=jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=? FOR UPDATE",BigDecimal.class,ticket.userId());
|
||||
BigDecimal after=before.subtract(cost);
|
||||
jdbc.update("UPDATE user_wallets SET balance=?,reserved_balance=reserved_balance-?,updated_at=now() WHERE user_id=?",after,call.get("reserved_amount"),ticket.userId());
|
||||
if(cost.signum()>0) jdbc.update("INSERT INTO wallet_entries(id,user_id,request_id,amount,balance_before,balance_after,reason,operation,requested_amount) VALUES (?,?,?,?,?,?,?,'usage',?)",UUID.randomUUID(),ticket.userId(),ticket.id(),cost.negate(),before,after,"模型调用 · "+ticket.id(),cost);
|
||||
usage.finishChat(ticket.id(),"succeeded",tokens,latency,firstTokenMs,cost,response,null,null);
|
||||
jdbc.update("UPDATE chat_billing_requests SET status='succeeded',cost_amount=? WHERE id=?",cost,ticket.id());
|
||||
return cost;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void fail(Ticket ticket, String code, String message, int latency) { release(ticket.id(),code,message,latency); }
|
||||
|
||||
private void release(UUID id, String code, String message, int latency) {
|
||||
var call=jdbc.queryForMap("SELECT * FROM chat_billing_requests WHERE id=? FOR UPDATE",id);
|
||||
if(!"pending".equals(call.get("status"))) return;
|
||||
jdbc.update("UPDATE user_wallets SET reserved_balance=reserved_balance-?,updated_at=now() WHERE user_id=?",call.get("reserved_amount"),call.get("user_id"));
|
||||
jdbc.update("UPDATE chat_billing_requests SET status='failed',cost_amount=0 WHERE id=?",id);
|
||||
usage.finishChat(id,"failed",new ChatMeter.Tokens(0,0,0),latency,null,BigDecimal.ZERO,null,code,message);
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString="${moyi.billing.recovery-interval-ms:60000}")
|
||||
@Transactional
|
||||
public void recoverExpired() {
|
||||
var ids=jdbc.queryForList("SELECT id FROM chat_billing_requests WHERE status='pending' AND expires_at<now() ORDER BY expires_at LIMIT 100 FOR UPDATE SKIP LOCKED",UUID.class);
|
||||
for(UUID id:ids) release(id,"CALL_INTERRUPTED","调用中断,未取得可结算用量,预留额度已释放",0);
|
||||
}
|
||||
private String encode(Object value) { try { return json.writeValueAsString(value); } catch(Exception e) { throw new ApiException("INVALID_REQUEST","请求无法编码"); } }
|
||||
private Map<String,Object> decode(String value) { try { return json.readValue(value,new com.fasterxml.jackson.core.type.TypeReference<>(){}); } catch(Exception e) { throw new IllegalStateException("Invalid billing snapshot",e); } }
|
||||
private String hash(String value) { try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); } catch(Exception e) { throw new IllegalStateException(e); } }
|
||||
}
|
||||
@@ -25,7 +25,7 @@ public class WalletService {
|
||||
|
||||
public List<Map<String,Object>> entries(String token, UUID userId) {
|
||||
auth.requireAdmin(token);
|
||||
return jdbc.queryForList("SELECT e.id, e.amount, e.operation, e.balance_before AS \"balanceBefore\", e.balance_after AS \"balanceAfter\", e.currency, e.reason, e.created_at AS \"createdAt\", u.display_name AS actor FROM wallet_entries e JOIN users u ON u.id=e.actor_user_id WHERE e.user_id=? ORDER BY e.created_at DESC LIMIT 100",userId);
|
||||
return jdbc.queryForList("SELECT e.id, e.request_id AS \"requestId\", e.amount, e.operation, e.balance_before AS \"balanceBefore\", e.balance_after AS \"balanceAfter\", e.currency, e.reason, e.created_at AS \"createdAt\", COALESCE(u.display_name,'系统计费') AS actor FROM wallet_entries e LEFT JOIN users u ON u.id=e.actor_user_id WHERE e.user_id=? ORDER BY e.created_at DESC LIMIT 100",userId);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -64,7 +64,8 @@ public class WalletService {
|
||||
BigDecimal balance=jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=? FOR UPDATE",BigDecimal.class,userId);
|
||||
if(expected!=null && balance.compareTo(expected)!=0) throw new ApiException("BALANCE_CHANGED","余额已变化,请刷新后重新确认调整",409);
|
||||
BigDecimal next=switch(operation) { case "decrease" -> balance.subtract(amount); case "set" -> amount; default -> balance.add(amount); };
|
||||
if(next.signum()<0) throw new ApiException("INSUFFICIENT_BALANCE","减少金额不能超过当前余额",409);
|
||||
BigDecimal reserved=jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,userId);
|
||||
if(!"increase".equals(operation) && (next.signum()<0 || next.compareTo(reserved)<0)) throw new ApiException("INSUFFICIENT_BALANCE","减少后的余额不能低于 0 或正在调用的预留额度",409);
|
||||
BigDecimal delta=next.subtract(balance);
|
||||
UUID id=UUID.randomUUID();
|
||||
jdbc.update("UPDATE user_wallets SET balance=?, updated_at=now() WHERE user_id=?",next,userId);
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
-- A hold does not consume credit. Only a completed, metered call creates a debit.
|
||||
ALTER TABLE user_wallets ADD COLUMN reserved_balance numeric(20,8) NOT NULL DEFAULT 0 CHECK (reserved_balance >= 0);
|
||||
-- If a provider exceeds the token reservation, retain the actual charge as debt.
|
||||
-- New calls and administrative withdrawals cannot spend this debt.
|
||||
ALTER TABLE user_wallets DROP CONSTRAINT user_wallets_balance_check;
|
||||
ALTER TABLE wallet_entries DROP CONSTRAINT wallet_entries_balance_check;
|
||||
ALTER TABLE wallet_entries ADD CONSTRAINT wallet_entries_balance_check CHECK (balance_after=balance_before+amount);
|
||||
ALTER TABLE wallet_entries DROP CONSTRAINT wallet_entries_operation_check;
|
||||
ALTER TABLE wallet_entries ADD CONSTRAINT wallet_entries_operation_check CHECK (operation IN ('increase','decrease','set','usage'));
|
||||
ALTER TABLE wallet_entries ALTER COLUMN actor_user_id DROP NOT NULL;
|
||||
ALTER TABLE usage_events ALTER COLUMN cost_amount TYPE numeric(20,8);
|
||||
|
||||
CREATE TABLE chat_billing_requests (
|
||||
id uuid PRIMARY KEY REFERENCES usage_events(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
idempotency_hash varchar(64),
|
||||
request_hash varchar(64) NOT NULL,
|
||||
status varchar(16) NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','succeeded','failed')),
|
||||
reserved_amount numeric(20,8) NOT NULL CHECK (reserved_amount >= 0),
|
||||
pricing jsonb NOT NULL,
|
||||
cost_amount numeric(20,8),
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE(user_id,idempotency_hash)
|
||||
);
|
||||
CREATE INDEX chat_billing_expiration_idx ON chat_billing_requests(expires_at) WHERE status='pending';
|
||||
+168
-1
@@ -7,6 +7,8 @@ import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.common.ApiExceptionHandler;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.moyi.controlplane.wallet.WalletService;
|
||||
import com.moyi.controlplane.wallet.ChatBillingService;
|
||||
import com.moyi.controlplane.gateway.ChatMeter;
|
||||
import com.moyi.controlplane.catalog.CatalogController;
|
||||
import com.moyi.controlplane.usage.*;
|
||||
import jakarta.servlet.http.Cookie;
|
||||
@@ -78,6 +80,8 @@ class PlatformAdminPostgresTest {
|
||||
context.registerBean(AuthService.class, () -> new AuthService(jdbc, new PasswordHasher("test-pepper"), json, 168));
|
||||
context.registerBean(CatalogService.class, () -> new CatalogService(jdbc, json, new SecretCipher("test-encryption")));
|
||||
context.registerBean(WalletService.class, () -> new WalletService(jdbc, context.getBean(AuthService.class)));
|
||||
context.registerBean(UsageAuditService.class, () -> new UsageAuditService(jdbc,json));
|
||||
context.registerBean(ChatBillingService.class, () -> new ChatBillingService(jdbc,json,context.getBean(UsageAuditService.class)));
|
||||
context.registerBean(VerificationService.class, () -> new VerificationService(jdbc, sender, "test-pepper"));
|
||||
context.refresh();
|
||||
auth = context.getBean(AuthService.class);
|
||||
@@ -268,7 +272,7 @@ class PlatformAdminPostgresTest {
|
||||
org.springframework.web.context.request.RequestContextHolder.setRequestAttributes(new org.springframework.web.context.request.ServletRequestAttributes(request));
|
||||
try {
|
||||
audit.recordCall(memberId,null,groupId,providerId,"succeeded",200,100,20,
|
||||
Map.of("prompt","hello","authorization","hidden"),
|
||||
Map.of("prompt","hello","authorization","hidden","accessToken","hidden","refresh_token","hidden","prompt_tokens","hidden"),
|
||||
Map.of("usage",Map.of("prompt_tokens_details",Map.of("cached_tokens",80),"cache_creation_input_tokens",5)),null,null);
|
||||
} finally { org.springframework.web.context.request.RequestContextHolder.resetRequestAttributes(); }
|
||||
var saved=jdbc.queryForMap("SELECT * FROM usage_events WHERE user_id=?",memberId);
|
||||
@@ -280,6 +284,7 @@ class PlatformAdminPostgresTest {
|
||||
assertNull(saved.get("cost_amount"));
|
||||
assertNull(saved.get("first_token_ms"));
|
||||
assertFalse(saved.get("input_payload").toString().contains("hidden"));
|
||||
assertEquals(80,json.readTree(saved.get("output_payload").toString()).path("usage").path("prompt_tokens_details").path("cached_tokens").asInt());
|
||||
audit.recordTask(memberId,null,groupId,providerId,"local-timing-task","queued",Map.of("prompt","hello"),null,null,null);
|
||||
jdbc.update("UPDATE usage_events SET created_at=now()-interval '2 seconds' WHERE provider_task_id='local-timing-task'");
|
||||
audit.updateTask("local-timing-task","succeeded",Map.of("url","https://example.test/result"),null,null);
|
||||
@@ -329,4 +334,166 @@ class PlatformAdminPostgresTest {
|
||||
assertFalse(verification.capabilities().containsValue(true));
|
||||
assertThrows(ApiException.class, () -> verification.send("unavailable@example.test",invite,"127.0.0.1"));
|
||||
}
|
||||
|
||||
record ChatFixture(ChatBillingService billing, Map<String,Object> key, Map<String,Object> model, ChatMeter.Request request) {
|
||||
ChatBillingService.Ticket reserve(String idem) { return billing.reserve(key,model,request,idem); }
|
||||
}
|
||||
ChatFixture chatFixture() {
|
||||
var model=new LinkedHashMap<>(catalog.listModels(false).stream().filter(m -> "provider-deepseek".equals(m.get("providerId"))).findFirst().orElseThrow());
|
||||
model.put("pricing",Map.of("input",2,"output",4,"cachedInput",1,"unit","CNY / 1M tokens"));
|
||||
var created=catalog.createKey(memberToken,Map.of("name","Local billing test","modelIds",List.of(model.get("id"))),auth);
|
||||
var key=catalog.authenticateKey(created.get("key").toString());
|
||||
var input=ChatMeter.request(Map.of("model",model.get("id"),"messages",List.of(Map.of("role","user","content","你好")),"max_tokens",16),json);
|
||||
return new ChatFixture(context.getBean(ChatBillingService.class),key,model,input);
|
||||
}
|
||||
Map<String,Object> meteredResult() {
|
||||
return Map.of("choices",List.of(Map.of("message",Map.of("content","OK"),"finish_reason","stop")),"usage",Map.of("prompt_tokens",100,"completion_tokens",50,"prompt_tokens_details",Map.of("cached_tokens",60)));
|
||||
}
|
||||
@Test void actualUsageSettlesOnceWithSnapshotAndAtomicHistory() throws Exception {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
var ticket=f.reserve("same-call");
|
||||
var held=jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId);
|
||||
assertTrue(held.signum()>0);
|
||||
assertEquals("DUPLICATE_REQUEST",assertThrows(ApiException.class,()->f.reserve("same-call")).getCode());
|
||||
f.model().put("pricing",Map.of("input",999,"output",999,"unit","CNY / 1M tokens"));
|
||||
try(var executor=Executors.newFixedThreadPool(2)) {
|
||||
var a=executor.submit(()->f.billing().settle(ticket,meteredResult(),20,5));
|
||||
var b=executor.submit(()->f.billing().settle(ticket,meteredResult(),20,5));
|
||||
assertEquals(new BigDecimal("0.00034000"),a.get()); assertEquals(a.get(),b.get());
|
||||
}
|
||||
f.billing().fail(ticket,"LATE_FAILURE","Client closed after settlement",20);
|
||||
assertEquals(new BigDecimal("0.99966000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE request_id=?",Integer.class,ticket.id()));
|
||||
assertEquals(new BigDecimal("0.00034000"),jdbc.queryForObject("SELECT cost_amount FROM usage_events WHERE id=?",BigDecimal.class,ticket.id()));
|
||||
assertEquals(ticket.id().toString(),jdbc.queryForObject("SELECT request_id FROM usage_events WHERE id=?",String.class,ticket.id()));
|
||||
assertEquals(100,json.readTree(jdbc.queryForObject("SELECT output_payload::text FROM usage_events WHERE id=?",String.class,ticket.id())).at("/usage/prompt_tokens").intValue());
|
||||
assertEquals(2,wallet.entries(ownerToken,memberId).size());
|
||||
}
|
||||
@Test void reservationBlocksConcurrentSpendingAndAdminWithdrawals() throws Exception {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
jdbc.update("UPDATE user_wallets SET balance=0.004 WHERE user_id=?",memberId);
|
||||
try(var executor=Executors.newFixedThreadPool(2)) {
|
||||
var ready=new CountDownLatch(1);
|
||||
Callable<Object> attempt=()->{ ready.await(); try { return f.reserve(null); } catch(ApiException e) { return e.getCode(); } };
|
||||
var a=executor.submit(attempt); var b=executor.submit(attempt); ready.countDown();
|
||||
var results=List.of(a.get(),b.get());
|
||||
assertEquals(1,results.stream().filter(x -> x instanceof ChatBillingService.Ticket).count());
|
||||
assertTrue(results.contains("INSUFFICIENT_BALANCE"));
|
||||
}
|
||||
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->wallet.adjust(ownerToken,memberId,UUID.randomUUID(),"set","0","reserved balance test","0.004",meta)).getCode());
|
||||
}
|
||||
@Test void invalidUsageAndInterruptedCallsReleaseWithoutCharging() {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
var ticket=f.reserve(null);
|
||||
assertEquals("UPSTREAM_USAGE_MISSING",assertThrows(ApiException.class,()->f.billing().settle(ticket,Map.of(),1,null)).getCode());
|
||||
f.billing().fail(ticket,"UPSTREAM_USAGE_MISSING","No actual usage",1);
|
||||
f.billing().fail(ticket,"RETRY","Already released",1);
|
||||
var abandoned=f.reserve(null);
|
||||
jdbc.update("UPDATE chat_billing_requests SET expires_at=now()-interval '1 second' WHERE id=?",abandoned.id());
|
||||
f.billing().recoverExpired(); f.billing().recoverExpired();
|
||||
assertEquals(new BigDecimal("1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
|
||||
assertEquals(2,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE status='failed' AND cost_amount=0",Integer.class));
|
||||
}
|
||||
@Test void unexpectedProviderOverrunIsRecordedAsDebtAndBlocksNewCalls() {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
var ticket=f.reserve(null);
|
||||
f.billing().settle(ticket,Map.of("usage",Map.of("prompt_tokens",1_000_000,"completion_tokens",0)),10,null);
|
||||
assertEquals(new BigDecimal("-1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"0.50","partial debt repayment",meta);
|
||||
assertEquals(new BigDecimal("-0.50000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
}
|
||||
@Test void unpricedRoutesAndEmptyBalancesFailBeforeCreatingHolds() {
|
||||
var f=chatFixture();
|
||||
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
f.model().remove("pricing");
|
||||
assertEquals("MODEL_NOT_PRICED",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
f.model().put("pricing",Map.of("input",1,"output",2,"unit","USD / 1M tokens"));
|
||||
assertEquals("MODEL_NOT_PRICED",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM chat_billing_requests",Integer.class));
|
||||
}
|
||||
|
||||
@Test void gatewaySyncAndStreamRecordTheSameChargeAndPropagateFailures() throws Exception {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","gateway acceptance",meta);
|
||||
jdbc.update("UPDATE model_catalog SET pricing=?::jsonb WHERE id=?",json.writeValueAsString(f.model().get("pricing")),f.model().get("id"));
|
||||
String key=catalog.createKey(memberToken,Map.of("name","Local gateway test","modelIds",List.of(f.model().get("id"))),auth).get("key").toString();
|
||||
var upstream=com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress("127.0.0.1",0),0);
|
||||
var mode=new java.util.concurrent.atomic.AtomicReference<>("ok");
|
||||
var count=new java.util.concurrent.atomic.AtomicInteger();
|
||||
upstream.createContext("/v1/chat/completions",exchange->{
|
||||
count.incrementAndGet();
|
||||
var input=json.readTree(exchange.getRequestBody());
|
||||
if(mode.get().equals("http-error")) { exchange.sendResponseHeaders(500,-1); exchange.close(); return; }
|
||||
if(input.path("stream").asBoolean()) {
|
||||
exchange.getResponseHeaders().set("Content-Type","text/event-stream"); exchange.sendResponseHeaders(200,0);
|
||||
try(var out=exchange.getResponseBody()) {
|
||||
String frames="data: {\"model\":\"private-alias\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\n";
|
||||
if(!mode.get().equals("no-usage")) frames+="data: "+json.writeValueAsString(Map.of("choices",List.of(),"usage",meteredResult().get("usage")))+"\n\n";
|
||||
out.write((frames+"data: [DONE]\n\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
}
|
||||
} else {
|
||||
byte[] body=json.writeValueAsBytes(meteredResult()); exchange.getResponseHeaders().set("Content-Type","application/json"); exchange.sendResponseHeaders(200,body.length);
|
||||
try(var out=exchange.getResponseBody()) { out.write(body); }
|
||||
}
|
||||
}); upstream.start();
|
||||
var environment=new org.springframework.mock.env.MockEnvironment().withProperty("moyi.providers.deepseek.api-key","local-fake")
|
||||
.withProperty("moyi.providers.deepseek.base-url","http://127.0.0.1:"+upstream.getAddress().getPort()+"/v1");
|
||||
var provider=new com.moyi.controlplane.provider.OpenAiCompatibleProvider(json,environment,jdbc,new SecretCipher("local-test"));
|
||||
try {
|
||||
var controller=new com.moyi.controlplane.gateway.GatewayController(catalog,provider,org.mockito.Mockito.mock(com.moyi.controlplane.gateway.GatewayGuard.class),f.billing(),json);
|
||||
var http=MockMvcBuilders.standaloneSetup(controller).setControllerAdvice(new ApiExceptionHandler()).build();
|
||||
var input=new LinkedHashMap<>(f.request().body());
|
||||
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("Idempotency-Key","retry-test").contentType("application/json").content(json.writeValueAsBytes(input)))
|
||||
.andExpect(status().isOk()).andExpect(header().string("X-Moyi-Cost-CNY","0.00034000")).andExpect(jsonPath("$.model").value(f.model().get("id")));
|
||||
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("Idempotency-Key","retry-test").contentType("application/json").content(json.writeValueAsBytes(input))).andExpect(status().isConflict());
|
||||
assertEquals(1,count.get());
|
||||
input.put("stream",true);
|
||||
var streamed=http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("X-Moyi-Source","playground").contentType("application/json").content(json.writeValueAsBytes(input)))
|
||||
.andExpect(status().isOk()).andExpect(content().contentTypeCompatibleWith("text/event-stream")).andReturn().getResponse().getContentAsString();
|
||||
assertTrue(streamed.endsWith("data: [DONE]\n\n")); assertFalse(streamed.contains("private-alias"));
|
||||
assertEquals(new BigDecimal("0.99932000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE call_mode='stream' AND request_source='playground' AND cost_amount=0.00034 AND first_token_ms IS NOT NULL",Integer.class));
|
||||
mode.set("no-usage");
|
||||
var failed=http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).contentType("application/json").content(json.writeValueAsBytes(input))).andReturn().getResponse().getContentAsString();
|
||||
assertTrue(failed.contains("UPSTREAM_USAGE_MISSING"));
|
||||
mode.set("http-error");
|
||||
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).contentType("application/json").content(json.writeValueAsBytes(input))).andExpect(status().isBadGateway());
|
||||
assertEquals(new BigDecimal("0.99932000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
mode.set("ok");
|
||||
var disconnected=new org.springframework.mock.web.MockHttpServletResponse() {
|
||||
@Override public jakarta.servlet.ServletOutputStream getOutputStream() {
|
||||
return new jakarta.servlet.ServletOutputStream() {
|
||||
@Override public boolean isReady() { return true; }
|
||||
@Override public void setWriteListener(jakarta.servlet.WriteListener listener) { }
|
||||
@Override public void write(int b) throws java.io.IOException { throw new java.io.IOException("Browser closed"); }
|
||||
};
|
||||
}
|
||||
};
|
||||
controller.chatCompletions("Bearer "+key,input,"closed-browser",disconnected);
|
||||
assertEquals(new BigDecimal("0.99898000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(3,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=? AND operation='usage'",Integer.class,memberId));
|
||||
assertEquals(new BigDecimal("0.00102000"),catalog.overview(memberToken,auth).get("costThisMonth"));
|
||||
} finally { provider.close(); upstream.stop(0); }
|
||||
}
|
||||
|
||||
@Test void failedLedgerTransactionCannotPartiallyDebitTheWallet() {
|
||||
var f=chatFixture(); wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","atomicity test",meta);
|
||||
var ticket=f.reserve(null);
|
||||
jdbc.execute("ALTER TABLE usage_events ADD CONSTRAINT test_reject_cost CHECK (cost_amount IS NULL OR cost_amount=0)");
|
||||
assertThrows(org.springframework.dao.DataIntegrityViolationException.class,()->f.billing().settle(ticket,meteredResult(),5,null));
|
||||
assertEquals(new BigDecimal("1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
|
||||
assertEquals("pending",jdbc.queryForObject("SELECT status FROM chat_billing_requests WHERE id=?",String.class,ticket.id()));
|
||||
f.billing().fail(ticket,"SETTLEMENT_FAILED","Test rollback",5);
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package com.moyi.controlplane.gateway;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.mock.env.MockEnvironment;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class ChatStreamingTest {
|
||||
HttpServer server;
|
||||
OpenAiCompatibleProvider provider;
|
||||
ObjectMapper json=new ObjectMapper();
|
||||
Map<String,Object> request=Map.of("messages",List.of(Map.of("role","user","content","测试")),"max_tokens",16,"temperature",0.2,"stream",true);
|
||||
@BeforeEach void setup() throws Exception {
|
||||
server=HttpServer.create(new InetSocketAddress("127.0.0.1",0),0);
|
||||
var env=new MockEnvironment().withProperty("moyi.providers.deepseek.api-key","local-fake-key")
|
||||
.withProperty("moyi.providers.deepseek.base-url","http://127.0.0.1:"+server.getAddress().getPort()+"/v1")
|
||||
.withProperty("moyi.gateway.upstream-timeout-seconds","2");
|
||||
provider=new OpenAiCompatibleProvider(json,env,mock(JdbcTemplate.class),new SecretCipher("local-test-only"));
|
||||
}
|
||||
@AfterEach void close() { provider.close(); server.stop(0); }
|
||||
@Test void fragmentedUtf8StreamsBeforeCompletionAndCarriesActualUsage() throws Exception {
|
||||
var first=new CountDownLatch(1); var finish=new CountDownLatch(1);
|
||||
var received=new CompletableFuture<Map<?,?>>();
|
||||
server.createContext("/v1/chat/completions", exchange -> {
|
||||
received.complete(json.readValue(exchange.getRequestBody(),Map.class));
|
||||
exchange.getResponseHeaders().set("Content-Type","text/event-stream"); exchange.sendResponseHeaders(200,0);
|
||||
try(var out=exchange.getResponseBody()) {
|
||||
byte[] initial="data: {\"id\":\"test\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"你好\"}}]}\r\n\r\n".getBytes(StandardCharsets.UTF_8);
|
||||
for(byte b:initial) { out.write(b); out.flush(); }
|
||||
try { if(!finish.await(1,TimeUnit.SECONDS)) throw new AssertionError("first delta was buffered"); } catch(InterruptedException e) { Thread.currentThread().interrupt(); }
|
||||
out.write(("data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n"
|
||||
+"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":20,\"completion_tokens\":2,\"prompt_tokens_details\":{\"cached_tokens\":10}}}\n\n"
|
||||
+"data: [DONE]\n\n").getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
}); server.start();
|
||||
try(var pool=Executors.newSingleThreadExecutor()) {
|
||||
var result=pool.submit(()->{
|
||||
var collector=new ChatStreamCollector("public-route",System.currentTimeMillis());
|
||||
try(var stream=provider.stream("provider-deepseek","upstream-alias",request)) {
|
||||
stream.events(chunk->{ collector.accept(chunk); first.countDown(); });
|
||||
}
|
||||
assertNotNull(collector.firstTokenMs()); return collector.result();
|
||||
});
|
||||
assertTrue(first.await(1500,TimeUnit.MILLISECONDS)); assertFalse(result.isDone()); finish.countDown();
|
||||
var body=result.get(2,TimeUnit.SECONDS);
|
||||
assertEquals("你好",((Map<?,?>)((Map<?,?>)((List<?>)body.get("choices")).getFirst()).get("message")).get("content"));
|
||||
assertEquals(new ChatMeter.Tokens(20,2,10),ChatMeter.tokens(body));
|
||||
assertEquals("public-route",body.get("model"));
|
||||
}
|
||||
assertEquals("upstream-alias",received.get().get("model"));
|
||||
assertEquals(16,received.get().get("max_tokens"));
|
||||
assertEquals(Map.of("include_usage",true),received.get().get("stream_options"));
|
||||
}
|
||||
@Test void deadlineClosesAStreamThatNeverFinishes() {
|
||||
server.createContext("/v1/chat/completions", exchange->{
|
||||
exchange.getResponseHeaders().set("Content-Type","text/event-stream"); exchange.sendResponseHeaders(200,0);
|
||||
exchange.getResponseBody().write(": heartbeat\n\n".getBytes()); exchange.getResponseBody().flush();
|
||||
// Leave the body open: the client hard deadline must terminate the read.
|
||||
}); server.start();
|
||||
long started=System.nanoTime();
|
||||
assertThrows(Exception.class,()->{ try(var stream=provider.stream("provider-deepseek","alias",request)) { stream.events(chunk->{}); } });
|
||||
assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)<4000);
|
||||
}
|
||||
@Test void upstreamHttpFailureNeverBecomesSuccessfulStream() {
|
||||
server.createContext("/v1/chat/completions",e->{ e.sendResponseHeaders(401,-1); e.close(); }); server.start();
|
||||
assertEquals(502,assertThrows(ApiException.class,()->provider.stream("provider-deepseek","alias",request)).getStatus());
|
||||
}
|
||||
@Test void eofWithoutDoneAndMissingUsageAreNotBillable() {
|
||||
server.createContext("/v1/chat/completions",e->{
|
||||
e.getResponseHeaders().set("Content-Type","text/event-stream"); e.sendResponseHeaders(200,0);
|
||||
e.getResponseBody().write("data: {\"choices\":[]}\n\n".getBytes()); e.close();
|
||||
}); server.start();
|
||||
assertEquals("UPSTREAM_STREAM_INCOMPLETE",assertThrows(ApiException.class,()->{ try(var stream=provider.stream("provider-deepseek","alias",request)) { stream.events(chunk->{}); } }).getCode());
|
||||
assertThrows(ApiException.class,()->ChatMeter.tokens(Map.of("usage",Map.of("prompt_tokens",1.5,"completion_tokens",2))));
|
||||
assertThrows(ApiException.class,()->ChatMeter.tokens(Map.of("usage",Map.of("prompt_tokens",1,"completion_tokens",2,"prompt_cache_hit_tokens",3))));
|
||||
}
|
||||
@Test void tokenBoundsAndUnsupportedMultimodalAreRejectedBeforeUpstream() {
|
||||
assertThrows(ApiException.class,()->ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content",List.of(Map.of("type","image_url"))))),json));
|
||||
assertThrows(ApiException.class,()->ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content","test")),"max_tokens",0),json));
|
||||
assertEquals(4096,ChatMeter.request(Map.of("messages",List.of(Map.of("role","user","content","test"))),json).maxOutputTokens());
|
||||
}
|
||||
}
|
||||
+10
-5
@@ -6,6 +6,8 @@ import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
|
||||
import com.moyi.controlplane.task.AsyncTaskService;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import com.moyi.controlplane.wallet.ChatBillingService;
|
||||
import org.springframework.mock.web.MockHttpServletResponse;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
@@ -42,11 +44,11 @@ class GatewayDeploymentTest {
|
||||
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
||||
when(catalog.activeModel("model-real")).thenReturn(Map.of("id", "model-real", "providerId", "provider-deepseek", "mode", "sync"));
|
||||
when(catalog.keyCanInvoke(key, "model-real")).thenReturn(true);
|
||||
GatewayController controller = new GatewayController(catalog, provider, guard, usage);
|
||||
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "model-real")));
|
||||
GatewayController controller = new GatewayController(catalog, provider, guard, mock(ChatBillingService.class),new ObjectMapper());
|
||||
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "model-real"),null,new MockHttpServletResponse()));
|
||||
assertEquals("PROVIDER_NOT_CONFIGURED", error.getCode());
|
||||
assertEquals(503, error.getStatus());
|
||||
verify(provider, never()).chat(anyString(), anyString(), anyList());
|
||||
verify(provider, never()).chat(anyString(), anyString(), anyMap());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -56,8 +58,11 @@ class GatewayDeploymentTest {
|
||||
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
||||
when(catalog.activeModel("mock")).thenReturn(Map.of("id", "mock", "providerId", "provider-mock", "mode", "sync"));
|
||||
when(catalog.keyCanInvoke(key, "mock")).thenReturn(true);
|
||||
GatewayController controller = new GatewayController(catalog, mock(OpenAiCompatibleProvider.class), mock(GatewayGuard.class), mock(UsageAuditService.class));
|
||||
assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock")).getStatusCode().value());
|
||||
var billing=mock(ChatBillingService.class);
|
||||
when(billing.reserve(anyMap(),anyMap(),any(),any())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(),UUID.fromString(key.get("userId").toString()),ChatMeter.rates(null,true)));
|
||||
when(billing.settle(any(),anyMap(),anyInt(),any())).thenReturn(java.math.BigDecimal.ZERO);
|
||||
GatewayController controller = new GatewayController(catalog, mock(OpenAiCompatibleProvider.class), mock(GatewayGuard.class), billing,new ObjectMapper());
|
||||
assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock","messages",List.of(Map.of("role","user","content","test"))),null,new MockHttpServletResponse()).getStatusCode().value());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in new issue
Block a user