feat: stream chat and settle metered CNY usage atomically
This commit is contained in:
1 parent
7360cf3ebe
commit
b46ee72f2d
28 files changed
+1098
-86
No files matched your search
+168
-1
@@ -7,6 +7,8 @@ import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.common.ApiExceptionHandler;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.moyi.controlplane.wallet.WalletService;
|
||||
import com.moyi.controlplane.wallet.ChatBillingService;
|
||||
import com.moyi.controlplane.gateway.ChatMeter;
|
||||
import com.moyi.controlplane.catalog.CatalogController;
|
||||
import com.moyi.controlplane.usage.*;
|
||||
import jakarta.servlet.http.Cookie;
|
||||
@@ -78,6 +80,8 @@ class PlatformAdminPostgresTest {
|
||||
context.registerBean(AuthService.class, () -> new AuthService(jdbc, new PasswordHasher("test-pepper"), json, 168));
|
||||
context.registerBean(CatalogService.class, () -> new CatalogService(jdbc, json, new SecretCipher("test-encryption")));
|
||||
context.registerBean(WalletService.class, () -> new WalletService(jdbc, context.getBean(AuthService.class)));
|
||||
context.registerBean(UsageAuditService.class, () -> new UsageAuditService(jdbc,json));
|
||||
context.registerBean(ChatBillingService.class, () -> new ChatBillingService(jdbc,json,context.getBean(UsageAuditService.class)));
|
||||
context.registerBean(VerificationService.class, () -> new VerificationService(jdbc, sender, "test-pepper"));
|
||||
context.refresh();
|
||||
auth = context.getBean(AuthService.class);
|
||||
@@ -268,7 +272,7 @@ class PlatformAdminPostgresTest {
|
||||
org.springframework.web.context.request.RequestContextHolder.setRequestAttributes(new org.springframework.web.context.request.ServletRequestAttributes(request));
|
||||
try {
|
||||
audit.recordCall(memberId,null,groupId,providerId,"succeeded",200,100,20,
|
||||
Map.of("prompt","hello","authorization","hidden"),
|
||||
Map.of("prompt","hello","authorization","hidden","accessToken","hidden","refresh_token","hidden","prompt_tokens","hidden"),
|
||||
Map.of("usage",Map.of("prompt_tokens_details",Map.of("cached_tokens",80),"cache_creation_input_tokens",5)),null,null);
|
||||
} finally { org.springframework.web.context.request.RequestContextHolder.resetRequestAttributes(); }
|
||||
var saved=jdbc.queryForMap("SELECT * FROM usage_events WHERE user_id=?",memberId);
|
||||
@@ -280,6 +284,7 @@ class PlatformAdminPostgresTest {
|
||||
assertNull(saved.get("cost_amount"));
|
||||
assertNull(saved.get("first_token_ms"));
|
||||
assertFalse(saved.get("input_payload").toString().contains("hidden"));
|
||||
assertEquals(80,json.readTree(saved.get("output_payload").toString()).path("usage").path("prompt_tokens_details").path("cached_tokens").asInt());
|
||||
audit.recordTask(memberId,null,groupId,providerId,"local-timing-task","queued",Map.of("prompt","hello"),null,null,null);
|
||||
jdbc.update("UPDATE usage_events SET created_at=now()-interval '2 seconds' WHERE provider_task_id='local-timing-task'");
|
||||
audit.updateTask("local-timing-task","succeeded",Map.of("url","https://example.test/result"),null,null);
|
||||
@@ -329,4 +334,166 @@ class PlatformAdminPostgresTest {
|
||||
assertFalse(verification.capabilities().containsValue(true));
|
||||
assertThrows(ApiException.class, () -> verification.send("unavailable@example.test",invite,"127.0.0.1"));
|
||||
}
|
||||
|
||||
record ChatFixture(ChatBillingService billing, Map<String,Object> key, Map<String,Object> model, ChatMeter.Request request) {
|
||||
ChatBillingService.Ticket reserve(String idem) { return billing.reserve(key,model,request,idem); }
|
||||
}
|
||||
ChatFixture chatFixture() {
|
||||
var model=new LinkedHashMap<>(catalog.listModels(false).stream().filter(m -> "provider-deepseek".equals(m.get("providerId"))).findFirst().orElseThrow());
|
||||
model.put("pricing",Map.of("input",2,"output",4,"cachedInput",1,"unit","CNY / 1M tokens"));
|
||||
var created=catalog.createKey(memberToken,Map.of("name","Local billing test","modelIds",List.of(model.get("id"))),auth);
|
||||
var key=catalog.authenticateKey(created.get("key").toString());
|
||||
var input=ChatMeter.request(Map.of("model",model.get("id"),"messages",List.of(Map.of("role","user","content","你好")),"max_tokens",16),json);
|
||||
return new ChatFixture(context.getBean(ChatBillingService.class),key,model,input);
|
||||
}
|
||||
Map<String,Object> meteredResult() {
|
||||
return Map.of("choices",List.of(Map.of("message",Map.of("content","OK"),"finish_reason","stop")),"usage",Map.of("prompt_tokens",100,"completion_tokens",50,"prompt_tokens_details",Map.of("cached_tokens",60)));
|
||||
}
|
||||
@Test void actualUsageSettlesOnceWithSnapshotAndAtomicHistory() throws Exception {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
var ticket=f.reserve("same-call");
|
||||
var held=jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId);
|
||||
assertTrue(held.signum()>0);
|
||||
assertEquals("DUPLICATE_REQUEST",assertThrows(ApiException.class,()->f.reserve("same-call")).getCode());
|
||||
f.model().put("pricing",Map.of("input",999,"output",999,"unit","CNY / 1M tokens"));
|
||||
try(var executor=Executors.newFixedThreadPool(2)) {
|
||||
var a=executor.submit(()->f.billing().settle(ticket,meteredResult(),20,5));
|
||||
var b=executor.submit(()->f.billing().settle(ticket,meteredResult(),20,5));
|
||||
assertEquals(new BigDecimal("0.00034000"),a.get()); assertEquals(a.get(),b.get());
|
||||
}
|
||||
f.billing().fail(ticket,"LATE_FAILURE","Client closed after settlement",20);
|
||||
assertEquals(new BigDecimal("0.99966000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE request_id=?",Integer.class,ticket.id()));
|
||||
assertEquals(new BigDecimal("0.00034000"),jdbc.queryForObject("SELECT cost_amount FROM usage_events WHERE id=?",BigDecimal.class,ticket.id()));
|
||||
assertEquals(ticket.id().toString(),jdbc.queryForObject("SELECT request_id FROM usage_events WHERE id=?",String.class,ticket.id()));
|
||||
assertEquals(100,json.readTree(jdbc.queryForObject("SELECT output_payload::text FROM usage_events WHERE id=?",String.class,ticket.id())).at("/usage/prompt_tokens").intValue());
|
||||
assertEquals(2,wallet.entries(ownerToken,memberId).size());
|
||||
}
|
||||
@Test void reservationBlocksConcurrentSpendingAndAdminWithdrawals() throws Exception {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
jdbc.update("UPDATE user_wallets SET balance=0.004 WHERE user_id=?",memberId);
|
||||
try(var executor=Executors.newFixedThreadPool(2)) {
|
||||
var ready=new CountDownLatch(1);
|
||||
Callable<Object> attempt=()->{ ready.await(); try { return f.reserve(null); } catch(ApiException e) { return e.getCode(); } };
|
||||
var a=executor.submit(attempt); var b=executor.submit(attempt); ready.countDown();
|
||||
var results=List.of(a.get(),b.get());
|
||||
assertEquals(1,results.stream().filter(x -> x instanceof ChatBillingService.Ticket).count());
|
||||
assertTrue(results.contains("INSUFFICIENT_BALANCE"));
|
||||
}
|
||||
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->wallet.adjust(ownerToken,memberId,UUID.randomUUID(),"set","0","reserved balance test","0.004",meta)).getCode());
|
||||
}
|
||||
@Test void invalidUsageAndInterruptedCallsReleaseWithoutCharging() {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
var ticket=f.reserve(null);
|
||||
assertEquals("UPSTREAM_USAGE_MISSING",assertThrows(ApiException.class,()->f.billing().settle(ticket,Map.of(),1,null)).getCode());
|
||||
f.billing().fail(ticket,"UPSTREAM_USAGE_MISSING","No actual usage",1);
|
||||
f.billing().fail(ticket,"RETRY","Already released",1);
|
||||
var abandoned=f.reserve(null);
|
||||
jdbc.update("UPDATE chat_billing_requests SET expires_at=now()-interval '1 second' WHERE id=?",abandoned.id());
|
||||
f.billing().recoverExpired(); f.billing().recoverExpired();
|
||||
assertEquals(new BigDecimal("1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
|
||||
assertEquals(2,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE status='failed' AND cost_amount=0",Integer.class));
|
||||
}
|
||||
@Test void unexpectedProviderOverrunIsRecordedAsDebtAndBlocksNewCalls() {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
var ticket=f.reserve(null);
|
||||
f.billing().settle(ticket,Map.of("usage",Map.of("prompt_tokens",1_000_000,"completion_tokens",0)),10,null);
|
||||
assertEquals(new BigDecimal("-1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"0.50","partial debt repayment",meta);
|
||||
assertEquals(new BigDecimal("-0.50000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
}
|
||||
@Test void unpricedRoutesAndEmptyBalancesFailBeforeCreatingHolds() {
|
||||
var f=chatFixture();
|
||||
assertEquals("INSUFFICIENT_BALANCE",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
f.model().remove("pricing");
|
||||
assertEquals("MODEL_NOT_PRICED",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
f.model().put("pricing",Map.of("input",1,"output",2,"unit","USD / 1M tokens"));
|
||||
assertEquals("MODEL_NOT_PRICED",assertThrows(ApiException.class,()->f.reserve(null)).getCode());
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM chat_billing_requests",Integer.class));
|
||||
}
|
||||
|
||||
@Test void gatewaySyncAndStreamRecordTheSameChargeAndPropagateFailures() throws Exception {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","gateway acceptance",meta);
|
||||
jdbc.update("UPDATE model_catalog SET pricing=?::jsonb WHERE id=?",json.writeValueAsString(f.model().get("pricing")),f.model().get("id"));
|
||||
String key=catalog.createKey(memberToken,Map.of("name","Local gateway test","modelIds",List.of(f.model().get("id"))),auth).get("key").toString();
|
||||
var upstream=com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress("127.0.0.1",0),0);
|
||||
var mode=new java.util.concurrent.atomic.AtomicReference<>("ok");
|
||||
var count=new java.util.concurrent.atomic.AtomicInteger();
|
||||
upstream.createContext("/v1/chat/completions",exchange->{
|
||||
count.incrementAndGet();
|
||||
var input=json.readTree(exchange.getRequestBody());
|
||||
if(mode.get().equals("http-error")) { exchange.sendResponseHeaders(500,-1); exchange.close(); return; }
|
||||
if(input.path("stream").asBoolean()) {
|
||||
exchange.getResponseHeaders().set("Content-Type","text/event-stream"); exchange.sendResponseHeaders(200,0);
|
||||
try(var out=exchange.getResponseBody()) {
|
||||
String frames="data: {\"model\":\"private-alias\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\n";
|
||||
if(!mode.get().equals("no-usage")) frames+="data: "+json.writeValueAsString(Map.of("choices",List.of(),"usage",meteredResult().get("usage")))+"\n\n";
|
||||
out.write((frames+"data: [DONE]\n\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
}
|
||||
} else {
|
||||
byte[] body=json.writeValueAsBytes(meteredResult()); exchange.getResponseHeaders().set("Content-Type","application/json"); exchange.sendResponseHeaders(200,body.length);
|
||||
try(var out=exchange.getResponseBody()) { out.write(body); }
|
||||
}
|
||||
}); upstream.start();
|
||||
var environment=new org.springframework.mock.env.MockEnvironment().withProperty("moyi.providers.deepseek.api-key","local-fake")
|
||||
.withProperty("moyi.providers.deepseek.base-url","http://127.0.0.1:"+upstream.getAddress().getPort()+"/v1");
|
||||
var provider=new com.moyi.controlplane.provider.OpenAiCompatibleProvider(json,environment,jdbc,new SecretCipher("local-test"));
|
||||
try {
|
||||
var controller=new com.moyi.controlplane.gateway.GatewayController(catalog,provider,org.mockito.Mockito.mock(com.moyi.controlplane.gateway.GatewayGuard.class),f.billing(),json);
|
||||
var http=MockMvcBuilders.standaloneSetup(controller).setControllerAdvice(new ApiExceptionHandler()).build();
|
||||
var input=new LinkedHashMap<>(f.request().body());
|
||||
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("Idempotency-Key","retry-test").contentType("application/json").content(json.writeValueAsBytes(input)))
|
||||
.andExpect(status().isOk()).andExpect(header().string("X-Moyi-Cost-CNY","0.00034000")).andExpect(jsonPath("$.model").value(f.model().get("id")));
|
||||
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("Idempotency-Key","retry-test").contentType("application/json").content(json.writeValueAsBytes(input))).andExpect(status().isConflict());
|
||||
assertEquals(1,count.get());
|
||||
input.put("stream",true);
|
||||
var streamed=http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).header("X-Moyi-Source","playground").contentType("application/json").content(json.writeValueAsBytes(input)))
|
||||
.andExpect(status().isOk()).andExpect(content().contentTypeCompatibleWith("text/event-stream")).andReturn().getResponse().getContentAsString();
|
||||
assertTrue(streamed.endsWith("data: [DONE]\n\n")); assertFalse(streamed.contains("private-alias"));
|
||||
assertEquals(new BigDecimal("0.99932000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE call_mode='stream' AND request_source='playground' AND cost_amount=0.00034 AND first_token_ms IS NOT NULL",Integer.class));
|
||||
mode.set("no-usage");
|
||||
var failed=http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).contentType("application/json").content(json.writeValueAsBytes(input))).andReturn().getResponse().getContentAsString();
|
||||
assertTrue(failed.contains("UPSTREAM_USAGE_MISSING"));
|
||||
mode.set("http-error");
|
||||
http.perform(post("/v1/chat/completions").header("Authorization","Bearer "+key).contentType("application/json").content(json.writeValueAsBytes(input))).andExpect(status().isBadGateway());
|
||||
assertEquals(new BigDecimal("0.99932000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
mode.set("ok");
|
||||
var disconnected=new org.springframework.mock.web.MockHttpServletResponse() {
|
||||
@Override public jakarta.servlet.ServletOutputStream getOutputStream() {
|
||||
return new jakarta.servlet.ServletOutputStream() {
|
||||
@Override public boolean isReady() { return true; }
|
||||
@Override public void setWriteListener(jakarta.servlet.WriteListener listener) { }
|
||||
@Override public void write(int b) throws java.io.IOException { throw new java.io.IOException("Browser closed"); }
|
||||
};
|
||||
}
|
||||
};
|
||||
controller.chatCompletions("Bearer "+key,input,"closed-browser",disconnected);
|
||||
assertEquals(new BigDecimal("0.99898000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(3,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=? AND operation='usage'",Integer.class,memberId));
|
||||
assertEquals(new BigDecimal("0.00102000"),catalog.overview(memberToken,auth).get("costThisMonth"));
|
||||
} finally { provider.close(); upstream.stop(0); }
|
||||
}
|
||||
|
||||
@Test void failedLedgerTransactionCannotPartiallyDebitTheWallet() {
|
||||
var f=chatFixture(); wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","atomicity test",meta);
|
||||
var ticket=f.reserve(null);
|
||||
jdbc.execute("ALTER TABLE usage_events ADD CONSTRAINT test_reject_cost CHECK (cost_amount IS NULL OR cost_amount=0)");
|
||||
assertThrows(org.springframework.dao.DataIntegrityViolationException.class,()->f.billing().settle(ticket,meteredResult(),5,null));
|
||||
assertEquals(new BigDecimal("1.00000000"),jdbc.queryForObject("SELECT balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
|
||||
assertEquals("pending",jdbc.queryForObject("SELECT status FROM chat_billing_requests WHERE id=?",String.class,ticket.id()));
|
||||
f.billing().fail(ticket,"SETTLEMENT_FAILED","Test rollback",5);
|
||||
assertEquals(0,jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user