feat: modularize gateway requests and recover settlements
This commit is contained in:
1 parent
7785cda6a6
commit
5533a483d9
40 files changed
+1598
-232
No files matched your search
+42
-33
@@ -4,7 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.catalog.CatalogService;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
|
||||
import com.moyi.controlplane.wallet.ChatBillingService;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestContext;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestProtocol;
|
||||
import com.moyi.controlplane.gateway.request.GatewayRequestExecutionService;
|
||||
import jakarta.servlet.http.*;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -15,9 +18,9 @@ import org.springframework.web.bind.annotation.*;
|
||||
@RestController
|
||||
public class NativeGatewayController {
|
||||
private final CatalogService catalog; private final GatewayGuard guard;
|
||||
private final ChatBillingService billing; private final OpenAiCompatibleProvider provider; private final ObjectMapper json;
|
||||
private final GatewayRequestExecutionService requests; private final OpenAiCompatibleProvider provider; private final ObjectMapper json;
|
||||
@org.springframework.beans.factory.annotation.Autowired private GatewayFailover failover;
|
||||
public NativeGatewayController(CatalogService catalog,GatewayGuard guard,ChatBillingService billing,OpenAiCompatibleProvider provider,ObjectMapper json) {this.catalog=catalog;this.guard=guard;this.billing=billing;this.provider=provider;this.json=json;}
|
||||
public NativeGatewayController(CatalogService catalog,GatewayGuard guard,GatewayRequestExecutionService requests,OpenAiCompatibleProvider provider,ObjectMapper json) {this.catalog=catalog;this.guard=guard;this.requests=requests;this.provider=provider;this.json=json;}
|
||||
@PostMapping({"/v1/embeddings","/v1/messages","/v1/responses"})
|
||||
public ResponseEntity<Map<String,Object>> invoke(@RequestBody Map<String,Object> input,HttpServletRequest request,HttpServletResponse response) {
|
||||
String protocol=request.getRequestURI().substring(request.getRequestURI().lastIndexOf('/')+1);
|
||||
@@ -41,7 +44,11 @@ public class NativeGatewayController {
|
||||
return executeForKey(protocol,input,streaming,key,request,response);
|
||||
}
|
||||
public ResponseEntity<Map<String,Object>> executeForKey(String protocol,Map<String,Object> input,boolean streaming,Map<String,Object> key,HttpServletRequest request,HttpServletResponse response) {
|
||||
UUID keyId=UUID.fromString(key.get("id").toString());guard.checkRate(keyId,false);
|
||||
return executeForKey(protocol,input,streaming,key,request,response,ApiRequestMetadata.fromHttp(request.getRemoteAddr()));
|
||||
}
|
||||
public ResponseEntity<Map<String,Object>> executeForKey(String protocol,Map<String,Object> input,boolean streaming,Map<String,Object> key,HttpServletRequest request,HttpServletResponse response,ApiRequestMetadata metadata) {
|
||||
UUID keyId=UUID.fromString(key.get("id").toString());
|
||||
guard.checkRate(keyId,false);
|
||||
String requested=String.valueOf(input.getOrDefault("model",""));
|
||||
var model=GatewayModelIds.resolve(catalog,key,requested);
|
||||
if(model==null) throw new ApiException("MODEL_NOT_FOUND","模型不存在或未授权",404);
|
||||
@@ -50,38 +57,40 @@ public class NativeGatewayController {
|
||||
if(!provider.isConfigured(providerId,route,upstream)) throw new ApiException("PROVIDER_NOT_CONFIGURED","渠道凭据未配置",503);
|
||||
var normalized=NativeProtocolMeter.request(protocol,input,json);
|
||||
long start=System.currentTimeMillis();
|
||||
try(var admission=guard.acquire(model,keyId,guard.queueWaitMs())) {
|
||||
var ticket=billing.reserve(key,model,normalized,request.getHeader("Idempotency-Key"));
|
||||
billing.endpoint(ticket.id(),"gemini".equals(protocol)?"/v1beta/models:generateContent":"/v1/"+protocol);
|
||||
ApiRequestProtocol requestProtocol = ApiRequestProtocol.from(protocol);
|
||||
ApiRequestContext context = new ApiRequestContext(UUID.fromString(key.get("userId").toString()), keyId,
|
||||
requestProtocol, requested, route, streaming, normalized.reservedInputTokens(), normalized.maxOutputTokens(),
|
||||
metadata, System.nanoTime());
|
||||
return requests.execute(context, key, model, normalized, request.getHeader("Idempotency-Key"), () -> {
|
||||
return guard.acquire(model,keyId,guard.queueWaitMs());
|
||||
}, ticket -> {
|
||||
requests.endpoint(ticket,"gemini".equals(protocol)?"/v1beta/models:generateContent":"/v1/"+protocol);
|
||||
response.setHeader("X-Request-Id",ticket.id().toString());
|
||||
try {
|
||||
catalog.touchKey(key);
|
||||
if(!streaming) {
|
||||
var result=failover!=null?failover.invoke(key,model,protocol,normalized.body(),ticket):provider.invoke(providerId,upstream,route,normalized.body(),protocol);
|
||||
if(result.containsKey("model")) result.put("model",requested);
|
||||
billing.observedUsage(ticket,protocol,result.get("gemini".equals(protocol)?"usageMetadata":"usage"));
|
||||
var cost=billing.settle(ticket,NativeProtocolMeter.normalize(protocol,result),elapsed(start),null);
|
||||
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(result);
|
||||
}
|
||||
try(var stream=failover!=null?failover.stream(key,model,protocol,normalized.body(),ticket):provider.nativeStream(providerId,upstream,route,normalized.body(),protocol)) {
|
||||
var collector=new NativeProtocolMeter.Collector(protocol);
|
||||
response.setContentType("text/event-stream");response.setCharacterEncoding("UTF-8");response.setHeader("Cache-Control","no-cache");response.setHeader("X-Accel-Buffering","no");
|
||||
boolean[] disconnected={false};Integer[] first={null};
|
||||
try {stream.events(chunk->{
|
||||
if(first[0]==null) first[0]=elapsed(start);
|
||||
collector.accept(chunk);
|
||||
if(!disconnected[0]) try {event(response,chunk);} catch(IOException e) {disconnected[0]=true;}
|
||||
},false);} finally {billing.observedUsage(ticket,protocol,collector.observedUsage());}
|
||||
billing.settle(ticket,collector.result(),elapsed(start),first[0]);
|
||||
}
|
||||
return null;
|
||||
} catch(Exception failure) {
|
||||
var error=failure instanceof ApiException api?api:new ApiException("CALL_INTERRUPTED","调用中断,渠道成本待核对",502);
|
||||
billing.fail(ticket,error.getCode(),error.getMessage(),elapsed(start));
|
||||
catalog.touchKey(key);
|
||||
if(!streaming) {
|
||||
var result=failover!=null?failover.invoke(key,model,protocol,normalized.body(),ticket):provider.invoke(providerId,upstream,route,normalized.body(),protocol);
|
||||
if(result.containsKey("model")) result.put("model",requested);
|
||||
requests.observedUsage(ticket,protocol,result.get("gemini".equals(protocol)?"usageMetadata":"usage"));
|
||||
var cost=requests.settle(ticket,NativeProtocolMeter.normalize(protocol,result),elapsed(start),null);
|
||||
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(result);
|
||||
}
|
||||
try(var stream=failover!=null?failover.stream(key,model,protocol,normalized.body(),ticket):provider.nativeStream(providerId,upstream,route,normalized.body(),protocol)) {
|
||||
var collector=new NativeProtocolMeter.Collector(protocol);
|
||||
response.setContentType("text/event-stream");response.setCharacterEncoding("UTF-8");response.setHeader("Cache-Control","no-cache");response.setHeader("X-Accel-Buffering","no");
|
||||
boolean[] disconnected={false};Integer[] first={null};
|
||||
try {stream.events(chunk->{
|
||||
if(first[0]==null) first[0]=elapsed(start);
|
||||
collector.accept(chunk);
|
||||
if(!disconnected[0]) try {event(response,chunk);} catch(IOException e) {disconnected[0]=true;}
|
||||
},false);} finally {requests.observedUsage(ticket,protocol,collector.observedUsage());}
|
||||
requests.settle(ticket,collector.result(),elapsed(start),first[0]);
|
||||
}
|
||||
return null;
|
||||
}, failure -> failure instanceof ApiException api ? api
|
||||
: new ApiException("CALL_INTERRUPTED","调用中断,渠道成本待核对",502), error -> {
|
||||
if(response.isCommitted()) {try {event(response,Map.of("type","error","error",Map.of("type",error.getCode(),"message",error.getMessage())));} catch(IOException ignored) {}return null;}
|
||||
response.resetBuffer();response.setContentType("application/json");throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
private int elapsed(long start) {return (int)Math.min(Integer.MAX_VALUE,System.currentTimeMillis()-start);}
|
||||
private void event(HttpServletResponse response,Map<String,Object> chunk) throws IOException {
|
||||
|
||||
Reference in new issue
Block a user