feat: route upstream keys by model group
This commit is contained in:
1 parent
ce0a718686
commit
0876dfe08e
9 files changed
+195
-41
No files matched your search
+6
-5
@@ -83,8 +83,9 @@ public class GatewayController {
|
||||
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线;请使用当前 API Key 的 /v1/models 返回的模型 ID", 404);
|
||||
if ("async".equals(model.get("mode"))) throw new ApiException("ASYNC_MODEL", "该模型需要使用异步任务接口", 400);
|
||||
String providerId=String.valueOf(model.get("providerId"));
|
||||
String routeId=String.valueOf(model.get("id"));
|
||||
boolean mock="provider-mock".equals(providerId);
|
||||
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||
if(!mock && !provider.isConfigured(providerId, routeId, String.valueOf(model.get("upstreamModel")))) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||
// Reserve both limits atomically. A queued request never consumes a
|
||||
@@ -95,10 +96,10 @@ public class GatewayController {
|
||||
try {
|
||||
catalogService.touchKey(key);
|
||||
if(Boolean.TRUE.equals(request.body().get("stream"))) {
|
||||
stream(model,modelId,request.body(),ticket,servletResponse,startedAt,mock);
|
||||
stream(model,modelId,routeId,request.body(),ticket,servletResponse,startedAt,mock);
|
||||
return null;
|
||||
}
|
||||
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),request.body());
|
||||
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),routeId,request.body());
|
||||
body.put("model",modelId);
|
||||
requireCompletion(body);
|
||||
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
|
||||
@@ -117,7 +118,7 @@ public class GatewayController {
|
||||
}
|
||||
}
|
||||
|
||||
private void stream(Map<String,Object> model, String modelId, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
||||
private void stream(Map<String,Object> model, String modelId, String routeId, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
||||
if(mock) {
|
||||
var body=mockResponse(modelId,input);
|
||||
prepareStream(response);
|
||||
@@ -130,7 +131,7 @@ public class GatewayController {
|
||||
}
|
||||
// Keep admission, the concurrency slot and the upstream stream alive on this request thread.
|
||||
// Upstream HTTP errors are inspected before committing a 200/SSE response.
|
||||
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),input)) {
|
||||
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),routeId,input)) {
|
||||
var collector=new ChatStreamCollector(modelId,started);
|
||||
prepareStream(response);
|
||||
boolean[] disconnected={false};
|
||||
|
||||
Reference in new issue
Block a user