feat: extend enterprise gateway policies billing protocols and operations
This commit is contained in:
1 parent
1528b71882
commit
af43947b5d
67 files changed
+1822
-69
No files matched your search
+73
-13
@@ -40,6 +40,7 @@ public class OpenAiCompatibleProvider {
|
||||
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
|
||||
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
||||
private final int timeoutSeconds;
|
||||
@org.springframework.beans.factory.annotation.Autowired private UpstreamHealth health;
|
||||
|
||||
public OpenAiCompatibleProvider(
|
||||
ObjectMapper objectMapper,
|
||||
@@ -99,47 +100,100 @@ public class OpenAiCompatibleProvider {
|
||||
}
|
||||
|
||||
private StreamResponse open(String providerId, String upstreamModel, String routeId, Map<String,Object> input, boolean streaming) {
|
||||
return openEndpoint(providerId,upstreamModel,routeId,input,streaming,"chat");
|
||||
}
|
||||
|
||||
public Map<String,Object> invoke(String providerId,String model,String routeId,Map<String,Object> input,String protocol) {
|
||||
try(var response=openEndpoint(providerId,model,routeId,input,false,protocol)) {
|
||||
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
|
||||
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
||||
Map<String,Object> result=objectMapper.readValue(bytes,new com.fasterxml.jackson.core.type.TypeReference<>(){});
|
||||
if(result.containsKey("error")) throw classifyUpstreamError(200,writeJson(result));
|
||||
return result;
|
||||
} catch(ApiException e) {throw e;} catch(Exception e) {throw new ApiException("UPSTREAM_PROVIDER_ERROR","上游响应无法读取",502);}
|
||||
}
|
||||
public StreamResponse nativeStream(String providerId,String model,String routeId,Map<String,Object> input,String protocol) {
|
||||
return openEndpoint(providerId,model,routeId,input,true,protocol);
|
||||
}
|
||||
private StreamResponse openEndpoint(String providerId,String upstreamModel,String routeId,Map<String,Object> input,boolean streaming,String protocol) {
|
||||
ProviderConfig provider = resolve(providerId);
|
||||
if (provider == null || !provider.hasKey(routeId, upstreamModel, environment)) {
|
||||
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
|
||||
}
|
||||
Map<String, Object> body = new LinkedHashMap<>(input);
|
||||
body.put("model", upstreamModel);
|
||||
body.put("stream", streaming);
|
||||
if(streaming) body.put("stream_options",Map.of("include_usage",true));
|
||||
if(!List.of("embeddings","gemini").contains(protocol)) body.put("stream", streaming);
|
||||
if("chat".equals(protocol)&&streaming) body.put("stream_options",Map.of("include_usage",true));
|
||||
else body.remove("stream_options");
|
||||
String base=normalizeUrl(provider.baseUrl());
|
||||
String path=switch(protocol) {
|
||||
case "chat" -> "/chat/completions";
|
||||
case "messages" -> "/messages";
|
||||
case "responses" -> "/responses";
|
||||
case "embeddings" -> "/embeddings";
|
||||
case "gemini" -> "/models/"+java.net.URLEncoder.encode(upstreamModel,StandardCharsets.UTF_8).replace("+","%20")+(streaming?":streamGenerateContent?alt=sse":":generateContent");
|
||||
default -> throw new ApiException("UNSUPPORTED_PROTOCOL","协议不支持",400);
|
||||
};
|
||||
if("gemini".equals(protocol)) {base=base.replaceFirst("/v1$","");if(!base.endsWith("/v1beta")) base+="/v1beta";body.remove("model");body.remove("stream");}
|
||||
String apiKey = provider.nextApiKey(routeId, upstreamModel, environment, keyCursors);
|
||||
for (int attempt = 0; attempt < 2; attempt++) {
|
||||
List<String> candidates=provider.routeKeys(routeId,upstreamModel,environment);
|
||||
if(candidates.isEmpty()) candidates=provider.apiKeys();
|
||||
String healthRoute=routeId==null?providerId:routeId;
|
||||
var attempted=new java.util.HashSet<String>();
|
||||
if(health!=null) {
|
||||
apiKey=health.select(healthRoute,candidates,attempted,Math.max(0,candidates.indexOf(apiKey)));
|
||||
if(apiKey==null) throw new UpstreamRejectedException(new ApiException("UPSTREAM_ACCOUNTS_COOLING","渠道账号暂时隔离,请稍后重试",503));
|
||||
}
|
||||
boolean compatibilityRetried=false;
|
||||
long overallStarted=System.nanoTime();
|
||||
if(org.springframework.web.context.request.RequestContextHolder.getRequestAttributes() instanceof org.springframework.web.context.request.ServletRequestAttributes attrs) {
|
||||
Object previous=attrs.getRequest().getAttribute("moyi.upstreamStarted");
|
||||
if(previous instanceof Long value) overallStarted=value;else attrs.getRequest().setAttribute("moyi.upstreamStarted",overallStarted);
|
||||
}
|
||||
for (int attempt = 0; attempt < 4; attempt++) {
|
||||
long remaining=timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted);
|
||||
if(remaining<=0) throw new ApiException("UPSTREAM_PROVIDER_TIMEOUT","上游请求超时",504);
|
||||
String json = writeJson(body);
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(normalizeUrl(provider.baseUrl()) + "/chat/completions"))
|
||||
.timeout(Duration.ofSeconds(timeoutSeconds))
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder()
|
||||
.uri(URI.create(base + path))
|
||||
.timeout(Duration.ofMillis(remaining))
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(json))
|
||||
.build();
|
||||
.POST(HttpRequest.BodyPublishers.ofString(json));
|
||||
if("messages".equals(protocol)) builder.header("x-api-key",apiKey).header("anthropic-version","2023-06-01");
|
||||
if("gemini".equals(protocol)) builder.header("x-goog-api-key",apiKey);
|
||||
HttpRequest request=builder.build();
|
||||
try {
|
||||
long started=System.nanoTime();
|
||||
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
if(health!=null) health.record(healthRoute,apiKey,response.statusCode(),(int)TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started));
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String details = readErrorBody(response.body());
|
||||
String details;
|
||||
try(var bounded=new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)))) {details=readErrorBody(bounded.body);}
|
||||
if (requiresLargerOutputBudget(details)) {
|
||||
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
|
||||
}
|
||||
// Some OpenAI-compatible relays use the newer spelling for
|
||||
// reasoning-model budgets. Retry only this deterministic
|
||||
// GPT request-shape error; other failures remain bill-safe.
|
||||
if (attempt == 0 && prepareCompatibilityRetry(body, details)) continue;
|
||||
throw classifyUpstreamError(response.statusCode(), details);
|
||||
if ("chat".equals(protocol) && !compatibilityRetried && prepareCompatibilityRetry(body, details)) {compatibilityRetried=true;continue;}
|
||||
if(health!=null && List.of(401,403,429).contains(response.statusCode()) && attempt<3 && TimeUnit.NANOSECONDS.toSeconds(System.nanoTime()-overallStarted)<timeoutSeconds) {
|
||||
attempted.add(apiKey);String next=health.select(healthRoute,candidates,attempted,0);
|
||||
if(next!=null) {apiKey=next;continue;}
|
||||
}
|
||||
ApiException rejection=classifyUpstreamError(response.statusCode(),details);
|
||||
if(List.of(401,402,403,429).contains(response.statusCode())) throw new UpstreamRejectedException(rejection);
|
||||
throw rejection;
|
||||
}
|
||||
if(streaming && !response.headers().firstValue("Content-Type").orElse("").toLowerCase().startsWith("text/event-stream")) {
|
||||
response.body().close();
|
||||
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
|
||||
}
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)));
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)));
|
||||
} catch (ApiException exception) {
|
||||
throw exception;
|
||||
} catch (HttpTimeoutException | SocketTimeoutException exception) {
|
||||
if(health!=null) health.record(healthRoute,apiKey,504,timeoutSeconds*1000);
|
||||
throw new ApiException("UPSTREAM_PROVIDER_TIMEOUT", "上游模型供应商响应超时", 504);
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
@@ -178,11 +232,16 @@ public class OpenAiCompatibleProvider {
|
||||
private final InputStream body;
|
||||
private final AtomicBoolean expired=new AtomicBoolean();
|
||||
private final ScheduledFuture<?> deadline;
|
||||
private Runnable afterClose=()->{};
|
||||
public StreamResponse onClose(Runnable callback) {afterClose=callback;return this;}
|
||||
private StreamResponse(InputStream body,long remainingMs) {
|
||||
this.body=body;
|
||||
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
|
||||
}
|
||||
public void events(EventConsumer consumer) throws IOException {
|
||||
events(consumer,true);
|
||||
}
|
||||
public void events(EventConsumer consumer,boolean requireDone) throws IOException {
|
||||
var reader=new BufferedReader(new InputStreamReader(body,StandardCharsets.UTF_8));
|
||||
StringBuilder event=new StringBuilder(), line=new StringBuilder();
|
||||
long total=0;
|
||||
@@ -210,9 +269,10 @@ public class OpenAiCompatibleProvider {
|
||||
if(expired.get()) throw new ApiException("UPSTREAM_PROVIDER_TIMEOUT","上游响应超时,本次未扣费",504);
|
||||
throw exception;
|
||||
}
|
||||
if(!requireDone && !expired.get() && event.isEmpty() && line.isEmpty()) return;
|
||||
throw new ApiException(expired.get()?"UPSTREAM_PROVIDER_TIMEOUT":"UPSTREAM_STREAM_INCOMPLETE","上游流式响应中断,本次未扣费",502);
|
||||
}
|
||||
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} }
|
||||
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} finally {afterClose.run();afterClose=()->{};} }
|
||||
}
|
||||
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
|
||||
@PreDestroy public void close() { deadlines.shutdownNow(); }
|
||||
|
||||
Reference in new issue
Block a user