feat: route upstream keys by model group
This commit is contained in:
1 parent
ce0a718686
commit
0876dfe08e
9 files changed
+195
-41
No files matched your search
+53
-7
@@ -65,8 +65,18 @@ public class OpenAiCompatibleProvider {
|
||||
return provider != null && provider.hasKey();
|
||||
}
|
||||
|
||||
public boolean isConfigured(String providerId, String routeId, String upstreamModel) {
|
||||
ProviderConfig provider = resolve(providerId);
|
||||
return provider != null && provider.hasKey(routeId, upstreamModel, environment);
|
||||
}
|
||||
|
||||
public Map<String, Object> chat(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||
try(var response=open(providerId,upstreamModel,input,false)) {
|
||||
return chat(providerId, upstreamModel, null, input);
|
||||
}
|
||||
|
||||
/** Route-aware overload. A route-specific key pool is preferred when configured. */
|
||||
public Map<String, Object> chat(String providerId, String upstreamModel, String routeId, Map<String,Object> input) {
|
||||
try(var response=open(providerId,upstreamModel,routeId,input,false)) {
|
||||
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
|
||||
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
||||
Map<String,Object> body=objectMapper.readValue(bytes, new com.fasterxml.jackson.core.type.TypeReference<>() {});
|
||||
@@ -80,12 +90,17 @@ public class OpenAiCompatibleProvider {
|
||||
}
|
||||
|
||||
public StreamResponse stream(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||
return open(providerId,upstreamModel,input,true);
|
||||
return stream(providerId, upstreamModel, null, input);
|
||||
}
|
||||
|
||||
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
|
||||
/** Route-aware overload. A route-specific key pool is preferred when configured. */
|
||||
public StreamResponse stream(String providerId, String upstreamModel, String routeId, Map<String,Object> input) {
|
||||
return open(providerId,upstreamModel,routeId,input,true);
|
||||
}
|
||||
|
||||
private StreamResponse open(String providerId, String upstreamModel, String routeId, Map<String,Object> input, boolean streaming) {
|
||||
ProviderConfig provider = resolve(providerId);
|
||||
if (provider == null || !provider.hasKey()) {
|
||||
if (provider == null || !provider.hasKey(routeId, upstreamModel, environment)) {
|
||||
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
|
||||
}
|
||||
Map<String, Object> body = new LinkedHashMap<>(input);
|
||||
@@ -93,7 +108,7 @@ public class OpenAiCompatibleProvider {
|
||||
body.put("stream", streaming);
|
||||
if(streaming) body.put("stream_options",Map.of("include_usage",true));
|
||||
else body.remove("stream_options");
|
||||
String apiKey = provider.nextApiKey();
|
||||
String apiKey = provider.nextApiKey(routeId, upstreamModel, environment, keyCursors);
|
||||
for (int attempt = 0; attempt < 2; attempt++) {
|
||||
String json = writeJson(body);
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
@@ -280,7 +295,7 @@ public class OpenAiCompatibleProvider {
|
||||
if (fallbackKey != null && !fallbackKey.isBlank()) keys.add(fallbackKey.trim());
|
||||
addEnvironmentKeys(keys, providerId, environment);
|
||||
if (providerCode != null && !providerCode.isBlank() && !providerCode.equals(providerId)) addEnvironmentKeys(keys, providerCode, environment);
|
||||
return new ProviderConfig(keys.stream().distinct().toList(), baseUrl, cursor);
|
||||
return new ProviderConfig(providerId, keys.stream().distinct().toList(), baseUrl, cursor);
|
||||
}
|
||||
|
||||
private void addEnvironmentKeys(List<String> keys, String alias, org.springframework.core.env.Environment environment) {
|
||||
@@ -289,8 +304,39 @@ public class OpenAiCompatibleProvider {
|
||||
if (configured != null && !configured.isBlank()) for (String key : configured.split("\\|")) if (!key.isBlank()) keys.add(key.trim());
|
||||
}
|
||||
|
||||
private record ProviderConfig(List<String> apiKeys, String baseUrl, AtomicInteger cursor) {
|
||||
private record ProviderConfig(String alias, List<String> apiKeys, String baseUrl, AtomicInteger cursor) {
|
||||
private boolean hasKey() { return !apiKeys.isEmpty(); }
|
||||
private boolean hasKey(String routeId, String upstreamModel, org.springframework.core.env.Environment environment) {
|
||||
return !apiKeys.isEmpty() || !routeKeys(routeId, upstreamModel, environment).isEmpty();
|
||||
}
|
||||
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
|
||||
|
||||
private String nextApiKey(String routeId, String upstreamModel, org.springframework.core.env.Environment environment,
|
||||
ConcurrentMap<String, AtomicInteger> cursors) {
|
||||
List<String> routeKeys = routeKeys(routeId, upstreamModel, environment);
|
||||
if (routeKeys.isEmpty()) return nextApiKey();
|
||||
String cursorKey = "route:" + (routeId == null ? "" : routeId);
|
||||
AtomicInteger routeCursor = cursors.computeIfAbsent(cursorKey, ignored -> new AtomicInteger());
|
||||
return routeKeys.get(Math.floorMod(routeCursor.getAndIncrement(), routeKeys.size()));
|
||||
}
|
||||
|
||||
private List<String> routeKeys(String routeId, String upstreamModel, org.springframework.core.env.Environment environment) {
|
||||
if (environment == null) return List.of();
|
||||
for (String prefix : List.of("MOYI_UPSTREAM_API_KEYS_ROUTE_", "MOYI_UPSTREAM_API_KEYS_MODEL_")) {
|
||||
if (prefix.startsWith("MOYI_UPSTREAM_API_KEYS_ROUTE_") && (routeId == null || routeId.isBlank())) continue;
|
||||
String selector = prefix.endsWith("MODEL_") ? upstreamModel : routeId;
|
||||
if (selector == null || selector.isBlank()) continue;
|
||||
String value = environment.getProperty(prefix + normalize(selector));
|
||||
if (value == null || value.isBlank()) continue;
|
||||
List<String> keys = new ArrayList<>();
|
||||
for (String key : value.split("\\|")) if (!key.isBlank()) keys.add(key.trim());
|
||||
if (!keys.isEmpty()) return keys.stream().distinct().toList();
|
||||
}
|
||||
return List.of();
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
return String.valueOf(value == null ? "" : value).toUpperCase(java.util.Locale.ROOT).replaceAll("[^A-Z0-9]", "_");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user