feat: route upstream keys by model group
This commit is contained in:
1 parent
ce0a718686
commit
0876dfe08e
9 files changed
+194
-40
No files matched your search
+6
-5
@@ -83,8 +83,9 @@ public class GatewayController {
|
|||||||
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线;请使用当前 API Key 的 /v1/models 返回的模型 ID", 404);
|
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线;请使用当前 API Key 的 /v1/models 返回的模型 ID", 404);
|
||||||
if ("async".equals(model.get("mode"))) throw new ApiException("ASYNC_MODEL", "该模型需要使用异步任务接口", 400);
|
if ("async".equals(model.get("mode"))) throw new ApiException("ASYNC_MODEL", "该模型需要使用异步任务接口", 400);
|
||||||
String providerId=String.valueOf(model.get("providerId"));
|
String providerId=String.valueOf(model.get("providerId"));
|
||||||
|
String routeId=String.valueOf(model.get("id"));
|
||||||
boolean mock="provider-mock".equals(providerId);
|
boolean mock="provider-mock".equals(providerId);
|
||||||
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
if(!mock && !provider.isConfigured(providerId, routeId, String.valueOf(model.get("upstreamModel")))) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||||
ChatMeter.Request request=ChatMeter.request(input,json);
|
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||||
// Reserve both limits atomically. A queued request never consumes a
|
// Reserve both limits atomically. A queued request never consumes a
|
||||||
@@ -95,10 +96,10 @@ public class GatewayController {
|
|||||||
try {
|
try {
|
||||||
catalogService.touchKey(key);
|
catalogService.touchKey(key);
|
||||||
if(Boolean.TRUE.equals(request.body().get("stream"))) {
|
if(Boolean.TRUE.equals(request.body().get("stream"))) {
|
||||||
stream(model,modelId,request.body(),ticket,servletResponse,startedAt,mock);
|
stream(model,modelId,routeId,request.body(),ticket,servletResponse,startedAt,mock);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),request.body());
|
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),routeId,request.body());
|
||||||
body.put("model",modelId);
|
body.put("model",modelId);
|
||||||
requireCompletion(body);
|
requireCompletion(body);
|
||||||
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
|
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
|
||||||
@@ -117,7 +118,7 @@ public class GatewayController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void stream(Map<String,Object> model, String modelId, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
private void stream(Map<String,Object> model, String modelId, String routeId, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
||||||
if(mock) {
|
if(mock) {
|
||||||
var body=mockResponse(modelId,input);
|
var body=mockResponse(modelId,input);
|
||||||
prepareStream(response);
|
prepareStream(response);
|
||||||
@@ -130,7 +131,7 @@ public class GatewayController {
|
|||||||
}
|
}
|
||||||
// Keep admission, the concurrency slot and the upstream stream alive on this request thread.
|
// Keep admission, the concurrency slot and the upstream stream alive on this request thread.
|
||||||
// Upstream HTTP errors are inspected before committing a 200/SSE response.
|
// Upstream HTTP errors are inspected before committing a 200/SSE response.
|
||||||
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),input)) {
|
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),routeId,input)) {
|
||||||
var collector=new ChatStreamCollector(modelId,started);
|
var collector=new ChatStreamCollector(modelId,started);
|
||||||
prepareStream(response);
|
prepareStream(response);
|
||||||
boolean[] disconnected={false};
|
boolean[] disconnected={false};
|
||||||
|
|||||||
+53
-7
@@ -65,8 +65,18 @@ public class OpenAiCompatibleProvider {
|
|||||||
return provider != null && provider.hasKey();
|
return provider != null && provider.hasKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean isConfigured(String providerId, String routeId, String upstreamModel) {
|
||||||
|
ProviderConfig provider = resolve(providerId);
|
||||||
|
return provider != null && provider.hasKey(routeId, upstreamModel, environment);
|
||||||
|
}
|
||||||
|
|
||||||
public Map<String, Object> chat(String providerId, String upstreamModel, Map<String,Object> input) {
|
public Map<String, Object> chat(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||||
try(var response=open(providerId,upstreamModel,input,false)) {
|
return chat(providerId, upstreamModel, null, input);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Route-aware overload. A route-specific key pool is preferred when configured. */
|
||||||
|
public Map<String, Object> chat(String providerId, String upstreamModel, String routeId, Map<String,Object> input) {
|
||||||
|
try(var response=open(providerId,upstreamModel,routeId,input,false)) {
|
||||||
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
|
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
|
||||||
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
||||||
Map<String,Object> body=objectMapper.readValue(bytes, new com.fasterxml.jackson.core.type.TypeReference<>() {});
|
Map<String,Object> body=objectMapper.readValue(bytes, new com.fasterxml.jackson.core.type.TypeReference<>() {});
|
||||||
@@ -80,12 +90,17 @@ public class OpenAiCompatibleProvider {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public StreamResponse stream(String providerId, String upstreamModel, Map<String,Object> input) {
|
public StreamResponse stream(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||||
return open(providerId,upstreamModel,input,true);
|
return stream(providerId, upstreamModel, null, input);
|
||||||
}
|
}
|
||||||
|
|
||||||
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
|
/** Route-aware overload. A route-specific key pool is preferred when configured. */
|
||||||
|
public StreamResponse stream(String providerId, String upstreamModel, String routeId, Map<String,Object> input) {
|
||||||
|
return open(providerId,upstreamModel,routeId,input,true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private StreamResponse open(String providerId, String upstreamModel, String routeId, Map<String,Object> input, boolean streaming) {
|
||||||
ProviderConfig provider = resolve(providerId);
|
ProviderConfig provider = resolve(providerId);
|
||||||
if (provider == null || !provider.hasKey()) {
|
if (provider == null || !provider.hasKey(routeId, upstreamModel, environment)) {
|
||||||
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
|
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
|
||||||
}
|
}
|
||||||
Map<String, Object> body = new LinkedHashMap<>(input);
|
Map<String, Object> body = new LinkedHashMap<>(input);
|
||||||
@@ -93,7 +108,7 @@ public class OpenAiCompatibleProvider {
|
|||||||
body.put("stream", streaming);
|
body.put("stream", streaming);
|
||||||
if(streaming) body.put("stream_options",Map.of("include_usage",true));
|
if(streaming) body.put("stream_options",Map.of("include_usage",true));
|
||||||
else body.remove("stream_options");
|
else body.remove("stream_options");
|
||||||
String apiKey = provider.nextApiKey();
|
String apiKey = provider.nextApiKey(routeId, upstreamModel, environment, keyCursors);
|
||||||
for (int attempt = 0; attempt < 2; attempt++) {
|
for (int attempt = 0; attempt < 2; attempt++) {
|
||||||
String json = writeJson(body);
|
String json = writeJson(body);
|
||||||
HttpRequest request = HttpRequest.newBuilder()
|
HttpRequest request = HttpRequest.newBuilder()
|
||||||
@@ -280,7 +295,7 @@ public class OpenAiCompatibleProvider {
|
|||||||
if (fallbackKey != null && !fallbackKey.isBlank()) keys.add(fallbackKey.trim());
|
if (fallbackKey != null && !fallbackKey.isBlank()) keys.add(fallbackKey.trim());
|
||||||
addEnvironmentKeys(keys, providerId, environment);
|
addEnvironmentKeys(keys, providerId, environment);
|
||||||
if (providerCode != null && !providerCode.isBlank() && !providerCode.equals(providerId)) addEnvironmentKeys(keys, providerCode, environment);
|
if (providerCode != null && !providerCode.isBlank() && !providerCode.equals(providerId)) addEnvironmentKeys(keys, providerCode, environment);
|
||||||
return new ProviderConfig(keys.stream().distinct().toList(), baseUrl, cursor);
|
return new ProviderConfig(providerId, keys.stream().distinct().toList(), baseUrl, cursor);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addEnvironmentKeys(List<String> keys, String alias, org.springframework.core.env.Environment environment) {
|
private void addEnvironmentKeys(List<String> keys, String alias, org.springframework.core.env.Environment environment) {
|
||||||
@@ -289,8 +304,39 @@ public class OpenAiCompatibleProvider {
|
|||||||
if (configured != null && !configured.isBlank()) for (String key : configured.split("\\|")) if (!key.isBlank()) keys.add(key.trim());
|
if (configured != null && !configured.isBlank()) for (String key : configured.split("\\|")) if (!key.isBlank()) keys.add(key.trim());
|
||||||
}
|
}
|
||||||
|
|
||||||
private record ProviderConfig(List<String> apiKeys, String baseUrl, AtomicInteger cursor) {
|
private record ProviderConfig(String alias, List<String> apiKeys, String baseUrl, AtomicInteger cursor) {
|
||||||
private boolean hasKey() { return !apiKeys.isEmpty(); }
|
private boolean hasKey() { return !apiKeys.isEmpty(); }
|
||||||
|
private boolean hasKey(String routeId, String upstreamModel, org.springframework.core.env.Environment environment) {
|
||||||
|
return !apiKeys.isEmpty() || !routeKeys(routeId, upstreamModel, environment).isEmpty();
|
||||||
|
}
|
||||||
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
|
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
|
||||||
|
|
||||||
|
private String nextApiKey(String routeId, String upstreamModel, org.springframework.core.env.Environment environment,
|
||||||
|
ConcurrentMap<String, AtomicInteger> cursors) {
|
||||||
|
List<String> routeKeys = routeKeys(routeId, upstreamModel, environment);
|
||||||
|
if (routeKeys.isEmpty()) return nextApiKey();
|
||||||
|
String cursorKey = "route:" + (routeId == null ? "" : routeId);
|
||||||
|
AtomicInteger routeCursor = cursors.computeIfAbsent(cursorKey, ignored -> new AtomicInteger());
|
||||||
|
return routeKeys.get(Math.floorMod(routeCursor.getAndIncrement(), routeKeys.size()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> routeKeys(String routeId, String upstreamModel, org.springframework.core.env.Environment environment) {
|
||||||
|
if (environment == null) return List.of();
|
||||||
|
for (String prefix : List.of("MOYI_UPSTREAM_API_KEYS_ROUTE_", "MOYI_UPSTREAM_API_KEYS_MODEL_")) {
|
||||||
|
if (prefix.startsWith("MOYI_UPSTREAM_API_KEYS_ROUTE_") && (routeId == null || routeId.isBlank())) continue;
|
||||||
|
String selector = prefix.endsWith("MODEL_") ? upstreamModel : routeId;
|
||||||
|
if (selector == null || selector.isBlank()) continue;
|
||||||
|
String value = environment.getProperty(prefix + normalize(selector));
|
||||||
|
if (value == null || value.isBlank()) continue;
|
||||||
|
List<String> keys = new ArrayList<>();
|
||||||
|
for (String key : value.split("\\|")) if (!key.isBlank()) keys.add(key.trim());
|
||||||
|
if (!keys.isEmpty()) return keys.stream().distinct().toList();
|
||||||
|
}
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String normalize(String value) {
|
||||||
|
return String.valueOf(value == null ? "" : value).toUpperCase(java.util.Locale.ROOT).replaceAll("[^A-Z0-9]", "_");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+69
-15
@@ -47,10 +47,18 @@ public class AsyncProviderClient {
|
|||||||
return submit(providerId, upstreamModel, taskType, Map.of("prompt", prompt));
|
return submit(providerId, upstreamModel, taskType, Map.of("prompt", prompt));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<String,Object> submit(String providerId, String upstreamModel, String routeId, String taskType, String prompt) {
|
||||||
|
return submit(providerId, upstreamModel, routeId, taskType, Map.of("prompt", prompt));
|
||||||
|
}
|
||||||
|
|
||||||
public Map<String,Object> submit(String providerId, String upstreamModel, String taskType, Map<String,Object> input) {
|
public Map<String,Object> submit(String providerId, String upstreamModel, String taskType, Map<String,Object> input) {
|
||||||
|
return submit(providerId, upstreamModel, null, taskType, input);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String,Object> submit(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
||||||
if (input == null) input = Map.of();
|
if (input == null) input = Map.of();
|
||||||
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
||||||
Provider provider = provider(providerId);
|
Provider provider = provider(providerId, routeId, upstreamModel);
|
||||||
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
|
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
|
||||||
if (isTripo(provider.code(), provider.baseUrl())) {
|
if (isTripo(provider.code(), provider.baseUrl())) {
|
||||||
Map<String,Object> body = new LinkedHashMap<>();
|
Map<String,Object> body = new LinkedHashMap<>();
|
||||||
@@ -60,7 +68,7 @@ public class AsyncProviderClient {
|
|||||||
else if (!prompt.isBlank()) body.put("prompt", prompt);
|
else if (!prompt.isBlank()) body.put("prompt", prompt);
|
||||||
body.put("texture", true);
|
body.put("texture", true);
|
||||||
body.put("pbr", true);
|
body.put("pbr", true);
|
||||||
return request("POST", tripoBase(provider.baseUrl()) + (media == null ? "/generation/text-to-model" : "/generation/image-to-model"), provider.nextApiKey(), body, "上游 3D 任务创建失败");
|
return request("POST", tripoBase(provider.baseUrl()) + (media == null ? "/generation/text-to-model" : "/generation/image-to-model"), provider.nextApiKey(routeId, upstreamModel, environment, keyCursors), body, "上游 3D 任务创建失败");
|
||||||
}
|
}
|
||||||
if (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com")) {
|
if (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com")) {
|
||||||
List<Map<String,Object>> content = new ArrayList<>();
|
List<Map<String,Object>> content = new ArrayList<>();
|
||||||
@@ -71,25 +79,30 @@ public class AsyncProviderClient {
|
|||||||
content.add(Map.of("type", "image_url", "image_url", Map.of("url", media)));
|
content.add(Map.of("type", "image_url", "image_url", Map.of("url", media)));
|
||||||
}
|
}
|
||||||
Map<String,Object> body = Map.of("model", upstreamModel, "content", content);
|
Map<String,Object> body = Map.of("model", upstreamModel, "content", content);
|
||||||
return request("POST", trim(provider.baseUrl()) + "/contents/generations/tasks", provider.nextApiKey(), body, "上游视频任务创建失败");
|
return request("POST", trim(provider.baseUrl()) + "/contents/generations/tasks", provider.nextApiKey(routeId, upstreamModel, environment, keyCursors), body, "上游视频任务创建失败");
|
||||||
}
|
}
|
||||||
if ("openai-compatible".equalsIgnoreCase(provider.protocol())) {
|
if ("openai-compatible".equalsIgnoreCase(provider.protocol())) {
|
||||||
return submitOpenAiCompatible(provider, upstreamModel, taskType, input);
|
return submitOpenAiCompatible(provider, routeId, upstreamModel, taskType, input);
|
||||||
}
|
}
|
||||||
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
|
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String,Object> poll(String providerId, String providerTaskId) {
|
public Map<String,Object> poll(String providerId, String providerTaskId) {
|
||||||
Provider provider = provider(providerId);
|
return poll(providerId, null, providerTaskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String,Object> poll(String providerId, String routeId, String providerTaskId) {
|
||||||
|
Provider provider = provider(providerId, routeId, null);
|
||||||
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
|
String code = provider.code().toLowerCase(java.util.Locale.ROOT);
|
||||||
String url;
|
String url;
|
||||||
|
String apiKey = provider.nextApiKey(routeId, null, environment, keyCursors);
|
||||||
if (isTripo(provider.code(), provider.baseUrl())) url = tripoBase(provider.baseUrl()) + "/tasks/" + encode(providerTaskId);
|
if (isTripo(provider.code(), provider.baseUrl())) url = tripoBase(provider.baseUrl()) + "/tasks/" + encode(providerTaskId);
|
||||||
else if (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com")) url = trim(provider.baseUrl()) + "/contents/generations/tasks/" + encode(providerTaskId);
|
else if (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com")) url = trim(provider.baseUrl()) + "/contents/generations/tasks/" + encode(providerTaskId);
|
||||||
else if ("openai-compatible".equalsIgnoreCase(provider.protocol())) {
|
else if ("openai-compatible".equalsIgnoreCase(provider.protocol())) {
|
||||||
return pollOpenAiCompatible(provider, providerTaskId);
|
return pollOpenAiCompatible(provider, routeId, providerTaskId);
|
||||||
}
|
}
|
||||||
else throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
|
else throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501);
|
||||||
return request("GET", url, provider.nextApiKey(), null, "上游异步任务查询失败");
|
return request("GET", url, apiKey, null, "上游异步任务查询失败");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -98,7 +111,7 @@ public class AsyncProviderClient {
|
|||||||
* generation completes in the request, so mark the normalized result for
|
* generation completes in the request, so mark the normalized result for
|
||||||
* AsyncTaskService to settle immediately instead of inventing a task id.
|
* AsyncTaskService to settle immediately instead of inventing a task id.
|
||||||
*/
|
*/
|
||||||
private Map<String,Object> submitOpenAiCompatible(Provider provider, String model, String taskType, Map<String,Object> input) {
|
private Map<String,Object> submitOpenAiCompatible(Provider provider, String routeId, String model, String taskType, Map<String,Object> input) {
|
||||||
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
||||||
String media = mediaUrl(input);
|
String media = mediaUrl(input);
|
||||||
String base = trim(provider.baseUrl());
|
String base = trim(provider.baseUrl());
|
||||||
@@ -112,8 +125,8 @@ public class AsyncProviderClient {
|
|||||||
copyParameter(input, body, "response_format");
|
copyParameter(input, body, "response_format");
|
||||||
copyParameter(input, body, "size");
|
copyParameter(input, body, "size");
|
||||||
Map<String,Object> response = media == null
|
Map<String,Object> response = media == null
|
||||||
? request("POST", base + "/images/generations", provider.nextApiKey(), body, "上游图片任务创建失败")
|
? request("POST", base + "/images/generations", provider.nextApiKey(routeId, model, environment, keyCursors), body, "上游图片任务创建失败")
|
||||||
: requestMultipart("POST", base + "/images/edits", provider.nextApiKey(), body, "image", media, "上游图片任务创建失败");
|
: requestMultipart("POST", base + "/images/edits", provider.nextApiKey(routeId, model, environment, keyCursors), body, "image", media, "上游图片任务创建失败");
|
||||||
Map<String,Object> result = compatibleResult(response);
|
Map<String,Object> result = compatibleResult(response);
|
||||||
if (result != null) return Map.of("_moyiInlineResult", result);
|
if (result != null) return Map.of("_moyiInlineResult", result);
|
||||||
return response;
|
return response;
|
||||||
@@ -125,7 +138,7 @@ public class AsyncProviderClient {
|
|||||||
// The standard OpenAI video endpoint is multipart even when no
|
// The standard OpenAI video endpoint is multipart even when no
|
||||||
// reference file is supplied. JSON requests are rejected by the
|
// reference file is supplied. JSON requests are rejected by the
|
||||||
// official API and by relays that proxy it faithfully.
|
// official API and by relays that proxy it faithfully.
|
||||||
String apiKey = provider.nextApiKey();
|
String apiKey = provider.nextApiKey(routeId, model, environment, keyCursors);
|
||||||
try {
|
try {
|
||||||
return requestMultipart("POST", base + "/videos", apiKey, body,
|
return requestMultipart("POST", base + "/videos", apiKey, body,
|
||||||
media == null ? null : "input_reference", media, "上游视频任务创建失败");
|
media == null ? null : "input_reference", media, "上游视频任务创建失败");
|
||||||
@@ -147,7 +160,7 @@ public class AsyncProviderClient {
|
|||||||
body.put("voice", input.getOrDefault("voice", "alloy"));
|
body.put("voice", input.getOrDefault("voice", "alloy"));
|
||||||
body.put("response_format", input.getOrDefault("response_format", "mp3"));
|
body.put("response_format", input.getOrDefault("response_format", "mp3"));
|
||||||
copyParameter(input, body, "speed");
|
copyParameter(input, body, "speed");
|
||||||
return Map.of("_moyiInlineResult", requestAudio("POST", base + "/audio/speech", provider.nextApiKey(), body, "上游语音任务创建失败"));
|
return Map.of("_moyiInlineResult", requestAudio("POST", base + "/audio/speech", provider.nextApiKey(routeId, model, environment, keyCursors), body, "上游语音任务创建失败"));
|
||||||
}
|
}
|
||||||
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道的媒体接口尚未配置", 501);
|
throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道的媒体接口尚未配置", 501);
|
||||||
}
|
}
|
||||||
@@ -171,9 +184,9 @@ public class AsyncProviderClient {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String,Object> pollOpenAiCompatible(Provider provider, String providerTaskId) {
|
private Map<String,Object> pollOpenAiCompatible(Provider provider, String routeId, String providerTaskId) {
|
||||||
String base = trim(provider.baseUrl());
|
String base = trim(provider.baseUrl());
|
||||||
String apiKey = provider.nextApiKey();
|
String apiKey = provider.nextApiKey(routeId, null, environment, keyCursors);
|
||||||
try {
|
try {
|
||||||
return request("GET", base + "/videos/" + encode(providerTaskId), apiKey, null, "上游异步任务查询失败");
|
return request("GET", base + "/videos/" + encode(providerTaskId), apiKey, null, "上游异步任务查询失败");
|
||||||
} catch (ApiException first) {
|
} catch (ApiException first) {
|
||||||
@@ -363,6 +376,10 @@ public class AsyncProviderClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private Provider provider(String id) {
|
private Provider provider(String id) {
|
||||||
|
return provider(id, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Provider provider(String id, String routeId, String upstreamModel) {
|
||||||
try {
|
try {
|
||||||
Map<String,Object> row = jdbc.queryForMap("SELECT code, protocol, base_url, credential_ciphertext FROM model_providers WHERE id=? AND status='active' AND deleted_at IS NULL", id);
|
Map<String,Object> row = jdbc.queryForMap("SELECT code, protocol, base_url, credential_ciphertext FROM model_providers WHERE id=? AND status='active' AND deleted_at IS NULL", id);
|
||||||
String ciphertext = row.get("credential_ciphertext") == null ? "" : String.valueOf(row.get("credential_ciphertext"));
|
String ciphertext = row.get("credential_ciphertext") == null ? "" : String.valueOf(row.get("credential_ciphertext"));
|
||||||
@@ -378,7 +395,7 @@ public class AsyncProviderClient {
|
|||||||
addEnvironmentKeys(keys, "TRIPO_API_KEY");
|
addEnvironmentKeys(keys, "TRIPO_API_KEY");
|
||||||
addEnvironmentKeys(keys, "MOYI_TRIPO_API_KEY");
|
addEnvironmentKeys(keys, "MOYI_TRIPO_API_KEY");
|
||||||
}
|
}
|
||||||
if (keys.isEmpty()) throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道尚未配置 API Key", 503);
|
if (keys.isEmpty() && routeKeys(code, id, routeId, upstreamModel).isEmpty()) throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道尚未配置 API Key", 503);
|
||||||
return new Provider(code, String.valueOf(row.get("protocol")), String.valueOf(row.get("base_url")), keys.stream().distinct().toList(), keyCursors.computeIfAbsent(id, ignored -> new AtomicInteger()));
|
return new Provider(code, String.valueOf(row.get("protocol")), String.valueOf(row.get("base_url")), keys.stream().distinct().toList(), keyCursors.computeIfAbsent(id, ignored -> new AtomicInteger()));
|
||||||
} catch (EmptyResultDataAccessException exception) { throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道不存在或已停用", 503); }
|
} catch (EmptyResultDataAccessException exception) { throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道不存在或已停用", 503); }
|
||||||
}
|
}
|
||||||
@@ -400,6 +417,22 @@ public class AsyncProviderClient {
|
|||||||
String configured = environment.getProperty(name);
|
String configured = environment.getProperty(name);
|
||||||
if (configured != null && !configured.isBlank()) for (String value : configured.split("\\|")) if (!value.isBlank()) keys.add(value.trim());
|
if (configured != null && !configured.isBlank()) for (String value : configured.split("\\|")) if (!value.isBlank()) keys.add(value.trim());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<String> routeKeys(String providerCode, String providerId, String routeId, String upstreamModel) {
|
||||||
|
for (String prefix : List.of("MOYI_UPSTREAM_API_KEYS_ROUTE_", "MOYI_UPSTREAM_API_KEYS_MODEL_")) {
|
||||||
|
if (prefix.startsWith("MOYI_UPSTREAM_API_KEYS_ROUTE_") && (routeId == null || routeId.isBlank())) continue;
|
||||||
|
String alias = prefix.endsWith("MODEL_") ? upstreamModel : routeId;
|
||||||
|
if (alias == null || alias.isBlank()) continue;
|
||||||
|
String configured = environment.getProperty(prefix + normalize(alias));
|
||||||
|
if (configured == null || configured.isBlank()) continue;
|
||||||
|
List<String> keys = new ArrayList<>();
|
||||||
|
for (String key : configured.split("\\|")) if (!key.isBlank()) keys.add(key.trim());
|
||||||
|
if (!keys.isEmpty()) return keys.stream().distinct().toList();
|
||||||
|
}
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String value) { return String.valueOf(value == null ? "" : value).toUpperCase(java.util.Locale.ROOT).replaceAll("[^A-Z0-9]", "_"); }
|
||||||
private boolean isTripo(String code, String baseUrl) {
|
private boolean isTripo(String code, String baseUrl) {
|
||||||
String normalizedCode = code == null ? "" : code.toLowerCase(java.util.Locale.ROOT);
|
String normalizedCode = code == null ? "" : code.toLowerCase(java.util.Locale.ROOT);
|
||||||
String normalizedUrl = baseUrl == null ? "" : baseUrl.toLowerCase(java.util.Locale.ROOT);
|
String normalizedUrl = baseUrl == null ? "" : baseUrl.toLowerCase(java.util.Locale.ROOT);
|
||||||
@@ -407,5 +440,26 @@ public class AsyncProviderClient {
|
|||||||
}
|
}
|
||||||
private record Provider(String code, String protocol, String baseUrl, List<String> apiKeys, AtomicInteger cursor) {
|
private record Provider(String code, String protocol, String baseUrl, List<String> apiKeys, AtomicInteger cursor) {
|
||||||
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
|
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
|
||||||
|
private String nextApiKey(String routeId, String upstreamModel, Environment environment, ConcurrentMap<String, AtomicInteger> cursors) {
|
||||||
|
List<String> routeKeys = routeKeys(routeId, upstreamModel, environment);
|
||||||
|
if (routeKeys.isEmpty()) return nextApiKey();
|
||||||
|
AtomicInteger routeCursor = cursors.computeIfAbsent("route:" + (routeId == null ? upstreamModel : routeId), ignored -> new AtomicInteger());
|
||||||
|
return routeKeys.get(Math.floorMod(routeCursor.getAndIncrement(), routeKeys.size()));
|
||||||
|
}
|
||||||
|
private List<String> routeKeys(String routeId, String upstreamModel, Environment environment) {
|
||||||
|
if (environment == null) return List.of();
|
||||||
|
for (String prefix : List.of("MOYI_UPSTREAM_API_KEYS_ROUTE_", "MOYI_UPSTREAM_API_KEYS_MODEL_")) {
|
||||||
|
if (prefix.startsWith("MOYI_UPSTREAM_API_KEYS_ROUTE_") && (routeId == null || routeId.isBlank())) continue;
|
||||||
|
String selector = prefix.endsWith("MODEL_") ? upstreamModel : routeId;
|
||||||
|
if (selector == null || selector.isBlank()) continue;
|
||||||
|
String value = environment.getProperty(prefix + normalize(selector));
|
||||||
|
if (value == null || value.isBlank()) continue;
|
||||||
|
List<String> keys = new ArrayList<>();
|
||||||
|
for (String key : value.split("\\|")) if (!key.isBlank()) keys.add(key.trim());
|
||||||
|
if (!keys.isEmpty()) return keys.stream().distinct().toList();
|
||||||
|
}
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
private static String normalize(String value) { return String.valueOf(value == null ? "" : value).toUpperCase(java.util.Locale.ROOT).replaceAll("[^A-Z0-9]", "_"); }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -180,7 +180,7 @@ public class AsyncTaskService {
|
|||||||
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
|
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
|
||||||
);
|
);
|
||||||
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
||||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.provider_task_id, t.provider_deadline_at, t.created_at, m.provider_id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_deadline_at, t.created_at, m.provider_id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||||
for (Map<String,Object> row : pending) pollProviderTask(row);
|
for (Map<String,Object> row : pending) pollProviderTask(row);
|
||||||
dispatchQueuedTasks();
|
dispatchQueuedTasks();
|
||||||
}
|
}
|
||||||
@@ -234,7 +234,7 @@ public class AsyncTaskService {
|
|||||||
// workers reliably.
|
// workers reliably.
|
||||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
|
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
|
||||||
String sql = """
|
String sql = """
|
||||||
SELECT t.id, t.api_key_id, m.provider_id, m.upstream_model, t.task_type, t.prompt, t.input
|
SELECT t.id, t.api_key_id, t.model_id, m.provider_id, m.upstream_model, t.task_type, t.prompt, t.input
|
||||||
FROM model_tasks t
|
FROM model_tasks t
|
||||||
JOIN model_catalog m ON m.id=t.model_id
|
JOIN model_catalog m ON m.id=t.model_id
|
||||||
JOIN model_providers p ON p.id=m.provider_id
|
JOIN model_providers p ON p.id=m.provider_id
|
||||||
@@ -344,15 +344,16 @@ public class AsyncTaskService {
|
|||||||
|
|
||||||
private Map<String,Object> submitProviderTask(Map<String,Object> task, Map<String,Object> input) {
|
private Map<String,Object> submitProviderTask(Map<String,Object> task, Map<String,Object> input) {
|
||||||
String providerId = String.valueOf(task.get("provider_id"));
|
String providerId = String.valueOf(task.get("provider_id"));
|
||||||
|
String routeId = String.valueOf(task.get("model_id"));
|
||||||
String model = String.valueOf(task.get("upstream_model"));
|
String model = String.valueOf(task.get("upstream_model"));
|
||||||
String type = String.valueOf(task.get("task_type"));
|
String type = String.valueOf(task.get("task_type"));
|
||||||
// Keep the legacy overload for rows created before V30. Apart from
|
// Keep the legacy overload for rows created before V30. Apart from
|
||||||
// compatibility with existing integrations, this makes a queued
|
// compatibility with existing integrations, this makes a queued
|
||||||
// prompt-only task indistinguishable from the original submit path.
|
// prompt-only task indistinguishable from the original submit path.
|
||||||
if (input.size() == 1 && input.containsKey("prompt")) {
|
if (input.size() == 1 && input.containsKey("prompt")) {
|
||||||
return providerClient.submit(providerId, model, type, String.valueOf(input.get("prompt")));
|
return providerClient.submit(providerId, model, routeId, type, String.valueOf(input.get("prompt")));
|
||||||
}
|
}
|
||||||
return providerClient.submit(providerId, model, type, input);
|
return providerClient.submit(providerId, model, routeId, type, input);
|
||||||
}
|
}
|
||||||
|
|
||||||
private OffsetDateTime now() { return OffsetDateTime.now(ZoneOffset.UTC); }
|
private OffsetDateTime now() { return OffsetDateTime.now(ZoneOffset.UTC); }
|
||||||
@@ -380,7 +381,7 @@ public class AsyncTaskService {
|
|||||||
}
|
}
|
||||||
Map<String,Object> response;
|
Map<String,Object> response;
|
||||||
try {
|
try {
|
||||||
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("provider_task_id")));
|
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("model_id")), String.valueOf(row.get("provider_task_id")));
|
||||||
} catch (RuntimeException exception) {
|
} catch (RuntimeException exception) {
|
||||||
// Even a permanent authentication/configuration error only proves
|
// Even a permanent authentication/configuration error only proves
|
||||||
// that polling failed, not that an already accepted job stopped.
|
// that polling failed, not that an already accepted job stopped.
|
||||||
|
|||||||
+5
-5
@@ -227,7 +227,7 @@ class GatewayDeploymentTest {
|
|||||||
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "model-real"),null,new MockHttpServletResponse()));
|
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "model-real"),null,new MockHttpServletResponse()));
|
||||||
assertEquals("PROVIDER_NOT_CONFIGURED", error.getCode());
|
assertEquals("PROVIDER_NOT_CONFIGURED", error.getCode());
|
||||||
assertEquals(503, error.getStatus());
|
assertEquals(503, error.getStatus());
|
||||||
verify(provider, never()).chat(anyString(), anyString(), anyMap());
|
verify(provider, never()).chat(anyString(), anyString(), anyString(), anyMap());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -256,13 +256,13 @@ class GatewayDeploymentTest {
|
|||||||
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
||||||
when(catalog.activeModel("reasoning-route")).thenReturn(model);
|
when(catalog.activeModel("reasoning-route")).thenReturn(model);
|
||||||
when(catalog.keyCanInvoke(key, "reasoning-route")).thenReturn(true);
|
when(catalog.keyCanInvoke(key, "reasoning-route")).thenReturn(true);
|
||||||
when(provider.isConfigured("provider-relay")).thenReturn(true);
|
when(provider.isConfigured("provider-relay", "reasoning-route", "reasoning-model")).thenReturn(true);
|
||||||
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
|
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
|
||||||
"id", "upstream-response",
|
"id", "upstream-response",
|
||||||
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", "", "reasoning_content", "private reasoning"), "finish_reason", "length")),
|
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", "", "reasoning_content", "private reasoning"), "finish_reason", "length")),
|
||||||
"usage", Map.of("prompt_tokens", 2, "completion_tokens", 16, "total_tokens", 18)
|
"usage", Map.of("prompt_tokens", 2, "completion_tokens", 16, "total_tokens", 18)
|
||||||
));
|
));
|
||||||
when(provider.chat(eq("provider-relay"), eq("reasoning-model"), anyMap())).thenReturn(upstreamResponse);
|
when(provider.chat(eq("provider-relay"), eq("reasoning-model"), eq("reasoning-route"), anyMap())).thenReturn(upstreamResponse);
|
||||||
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
|
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
|
||||||
when(billing.settle(any(), anyMap(), anyInt(), isNull())).thenReturn(java.math.BigDecimal.ZERO);
|
when(billing.settle(any(), anyMap(), anyInt(), isNull())).thenReturn(java.math.BigDecimal.ZERO);
|
||||||
|
|
||||||
@@ -288,12 +288,12 @@ class GatewayDeploymentTest {
|
|||||||
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
||||||
when(catalog.activeModel("empty-route")).thenReturn(model);
|
when(catalog.activeModel("empty-route")).thenReturn(model);
|
||||||
when(catalog.keyCanInvoke(key, "empty-route")).thenReturn(true);
|
when(catalog.keyCanInvoke(key, "empty-route")).thenReturn(true);
|
||||||
when(provider.isConfigured("provider-relay")).thenReturn(true);
|
when(provider.isConfigured("provider-relay", "empty-route", "empty-model")).thenReturn(true);
|
||||||
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
|
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
|
||||||
"id", "upstream-empty",
|
"id", "upstream-empty",
|
||||||
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", ""), "finish_reason", "stop"))
|
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", ""), "finish_reason", "stop"))
|
||||||
));
|
));
|
||||||
when(provider.chat(eq("provider-relay"), eq("empty-model"), anyMap())).thenReturn(upstreamResponse);
|
when(provider.chat(eq("provider-relay"), eq("empty-model"), eq("empty-route"), anyMap())).thenReturn(upstreamResponse);
|
||||||
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
|
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
|
||||||
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
|
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
|
||||||
|
|
||||||
|
|||||||
+19
-1
@@ -28,11 +28,12 @@ class OpenAiCompatibleProviderHttpTest {
|
|||||||
private final ObjectMapper json = new ObjectMapper();
|
private final ObjectMapper json = new ObjectMapper();
|
||||||
private HttpServer server;
|
private HttpServer server;
|
||||||
private OpenAiCompatibleProvider provider;
|
private OpenAiCompatibleProvider provider;
|
||||||
|
private MockEnvironment environment;
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
void setUp() throws Exception {
|
void setUp() throws Exception {
|
||||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
var environment = new MockEnvironment()
|
environment = new MockEnvironment()
|
||||||
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||||
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||||
.withProperty("moyi.gateway.upstream-timeout-seconds", "5");
|
.withProperty("moyi.gateway.upstream-timeout-seconds", "5");
|
||||||
@@ -93,6 +94,23 @@ class OpenAiCompatibleProviderHttpTest {
|
|||||||
assertEquals(512, input.get("max_tokens"));
|
assertEquals(512, input.get("max_tokens"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void prefersRouteKeyThenModelKeyBeforeProviderKey() throws Exception {
|
||||||
|
var authorization = new ArrayList<String>();
|
||||||
|
server.createContext("/v1/chat/completions", exchange -> {
|
||||||
|
authorization.add(exchange.getRequestHeaders().getFirst("Authorization"));
|
||||||
|
write(exchange, 200, "{\"id\":\"ok\",\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"OK\"}}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}");
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
environment.withProperty("MOYI_UPSTREAM_API_KEYS_ROUTE_ROUTE_A", "route-key")
|
||||||
|
.withProperty("MOYI_UPSTREAM_API_KEYS_MODEL_GPT_5", "model-key");
|
||||||
|
|
||||||
|
provider.chat("provider-deepseek", "gpt-5", "route-a", textInput(128));
|
||||||
|
provider.chat("provider-deepseek", "gpt-5", "route-b", textInput(128));
|
||||||
|
|
||||||
|
assertEquals(List.of("Bearer route-key", "Bearer model-key"), authorization);
|
||||||
|
}
|
||||||
|
|
||||||
private Map<String, Object> textInput(int maxTokens) {
|
private Map<String, Object> textInput(int maxTokens) {
|
||||||
var input = new LinkedHashMap<String, Object>();
|
var input = new LinkedHashMap<String, Object>();
|
||||||
input.put("messages", List.of(Map.of("role", "user", "content", "Reply with OK.")));
|
input.put("messages", List.of(Map.of("role", "user", "content", "Reply with OK.")));
|
||||||
|
|||||||
@@ -192,10 +192,26 @@ MOYI 用户对外使用的统一 API Base URL 为 `https://moyi.soulofmodel.com/
|
|||||||
|
|
||||||
## 一个渠道配置多个上游 Key
|
## 一个渠道配置多个上游 Key
|
||||||
|
|
||||||
可以。当前 OpenAI 兼容转发器支持为同一渠道配置多个上游 Key,并按请求轮换,适合把上游的单 Key 并发拆到多个 Key。服务器环境变量优先使用渠道编码(管理端渠道的 code),也兼容渠道 ID 转成大写下划线后的名字,Key 之间用 `|` 分隔。例如渠道编码为 `apipro` 时:
|
可以。当前 OpenAI 兼容转发器支持为同一渠道配置多个上游 Key,并按请求轮换,适合把上游的单 Key 并发拆到多个 Key。新版还支持“按 MOYI 模型分组固定 Key 池”:路由分组 Key 优先,其次是上游模型 Key,再其次是渠道级 Key,最后才回退到管理端渠道里的单个加密凭据。这样 APIPro、ApiHub、TokenFoundryX 可以为每个上游分组分别配置一个或多个 Key,不会把某个分组的 Key 错轮到另一个分组。
|
||||||
|
|
||||||
|
服务器环境变量优先使用渠道编码(管理端渠道的 code),也兼容渠道 ID 转成大写下划线后的名字,Key 之间用 `|` 分隔。例如渠道编码为 `apipro` 时:
|
||||||
|
|
||||||
```ini
|
```ini
|
||||||
MOYI_UPSTREAM_API_KEYS_APIPRO='key_a|key_b|key_c'
|
MOYI_UPSTREAM_API_KEYS_APIPRO='key_a|key_b|key_c'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
按分组配置时,使用管理端模型路由的内部 ID(不是 `/v1/models` 中可读的公开模型 ID),然后标准化:转大写,把非字母数字替换为下划线。例如路由 ID `relay-route-044e37bb-ab61-4c6d-88a8-13adda55e4c0b` 对应:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
MOYI_UPSTREAM_API_KEYS_ROUTE_RELAY_ROUTE_044E37BB_AB61_4C6D_88A8_13ADDA55E4C0B='group_key'
|
||||||
|
```
|
||||||
|
|
||||||
|
如果希望同一个上游模型的多个分组共用一个 Key 池,可以使用:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
MOYI_UPSTREAM_API_KEYS_MODEL_GLM_4_7='model_key_a|model_key_b'
|
||||||
|
```
|
||||||
|
|
||||||
|
动态变量由 `docker-compose.cloud.yml` 的 API 服务 `env_file: .env.cloud` 注入容器;真实 Key 只能写服务器上的 `.env.cloud`,不能写 Git、迁移文件或聊天记录。每个分组创建一把 Key 时只填单个值。同步请求支持用竖线分隔多把 Key 轮换;异步任务建议每个分组只配置一把 Key,因为部分供应商要求用创建任务的同一凭据轮询任务状态。调用和异步创建/轮询都使用同一优先级规则。
|
||||||
|
|
||||||
保留管理端渠道中的单个凭据作为回退 Key。同步聊天和已接入的异步执行器都会在每个请求或轮询请求前轮换 Key。轮换只解决 Key 分摊,不能突破上游账户总并发、余额、模型授权或速率限制;平台自己的模型并发和单个 MOYI Key 并发仍由网关队列控制。多个 Key 应来自同一渠道、具有相同模型授权,并在服务器密钥文件中保存,不能提交到 Git。当前没有把这些 Key 明文放进管理端数据库的功能,避免把上游密钥暴露给前端;需要新增或撤销 Key 时修改服务器 `.env.cloud` 后重启 API 容器即可。
|
保留管理端渠道中的单个凭据作为回退 Key。同步聊天和已接入的异步执行器都会在每个请求或轮询请求前轮换 Key。轮换只解决 Key 分摊,不能突破上游账户总并发、余额、模型授权或速率限制;平台自己的模型并发和单个 MOYI Key 并发仍由网关队列控制。多个 Key 应来自同一渠道、具有相同模型授权,并在服务器密钥文件中保存,不能提交到 Git。当前没有把这些 Key 明文放进管理端数据库的功能,避免把上游密钥暴露给前端;需要新增或撤销 Key 时修改服务器 `.env.cloud` 后重启 API 容器即可。
|
||||||
@@ -42,8 +42,13 @@ SESSION_TTL_HOURS=168
|
|||||||
JAVA_OPTS='-Xms256m -Xmx1536m -XX:+ExitOnOutOfMemoryError'
|
JAVA_OPTS='-Xms256m -Xmx1536m -XX:+ExitOnOutOfMemoryError'
|
||||||
GATEWAY_REQUESTS_PER_MINUTE=60
|
GATEWAY_REQUESTS_PER_MINUTE=60
|
||||||
GATEWAY_READS_PER_MINUTE=300
|
GATEWAY_READS_PER_MINUTE=300
|
||||||
GATEWAY_DEFAULT_MODEL_CONCURRENCY=4
|
# 0 disables an artificial MOYI-side model/group cap; the upstream provider
|
||||||
|
# still controls its own RPS/RPM/TPM/concurrency limits.
|
||||||
|
GATEWAY_DEFAULT_MODEL_CONCURRENCY=0
|
||||||
GATEWAY_DEFAULT_KEY_CONCURRENCY=4
|
GATEWAY_DEFAULT_KEY_CONCURRENCY=4
|
||||||
|
GATEWAY_DISABLE_MODEL_CONCURRENCY=false
|
||||||
|
GATEWAY_QUEUE_WAIT_MS=30000
|
||||||
|
GATEWAY_QUEUE_POLL_MS=100
|
||||||
TASK_MAX_QUEUED_PER_MODEL=100
|
TASK_MAX_QUEUED_PER_MODEL=100
|
||||||
MOYI_TASK_WORKER_INTERVAL_MS=1000
|
MOYI_TASK_WORKER_INTERVAL_MS=1000
|
||||||
# Accepted async tasks retry polling for up to this duration, then release
|
# Accepted async tasks retry polling for up to this duration, then release
|
||||||
@@ -85,3 +90,12 @@ OSS_ENDPOINT=oss-cn-beijing-internal.aliyuncs.com
|
|||||||
OSS_BUCKET=
|
OSS_BUCKET=
|
||||||
OSS_ACCESS_KEY_ID=
|
OSS_ACCESS_KEY_ID=
|
||||||
OSS_ACCESS_KEY_SECRET=
|
OSS_ACCESS_KEY_SECRET=
|
||||||
|
|
||||||
|
# Optional per-route upstream key pools. Do not commit real values. Replace
|
||||||
|
# ROUTE_ID with the normalized MOYI route/model id (uppercase, non-alphanumerics
|
||||||
|
# replaced by underscores), and separate multiple keys with '|'. Route pools
|
||||||
|
# take precedence over model pools, then channel-level pools, then the single
|
||||||
|
# encrypted credential stored in the admin channel record.
|
||||||
|
# MOYI_UPSTREAM_API_KEYS_ROUTE_<NORMALIZED_ROUTE_ID>='upstream_key'
|
||||||
|
# MOYI_UPSTREAM_API_KEYS_MODEL_<NORMALIZED_UPSTREAM_MODEL>='upstream_key'
|
||||||
|
# MOYI_UPSTREAM_API_KEYS_APIPRO='key_for_channel_fallback_1|key_for_channel_fallback_2'
|
||||||
@@ -9,6 +9,10 @@ services:
|
|||||||
JAVA_RUNTIME_IMAGE: ${JAVA_RUNTIME_IMAGE:-eclipse-temurin:21-jre-alpine}
|
JAVA_RUNTIME_IMAGE: ${JAVA_RUNTIME_IMAGE:-eclipse-temurin:21-jre-alpine}
|
||||||
image: moyi-api:${MOYI_IMAGE_TAG:-dev}
|
image: moyi-api:${MOYI_IMAGE_TAG:-dev}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
# Pass per-route key pools from the deployment secret file without
|
||||||
|
# committing secret names or values to the repository.
|
||||||
|
env_file:
|
||||||
|
- .env.cloud
|
||||||
environment:
|
environment:
|
||||||
CONTROL_PLANE_HOST: 0.0.0.0
|
CONTROL_PLANE_HOST: 0.0.0.0
|
||||||
CONTROL_PLANE_PORT: 8787
|
CONTROL_PLANE_PORT: 8787
|
||||||
|
|||||||
Reference in new issue
Block a user