fix async queue and provider key rotation
This commit is contained in:
1 parent
962eef26e5
commit
9750aceb03
4 files changed
+133
-26
No files matched your search
+13
-4
@@ -17,6 +17,8 @@ import java.util.Map;
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.springframework.dao.EmptyResultDataAccessException;
|
||||
@@ -31,6 +33,9 @@ public class OpenAiCompatibleProvider {
|
||||
private final SecretCipher secretCipher;
|
||||
private final Map<String, ProviderConfig> providers;
|
||||
private final org.springframework.core.env.Environment environment;
|
||||
/** Cursor state must survive dynamic provider resolution; otherwise every
|
||||
* DB-backed channel request starts at key 0 and never rotates. */
|
||||
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
|
||||
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
||||
private final int timeoutSeconds;
|
||||
|
||||
@@ -189,7 +194,8 @@ public class OpenAiCompatibleProvider {
|
||||
String ciphertext = row.get("credential_ciphertext") == null ? "" : String.valueOf(row.get("credential_ciphertext"));
|
||||
String apiKey = ciphertext.isBlank() ? "" : secretCipher.decrypt(ciphertext);
|
||||
String baseUrl = row.get("base_url") == null ? "" : String.valueOf(row.get("base_url"));
|
||||
return configured(providerId, providerCode, apiKey, baseUrl, environment);
|
||||
return configured(providerId, providerCode, apiKey, baseUrl, environment,
|
||||
keyCursors.computeIfAbsent(providerId, ignored -> new AtomicInteger()));
|
||||
} catch (EmptyResultDataAccessException exception) {
|
||||
return fixed;
|
||||
}
|
||||
@@ -204,15 +210,19 @@ public class OpenAiCompatibleProvider {
|
||||
}
|
||||
|
||||
private ProviderConfig configured(String providerId, String fallbackKey, String baseUrl, org.springframework.core.env.Environment environment) {
|
||||
return configured(providerId, providerId, fallbackKey, baseUrl, environment);
|
||||
return configured(providerId, providerId, fallbackKey, baseUrl, environment, new AtomicInteger());
|
||||
}
|
||||
|
||||
private ProviderConfig configured(String providerId, String providerCode, String fallbackKey, String baseUrl, org.springframework.core.env.Environment environment) {
|
||||
return configured(providerId, providerCode, fallbackKey, baseUrl, environment, new AtomicInteger());
|
||||
}
|
||||
|
||||
private ProviderConfig configured(String providerId, String providerCode, String fallbackKey, String baseUrl, org.springframework.core.env.Environment environment, AtomicInteger cursor) {
|
||||
List<String> keys = new ArrayList<>();
|
||||
if (fallbackKey != null && !fallbackKey.isBlank()) keys.add(fallbackKey.trim());
|
||||
addEnvironmentKeys(keys, providerId, environment);
|
||||
if (providerCode != null && !providerCode.isBlank() && !providerCode.equals(providerId)) addEnvironmentKeys(keys, providerCode, environment);
|
||||
return new ProviderConfig(keys.stream().distinct().toList(), baseUrl);
|
||||
return new ProviderConfig(keys.stream().distinct().toList(), baseUrl, cursor);
|
||||
}
|
||||
|
||||
private void addEnvironmentKeys(List<String> keys, String alias, org.springframework.core.env.Environment environment) {
|
||||
@@ -222,7 +232,6 @@ public class OpenAiCompatibleProvider {
|
||||
}
|
||||
|
||||
private record ProviderConfig(List<String> apiKeys, String baseUrl, AtomicInteger cursor) {
|
||||
private ProviderConfig(List<String> apiKeys, String baseUrl) { this(apiKeys, baseUrl, new AtomicInteger()); }
|
||||
private boolean hasKey() { return !apiKeys.isEmpty(); }
|
||||
private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); }
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user