Files
moyi-model-intelligence-hub/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayFailover.java
T

46 lines
3.4 KiB
Java

package com.moyi.controlplane.gateway;
import com.moyi.controlplane.catalog.CatalogService;
import com.moyi.controlplane.provider.*;
import com.moyi.controlplane.wallet.ChatBillingService;
import java.util.*;
import org.springframework.stereotype.Service;
@Service
public class GatewayFailover {
private final CatalogService catalog;private final GatewayGuard guard;private final ChatBillingService billing;private final OpenAiCompatibleProvider provider;
public GatewayFailover(CatalogService catalog,GatewayGuard guard,ChatBillingService billing,OpenAiCompatibleProvider provider) {this.catalog=catalog;this.guard=guard;this.billing=billing;this.provider=provider;}
public List<Map<String,Object>> candidates(Map<String,Object> key,Map<String,Object> primary,String protocol) {
var routes=new ArrayList<Map<String,Object>>();routes.add(primary);
if(!(primary.get("capabilities") instanceof Map<?,?> cap)||!Boolean.TRUE.equals(cap.get("failover"))||primary.get("logicalModelId")==null) return routes;
GatewayModelIds.authorizedModels(catalog,key).stream()
.filter(m->!Objects.equals(m.get("id"),primary.get("id"))&&Objects.equals(m.get("logicalModelId"),primary.get("logicalModelId"))&&Objects.equals(m.get("upstreamModel"),primary.get("upstreamModel"))&&Objects.equals(m.get("type"),primary.get("type"))&&!"async".equals(m.get("mode")))
.filter(m->"chat".equals(protocol)||NativeProtocolMeter.supports(m,protocol))
.filter(m->provider.isConfigured(m.get("providerId").toString(),m.get("id").toString(),m.get("upstreamModel").toString()))
.sorted(Comparator.comparingInt(m->m.get("priority") instanceof Number n?n.intValue():0)).limit(2).forEach(routes::add);
return routes;
}
public Map<String,Object> invoke(Map<String,Object> key,Map<String,Object> model,String protocol,Map<String,Object> input,ChatBillingService.Ticket ticket) {
var routes=candidates(key,model,protocol);
for(int i=0;i<routes.size();i++) {
var route=routes.get(i);try(var slot=i==0?null:guard.acquire(route,1)) {
if(i>0) billing.actualRoute(ticket,route);
try {return "chat".equals(protocol)?provider.chat(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input):provider.invoke(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input,protocol);}
catch(UpstreamRejectedException e) {if(i==routes.size()-1) throw e;}
}
}
throw new IllegalStateException("No route");
}
public OpenAiCompatibleProvider.StreamResponse stream(Map<String,Object> key,Map<String,Object> model,String protocol,Map<String,Object> input,ChatBillingService.Ticket ticket) {
var routes=candidates(key,model,protocol);
for(int i=0;i<routes.size();i++) {
var route=routes.get(i);GatewayGuard.Slot slot=i==0?null:guard.acquire(route,1);
try {
if(i>0) billing.actualRoute(ticket,route);
var response="chat".equals(protocol)?provider.stream(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input):provider.nativeStream(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input,protocol);
if(slot!=null) response.onClose(slot::close);return response;
} catch(RuntimeException e) {if(slot!=null) slot.close();if(!(e instanceof UpstreamRejectedException)||i==routes.size()-1) throw e;}
}
throw new IllegalStateException("No route");
}
}