package com.moyi.controlplane.task; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.moyi.controlplane.common.ApiException; import com.moyi.controlplane.security.SecretCipher; import java.net.URI; import java.net.InetAddress; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Base64; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.ArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; /** Real async adapters for the providers currently represented in the catalog. */ @Service public class AsyncProviderClient { private static final Duration UPSTREAM_REQUEST_TIMEOUT = Duration.ofSeconds(60); private final JdbcTemplate jdbc; private final SecretCipher cipher; private final ObjectMapper mapper; private final Environment environment; /** Keep rotation state across submit/poll calls; rebuilding a Provider on * every request otherwise selected the first upstream key forever. */ private final ConcurrentMap keyCursors = new ConcurrentHashMap<>(); private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); public AsyncProviderClient(JdbcTemplate jdbc, SecretCipher cipher, ObjectMapper mapper, Environment environment) { this.jdbc = jdbc; this.cipher = cipher; this.mapper = mapper; this.environment = environment; } public Map submit(String providerId, String upstreamModel, String taskType, String prompt) { return submit(providerId, upstreamModel, taskType, Map.of("prompt", prompt)); } public Map submit(String providerId, String upstreamModel, String taskType, Map input) { if (input == null) input = Map.of(); String prompt = String.valueOf(input.getOrDefault("prompt", "")); Provider provider = provider(providerId); String code = provider.code().toLowerCase(java.util.Locale.ROOT); if (isTripo(provider.code(), provider.baseUrl())) { Map body = new LinkedHashMap<>(); body.put("model", upstreamModel); String media = mediaUrl(input); if (media != null) body.put("input", media); else if (!prompt.isBlank()) body.put("prompt", prompt); body.put("texture", true); body.put("pbr", true); return request("POST", tripoBase(provider.baseUrl()) + (media == null ? "/generation/text-to-model" : "/generation/image-to-model"), provider.nextApiKey(), body, "上游 3D 任务创建失败"); } if (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com")) { List> content = new ArrayList<>(); if (!prompt.isBlank()) content.add(Map.of("type", "text", "text", prompt)); String media = mediaUrl(input); if (media != null) { if (media.startsWith("file_")) throw new ApiException("MEDIA_INPUT_UNSUPPORTED", "字节跳动视频执行器需要可公开访问的图片 URL", 400); content.add(Map.of("type", "image_url", "image_url", Map.of("url", media))); } Map body = Map.of("model", upstreamModel, "content", content); return request("POST", trim(provider.baseUrl()) + "/contents/generations/tasks", provider.nextApiKey(), body, "上游视频任务创建失败"); } if ("openai-compatible".equalsIgnoreCase(provider.protocol())) { return submitOpenAiCompatible(provider, upstreamModel, taskType, input); } throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501); } public Map poll(String providerId, String providerTaskId) { Provider provider = provider(providerId); String code = provider.code().toLowerCase(java.util.Locale.ROOT); String url; if (isTripo(provider.code(), provider.baseUrl())) url = tripoBase(provider.baseUrl()) + "/tasks/" + encode(providerTaskId); else if (code.contains("byte") || code.contains("seedance") || provider.baseUrl().contains("volces.com")) url = trim(provider.baseUrl()) + "/contents/generations/tasks/" + encode(providerTaskId); else if ("openai-compatible".equalsIgnoreCase(provider.protocol())) { return pollOpenAiCompatible(provider, providerTaskId); } else throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道没有可用的执行器", 501); return request("GET", url, provider.nextApiKey(), null, "上游异步任务查询失败"); } /** * OpenAI-compatible relay providers expose the standard image endpoint and * increasingly expose the asynchronous video endpoint as well. Image * generation completes in the request, so mark the normalized result for * AsyncTaskService to settle immediately instead of inventing a task id. */ private Map submitOpenAiCompatible(Provider provider, String model, String taskType, Map input) { String prompt = String.valueOf(input.getOrDefault("prompt", "")); String media = mediaUrl(input); String base = trim(provider.baseUrl()); if ("image".equalsIgnoreCase(taskType)) { Map body = new LinkedHashMap<>(); body.put("model", model); body.put("prompt", prompt); body.put("n", 1); // DALL-E accepts `url` while GPT-image models require // `b64_json` and reject response_format altogether. Forward an // explicitly requested format only; otherwise let the provider // choose its documented default and normalize either shape. copyParameter(input, body, "response_format"); copyParameter(input, body, "size"); Map response = media == null ? request("POST", base + "/images/generations", provider.nextApiKey(), body, "上游图片任务创建失败") : requestMultipart("POST", base + "/images/edits", provider.nextApiKey(), body, "image", media, "上游图片任务创建失败"); Map result = compatibleResult(response); if (result != null) return Map.of("_moyiInlineResult", result); return response; } if ("video".equalsIgnoreCase(taskType)) { Map body = new LinkedHashMap<>(); body.put("model", model); body.put("prompt", prompt); copyParameter(input, body, "seconds"); copyParameter(input, body, "size"); // The standard OpenAI video endpoint is multipart even when no // reference file is supplied. JSON requests are rejected by the // official API and by relays that proxy it faithfully. String apiKey = provider.nextApiKey(); try { return requestMultipart("POST", base + "/videos", apiKey, body, media == null ? null : "input_reference", media, "上游视频任务创建失败"); } catch (ApiException first) { // A few older relays retained /videos/generations. Retry only // on a route miss; authentication/credit errors are returned. if (!isNotFound(first)) throw first; return requestMultipart("POST", base + "/videos/generations", apiKey, body, media == null ? null : "input_reference", media, "上游视频任务创建失败"); } } if ("audio".equalsIgnoreCase(taskType)) { if (media != null) throw new ApiException("MEDIA_INPUT_UNSUPPORTED", "该语音生成接口接收文本;音频转写需使用专用转写接口", 400); // OpenAI-compatible TTS returns an audio/* binary body. Keep the // payload bounded before embedding it into the task result so a // provider cannot make the database grow without limit. Map body = new LinkedHashMap<>(); body.put("model", model); body.put("input", prompt); body.put("voice", input.getOrDefault("voice", "alloy")); body.put("response_format", input.getOrDefault("response_format", "mp3")); copyParameter(input, body, "speed"); return Map.of("_moyiInlineResult", requestAudio("POST", base + "/audio/speech", provider.nextApiKey(), body, "上游语音任务创建失败")); } throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "该异步渠道的媒体接口尚未配置", 501); } private void copyParameter(Map input, Map body, String name) { Object value = input.get(name); if (value != null) body.put(name, value); } private String mediaUrl(Map input) { Object raw = input.get("media_url"); if (raw == null || String.valueOf(raw).isBlank()) return null; String value = String.valueOf(raw).trim(); if (value.startsWith("file_")) return value; // Tripo file token from /v3/files try { URI url = URI.create(value); if (!"https".equalsIgnoreCase(url.getScheme()) || url.getHost() == null || url.getUserInfo() != null) throw new IllegalArgumentException(); } catch (IllegalArgumentException exception) { throw new ApiException("INVALID_MEDIA_URL", "媒体地址必须是可公开访问的 HTTPS URL", 400); } return value; } private Map pollOpenAiCompatible(Provider provider, String providerTaskId) { String base = trim(provider.baseUrl()); String apiKey = provider.nextApiKey(); try { return request("GET", base + "/videos/" + encode(providerTaskId), apiKey, null, "上游异步任务查询失败"); } catch (ApiException first) { if (!isNotFound(first)) throw first; return request("GET", base + "/videos/generations/" + encode(providerTaskId), apiKey, null, "上游异步任务查询失败"); } } private boolean isNotFound(ApiException exception) { String message = exception.getMessage(); return message != null && (message.contains("HTTP 404") || message.contains("HTTP 405")); } /** Normalize OpenAI image/audio response fields to the task result shape. */ private Map compatibleResult(Map response) { Object data = response.get("data"); if (data instanceof List list && !list.isEmpty()) { Object first = list.getFirst(); if (first instanceof Map map) { Map result = new LinkedHashMap<>(); map.forEach((key, value) -> result.put(String.valueOf(key), value)); Object url = result.get("url"); if (url == null) url = result.get("audio_url"); if (url != null) result.put("url", url); if (result.get("b64_json") != null) result.put("base64", result.get("b64_json")); return result; } } Object url = response.get("url"); if (url != null) return Map.of("url", url); return null; } private Map requestAudio(String method, String url, String key, Map body, String message) { try { HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT) .header("Authorization", "Bearer " + key).header("Content-Type", "application/json"); HttpResponse response = client.send(builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(), HttpResponse.BodyHandlers.ofByteArray()); if (response.statusCode() < 200 || response.statusCode() >= 300) { String code = switch (response.statusCode()) { case 401, 403 -> "UPSTREAM_AUTH_FAILED"; case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT"; case 429 -> "UPSTREAM_RATE_LIMITED"; default -> "UPSTREAM_PROVIDER_ERROR"; }; throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502); } byte[] bytes = response.body(); if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502); String contentType = response.headers().firstValue("Content-Type").orElse("audio/mpeg").split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT); if (!contentType.startsWith("audio/")) throw new ApiException("UPSTREAM_INVALID_MEDIA", message + "(返回内容不是音频)", 502); Map result = new LinkedHashMap<>(); result.put("base64", Base64.getEncoder().encodeToString(bytes)); result.put("mimeType", contentType); result.put("format", String.valueOf(body.getOrDefault("response_format", "mp3"))); return result; } catch (ApiException exception) { throw exception; } catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } } /** * Send the file based forms used by the standard OpenAI image-edit and * video-create endpoints. The caller supplies an HTTPS URL; the server * fetches it with redirects disabled and a strict 8 MiB limit, then sends * the bytes as a multipart part. This avoids putting an arbitrary URL in a * JSON field that the upstream endpoint does not understand. */ private Map requestMultipart(String method, String url, String key, Map fields, String fileField, String media, String message) { try { String boundary = "----moyi-" + java.util.UUID.randomUUID(); ByteArrayOutputStream out = new ByteArrayOutputStream(); for (Map.Entry entry : fields.entrySet()) { multipartField(out, boundary, entry.getKey(), String.valueOf(entry.getValue())); } if (fileField != null && media != null) { MediaBytes file = downloadMedia(media); multipartFile(out, boundary, fileField, file.filename(), file.contentType(), file.bytes()); } out.write(("--" + boundary + "--\r\n").getBytes(java.nio.charset.StandardCharsets.UTF_8)); HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT) .header("Authorization", "Bearer " + key) .header("Content-Type", "multipart/form-data; boundary=" + boundary) .POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message); Map parsed = mapper.readValue(response.body(), new TypeReference<>() {}); Object code = parsed.get("code"); if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } return parsed; } catch (ApiException exception) { throw exception; } catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } } private ApiException upstreamStatus(int status, String message) { String code = switch (status) { case 401, 403 -> "UPSTREAM_AUTH_FAILED"; case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT"; case 429 -> "UPSTREAM_RATE_LIMITED"; default -> "UPSTREAM_PROVIDER_ERROR"; }; return new ApiException(code, message + "(HTTP " + status + ")", 502); } private void multipartField(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException { out.write(("--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + name + "\"\r\n\r\n" + value + "\r\n") .getBytes(java.nio.charset.StandardCharsets.UTF_8)); } private void multipartFile(ByteArrayOutputStream out, String boundary, String field, String filename, String contentType, byte[] bytes) throws IOException { out.write(("--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + field + "\"; filename=\"" + filename + "\"\r\nContent-Type: " + contentType + "\r\n\r\n").getBytes(java.nio.charset.StandardCharsets.UTF_8)); out.write(bytes); out.write("\r\n".getBytes(java.nio.charset.StandardCharsets.UTF_8)); } private MediaBytes downloadMedia(String value) throws IOException, InterruptedException { URI uri = URI.create(value); if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null || uri.getUserInfo() != null) { throw new ApiException("INVALID_MEDIA_URL", "媒体地址必须是可公开访问的 HTTPS URL", 400); } for (InetAddress address : InetAddress.getAllByName(uri.getHost())) { if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress() || address.isSiteLocalAddress() || address.isMulticastAddress()) { throw new ApiException("INVALID_MEDIA_URL", "媒体地址不能指向内网或本机地址", 400); } } HttpRequest request = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(15)).GET().build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); if (response.statusCode() < 200 || response.statusCode() >= 300) { throw new ApiException("MEDIA_FETCH_FAILED", "无法读取媒体输入(HTTP " + response.statusCode() + ")", 400); } byte[] bytes = response.body(); if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) { throw new ApiException("MEDIA_TOO_LARGE", "媒体输入为空或超过 8 MiB", 400); } String contentType = response.headers().firstValue("Content-Type").orElse("application/octet-stream") .split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT); if (!contentType.startsWith("image/")) { throw new ApiException("MEDIA_TYPE_UNSUPPORTED", "图像编辑和视频参考图必须是 image/* 文件", 400); } String filename = "input" + extension(contentType); return new MediaBytes(bytes, contentType, filename); } private String extension(String contentType) { return switch (contentType) { case "image/jpeg" -> ".jpg"; case "image/png" -> ".png"; case "image/webp" -> ".webp"; case "image/gif" -> ".gif"; default -> ".bin"; }; } private record MediaBytes(byte[] bytes, String contentType, String filename) {} private Map request(String method, String url, String key, Map body, String message) { try { HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT) .header("Authorization", "Bearer " + key).header("Content-Type", "application/json"); if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body))); else builder.GET(); HttpResponse response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString()); if (response.statusCode() < 200 || response.statusCode() >= 300) { String code = switch (response.statusCode()) { case 401, 403 -> "UPSTREAM_AUTH_FAILED"; case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT"; case 429 -> "UPSTREAM_RATE_LIMITED"; default -> "UPSTREAM_PROVIDER_ERROR"; }; throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502); } Map parsed = mapper.readValue(response.body(), new TypeReference<>() {}); Object code = parsed.get("code"); if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) { // Upstream messages can echo credentials or account details. // Only locally defined diagnostics may reach task history/UI. throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } return parsed; } catch (ApiException exception) { throw exception; } catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } } private Provider provider(String id) { try { Map row = jdbc.queryForMap("SELECT code, protocol, base_url, credential_ciphertext FROM model_providers WHERE id=? AND status='active' AND deleted_at IS NULL", id); String ciphertext = row.get("credential_ciphertext") == null ? "" : String.valueOf(row.get("credential_ciphertext")); String key = ciphertext.isBlank() ? "" : cipher.decrypt(ciphertext); String code = String.valueOf(row.get("code")); List keys = new ArrayList<>(); if (!key.isBlank()) keys.add(key); addEnvironmentKeys(keys, code); addEnvironmentKeys(keys, id); // Tripo's official API key may be supplied through the server secret // environment so it never has to be committed or exposed in SQL. if (key.isBlank() && isTripo(code, String.valueOf(row.get("base_url")))) { addEnvironmentKeys(keys, "TRIPO_API_KEY"); addEnvironmentKeys(keys, "MOYI_TRIPO_API_KEY"); } if (keys.isEmpty()) throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道尚未配置 API Key", 503); return new Provider(code, String.valueOf(row.get("protocol")), String.valueOf(row.get("base_url")), keys.stream().distinct().toList(), keyCursors.computeIfAbsent(id, ignored -> new AtomicInteger())); } catch (EmptyResultDataAccessException exception) { throw new ApiException("PROVIDER_NOT_CONFIGURED", "异步渠道不存在或已停用", 503); } } private String tripoBase(String value) { String base = trim(value); // Tripo's current documented endpoint is openapi.tripo3d.com. Older // seeded releases used api.tripo3d.ai; keep custom/local test hosts, // but transparently repair those stale official hosts. String lower = base.toLowerCase(java.util.Locale.ROOT); if (lower.contains("api.tripo3d.ai") || lower.contains("openapi.tripo3d.ai")) base = "https://openapi.tripo3d.com"; return base.endsWith("/v3") ? base : base + "/v3"; } private String trim(String value) { return value == null ? "" : value.replaceAll("/+$", ""); } private String encode(String value) { return java.net.URLEncoder.encode(value, java.nio.charset.StandardCharsets.UTF_8); } private String write(Object value) throws JsonProcessingException { return mapper.writeValueAsString(value == null ? Map.of() : value); } private void addEnvironmentKeys(List keys, String alias) { String name = alias.endsWith("_API_KEY") ? alias : "MOYI_UPSTREAM_API_KEYS_" + alias.toUpperCase(java.util.Locale.ROOT).replace('-', '_'); String configured = environment.getProperty(name); if (configured != null && !configured.isBlank()) for (String value : configured.split("\\|")) if (!value.isBlank()) keys.add(value.trim()); } private boolean isTripo(String code, String baseUrl) { String normalizedCode = code == null ? "" : code.toLowerCase(java.util.Locale.ROOT); String normalizedUrl = baseUrl == null ? "" : baseUrl.toLowerCase(java.util.Locale.ROOT); return normalizedCode.contains("tripo") || normalizedUrl.contains("tripo3d"); } private record Provider(String code, String protocol, String baseUrl, List apiKeys, AtomicInteger cursor) { private String nextApiKey() { return apiKeys.get(Math.floorMod(cursor.getAndIncrement(), apiKeys.size())); } } }