feat: bound async HTTP attempts and track remaining security work

This commit is contained in:
xpp committed 2026-10-21 22:07:56 +08:00
1 parent d994ffb9a5
commit 926aa64ba3
13 files changed
+479 -8

No files matched your search

+4
View File
@@ -25,6 +25,10 @@
`provider.credentials.UpstreamCredentialResolver` 共享密钥池解析与轮转;`provider.transport.SseEventReader` 专门负责 SSE 解帧、UTF-8 校验和有界读取,不承担风控、路由或计费。完整模块边界、验证命令和后续风险清单见 `../docs/21-api-request-modularization.md`。
`V49` 增加数据库共享登录窗口;所有新密码统一 8–256 位,改密撤销全部浏览器会话,登录/改密/重置通过用户行锁协调。初次创建管理员必须显式配置非默认密码;已存在管理员不会被启动配置重置。API Key 不随改密自动撤销。
`provider.transport.BoundedHttpTransport` 已供异步 JSON/multipart/音频/媒体读取复用:接收中限量、总期限覆盖响应体、取消订阅、不自动跟随重定向。它不等于 SSRF 已解决,DNS 预检与实际解析的绑定仍需补齐;同步/SSE 生命周期和 Codex 长请求亦未统一完成。详见 `../docs/25-auth-and-bounded-transport.md` 与 `../docs/24-security-and-cpr-task-tracker.md`。
Flyway 脚本位于 `src/main/resources/db/migration/`,与现有 TypeScript 服务使用同一套基础表命名,后续迁移数据时可以按表逐步切换。
本目录的镜像由 `infra/moyi/docker-compose.yml` 构建。生产发布前应验证目标渠道能力、价格快照、Redis 准入和 PostgreSQL 集成测试;本地通过不代表线上部署验收。TypeScript Control Plane 保留为历史参考实现,不与 Java 后端同时承担同一钱包的写入。
@@ -0,0 +1,114 @@
package com.moyi.controlplane.provider.transport;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.http.HttpClient;
import java.net.http.HttpHeaders;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Flow;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/** One bounded HTTP attempt. No credentials, route selection, protocol mapping, retry, risk or billing policy. */
public final class BoundedHttpTransport {
private static final int ERROR_BODY_LIMIT = 64 * 1024;
private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10))
.followRedirects(HttpClient.Redirect.NEVER).build();
public record Response(int statusCode, HttpHeaders headers, byte[] body) {
@Override public String toString() { return "TransportResponse[status=" + statusCode + ", bytes=" + body.length + "]"; }
}
public enum Reason { TIMEOUT, INTERRUPTED, BODY_TOO_LARGE, IO }
public static final class Failure extends IOException {
private final Reason reason;
private final Integer observedStatus;
private Failure(Reason reason, Integer status) {
super("Bounded HTTP attempt failed: " + reason);
this.reason = reason; this.observedStatus = status;
}
public Reason reason() { return reason; }
public Integer observedStatus() { return observedStatus; }
}
public Response send(HttpRequest request, int maxResponseBytes) throws Failure {
if (maxResponseBytes < 1 || maxResponseBytes > 32 * 1024 * 1024) throw new IllegalArgumentException("Invalid response limit");
Duration timeout = request.timeout().orElseThrow(() -> new IllegalArgumentException("An explicit total HTTP deadline is required"));
var status = new AtomicInteger();
var subscriber = new AtomicReference<LimitedBody>();
CompletableFuture<HttpResponse<byte[]>> attempt = client.sendAsync(request, info -> {
status.set(info.statusCode());
int limit = info.statusCode() >= 200 && info.statusCode() < 300 ? maxResponseBytes : Math.min(maxResponseBytes, ERROR_BODY_LIMIT);
var body = new LimitedBody(limit, info.statusCode(), info.headers().firstValueAsLong("Content-Length").orElse(-1));
subscriber.set(body);
return body;
});
try {
// Unlike a header-only timeout, this also bounds a server that drips or never finishes the body.
HttpResponse<byte[]> response = attempt.get(Math.max(1, timeout.toMillis()), TimeUnit.MILLISECONDS);
return new Response(response.statusCode(), response.headers(), response.body());
} catch (TimeoutException exception) {
throw cancel(attempt, subscriber.get(), Reason.TIMEOUT, status.get());
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw cancel(attempt, subscriber.get(), Reason.INTERRUPTED, status.get());
} catch (ExecutionException exception) {
Throwable cause = exception.getCause();
while (cause instanceof java.util.concurrent.CompletionException && cause.getCause() != null) cause = cause.getCause();
if (cause instanceof Failure failure) throw failure;
throw cancel(attempt, subscriber.get(), cause instanceof java.net.http.HttpTimeoutException ? Reason.TIMEOUT : Reason.IO, status.get());
}
}
private Failure cancel(CompletableFuture<?> attempt, LimitedBody body, Reason reason, int status) {
Failure failure = new Failure(reason, status == 0 ? null : status);
if (body != null) body.cancel(failure);
attempt.cancel(true);
return failure;
}
private static final class LimitedBody implements HttpResponse.BodySubscriber<byte[]> {
private final CompletableFuture<byte[]> completed = new CompletableFuture<>();
private final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
private final int limit, status;
private final long declaredLength;
private Flow.Subscription subscription;
private int received;
LimitedBody(int limit, int status, long declaredLength) { this.limit = limit; this.status = status; this.declaredLength = declaredLength; }
@Override public CompletionStage<byte[]> getBody() { return completed; }
@Override public void onSubscribe(Flow.Subscription subscription) {
this.subscription = subscription;
if (completed.isDone()) { subscription.cancel(); return; }
if (declaredLength > limit) { cancel(new Failure(Reason.BODY_TOO_LARGE, status)); return; }
subscription.request(1);
}
@Override public void onNext(List<ByteBuffer> buffers) {
if (completed.isDone()) { if (subscription != null) subscription.cancel(); return; }
long chunkBytes = 0;
for (ByteBuffer buffer : buffers) chunkBytes += buffer.remaining();
if (chunkBytes > limit - received) { cancel(new Failure(Reason.BODY_TOO_LARGE, status)); return; }
received += (int) chunkBytes;
byte[] part = new byte[8192];
for (ByteBuffer buffer : buffers) while (buffer.hasRemaining()) {
int count = Math.min(part.length, buffer.remaining());
buffer.get(part, 0, count); bytes.write(part, 0, count);
}
subscription.request(1);
}
@Override public void onError(Throwable error) { completed.completeExceptionally(new Failure(Reason.IO, status)); }
@Override public void onComplete() { completed.complete(bytes.toByteArray()); }
void cancel(Failure failure) {
completed.completeExceptionally(failure);
if (subscription != null) subscription.cancel();
}
}
}
@@ -7,11 +7,10 @@ import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.security.SecretCipher;
import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver;
import com.moyi.controlplane.provider.transport.ProviderRequestException;
import com.moyi.controlplane.provider.transport.BoundedHttpTransport;
import java.net.URI;
import java.net.InetAddress;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -42,7 +41,7 @@ public class AsyncProviderClient {
/** Keep rotation state across submit/poll calls; rebuilding a Provider on
* every request otherwise selected the first upstream key forever. */
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
private final BoundedHttpTransport transport = new BoundedHttpTransport();
public AsyncProviderClient(JdbcTemplate jdbc, SecretCipher cipher, ObjectMapper mapper, Environment environment) {
this.jdbc = jdbc; this.cipher = cipher; this.mapper = mapper; this.environment = environment;
@@ -321,7 +320,7 @@ public class AsyncProviderClient {
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
HttpRequest request = builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build();
sending = true;
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
var response = transport.send(request, 8 * 1024 * 1024);
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
byte[] bytes = response.body();
if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502);
@@ -361,7 +360,7 @@ public class AsyncProviderClient {
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
.POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build();
sending = true;
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
var response = transport.send(request, 8 * 1024 * 1024);
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
Object code = parsed.get("code");
@@ -379,6 +378,10 @@ public class AsyncProviderClient {
private ProviderRequestException requestFailure(Exception exception, boolean sending, String message) {
if (exception instanceof InterruptedException) Thread.currentThread().interrupt();
if (exception instanceof ProviderRequestException failure) return failure;
if (exception instanceof BoundedHttpTransport.Failure failure && failure.observedStatus() != null
&& (failure.observedStatus() < 200 || failure.observedStatus() >= 300)) {
return ProviderRequestException.http(failure.observedStatus(), message);
}
if (sending) return ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", message);
if (exception instanceof ApiException error) return ProviderRequestException.notSent(error.getCode(), error.getMessage(), error.getStatus());
return ProviderRequestException.notSent("UPSTREAM_REQUEST_NOT_SENT", "请求准备失败,尚未提交上游生成", 502);
@@ -409,7 +412,12 @@ public class AsyncProviderClient {
}
}
HttpRequest request = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(15)).GET().build();
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
BoundedHttpTransport.Response response;
try { response = transport.send(request, 8 * 1024 * 1024); }
catch (BoundedHttpTransport.Failure failure) {
if (failure.reason() == BoundedHttpTransport.Reason.BODY_TOO_LARGE) throw new ApiException("MEDIA_TOO_LARGE", "媒体输入超过 8 MiB", 400);
throw new ApiException("MEDIA_FETCH_FAILED", "无法在期限内读取媒体输入", 400);
}
if (response.statusCode() < 200 || response.statusCode() >= 300) {
throw new ApiException("MEDIA_FETCH_FAILED", "无法读取媒体输入(HTTP " + response.statusCode() + ")", 400);
}
@@ -447,7 +455,7 @@ public class AsyncProviderClient {
else builder.GET();
HttpRequest request = builder.build();
sending = true;
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
var response = transport.send(request, 8 * 1024 * 1024);
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
Object code = parsed.get("code");
@@ -0,0 +1,32 @@
package com.moyi.controlplane.gateway.request;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/** Lightweight source dependency gates, not a substitute for design review or a sandbox. */
class ModuleDependencyTest {
private final Path root = Path.of(System.getProperty("basedir", "."), "src/main/java/com/moyi/controlplane");
@Test void requestApplicationBoundaryDoesNotOwnHttpDatabaseOrProviderCredentials() throws Exception {
disallow(root.resolve("gateway/request"), List.of("java.net.http", "org.springframework.jdbc", ".security.SecretCipher", ".provider.credentials", ".task.AsyncProviderClient"));
}
@Test void transportDoesNotOwnRiskRoutingOrTheWallet() throws Exception {
disallow(root.resolve("provider/transport"), List.of(".gateway.", ".wallet.", ".auth.", ".task.", "org.springframework.jdbc", ".security.SecretCipher"));
}
@Test void taskSettlementDoesNotCallAnUpstream() throws Exception {
assertTrue(Files.exists(root.resolve("task/TaskSettlementService.java")));
disallow(root.resolve("task/TaskSettlementService.java"), List.of("java.net.http", "AsyncProviderClient", "OpenAiCompatibleProvider"));
}
private void disallow(Path path, List<String> forbidden) throws Exception {
assertTrue(Files.exists(path), "Expected source boundary: " + path);
try (var paths = Files.isDirectory(path) ? Files.walk(path) : java.util.stream.Stream.of(path)) {
for (Path file : paths.filter(p -> p.toString().endsWith(".java")).toList()) {
String source = Files.readString(file);
for (String dependency : forbidden) assertFalse(source.contains(dependency), file + " crosses boundary via " + dependency);
}
}
}
}
@@ -0,0 +1,113 @@
package com.moyi.controlplane.provider.transport;
import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpRequest;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class BoundedHttpTransportTest {
private HttpServer server;
private final BoundedHttpTransport transport = new BoundedHttpTransport();
private final CountDownLatch release = new CountDownLatch(1);
@BeforeEach void setup() throws Exception { server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); }
@AfterEach void cleanup() { release.countDown(); server.stop(0); }
private HttpRequest request(String path, long timeoutMs) { return HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path)).timeout(Duration.ofMillis(timeoutMs)).GET().build(); }
@Test void returnsBoundedBytesWithoutExposingBodyInDiagnostics() throws Exception {
server.createContext("/", exchange -> {
byte[] bytes = "private-provider-result".getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(200, bytes.length);
try (var output = exchange.getResponseBody()) { output.write(bytes); }
}); server.start();
var response = transport.send(request("/", 2000), 1024);
assertEquals("private-provider-result", new String(response.body(), StandardCharsets.UTF_8));
assertFalse(response.toString().contains("private-provider-result"));
}
@Test void oversizedChunkedResponseIsCancelledBeforeUnboundedBuffering() {
var calls = new AtomicInteger();
server.createContext("/", exchange -> {
calls.incrementAndGet(); exchange.sendResponseHeaders(200, 0);
try (var output = exchange.getResponseBody()) { for (int i = 0; i < 100; i++) output.write(new byte[4096]); }
catch (java.io.IOException expectedAfterCancellation) { }
}); server.start();
var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/", 2000), 8192));
assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason());
assertEquals(200, failure.observedStatus()); assertEquals(1, calls.get());
}
@Test void declaredOversizeIsRejectedWithoutWaitingForTheBody() {
server.createContext("/", exchange -> {
exchange.sendResponseHeaders(200, 1_000_000);
try { release.await(3, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); }
exchange.close();
}); server.start();
var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/", 1000), 8192));
assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason());
}
@Test void totalDeadlineIncludesBodyReadsAfterHeaders() {
var calls = new AtomicInteger();
server.createContext("/", exchange -> {
calls.incrementAndGet(); exchange.sendResponseHeaders(200, 0);
try (var output = exchange.getResponseBody()) {
output.write('x'); output.flush();
try { release.await(3, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); }
} catch (java.io.IOException expectedAfterCancellation) { }
}); server.start();
long start = System.nanoTime();
var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/", 200), 8192));
assertEquals(BoundedHttpTransport.Reason.TIMEOUT, failure.reason());
assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < 2000);
assertEquals(1, calls.get());
}
@Test void errorBodiesUseASmallerLimitAndRetainOnlyObservedStatus() {
server.createContext("/", exchange -> { exchange.sendResponseHeaders(401, 1_000_000); exchange.close(); }); server.start();
var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/?private-query", 1000), 8 * 1024 * 1024));
assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason());
assertEquals(401, failure.observedStatus());
assertFalse(failure.toString().contains("private-query"));
}
@Test void redirectsAreReturnedAndNeverFollowed() throws Exception {
var redirected = new AtomicInteger();
server.createContext("/start", exchange -> { exchange.getResponseHeaders().set("Location", "/internal"); exchange.sendResponseHeaders(302, -1); exchange.close(); });
server.createContext("/internal", exchange -> { redirected.incrementAndGet(); exchange.sendResponseHeaders(200, -1); exchange.close(); }); server.start();
assertEquals(302, transport.send(request("/start", 2000), 1024).statusCode());
assertEquals(0, redirected.get());
}
@Test void interruptionCancelsTheAttemptAndRestoresInterruptFlag() throws Exception {
var arrived = new CountDownLatch(1); var result = new AtomicReference<BoundedHttpTransport.Reason>(); var interrupted = new AtomicReference<Boolean>();
server.createContext("/", exchange -> {
exchange.sendResponseHeaders(200, 0); arrived.countDown();
try { release.await(5, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); }
exchange.close();
}); server.start();
var worker = new Thread(() -> {
try { transport.send(request("/", 5000), 1024); }
catch (BoundedHttpTransport.Failure failure) { result.set(failure.reason()); interrupted.set(Thread.currentThread().isInterrupted()); }
}); worker.setDaemon(true); worker.start();
try {
assertTrue(arrived.await(2, TimeUnit.SECONDS)); worker.interrupt(); worker.join(2000);
assertFalse(worker.isAlive()); assertEquals(BoundedHttpTransport.Reason.INTERRUPTED, result.get()); assertEquals(true, interrupted.get());
} finally { release.countDown(); worker.interrupt(); }
}
@Test void callersCannotOmitDeadlinesOrRequestUnboundedResponses() {
var request = HttpRequest.newBuilder(URI.create("http://127.0.0.1:1/")).GET().build();
assertThrows(IllegalArgumentException.class, () -> transport.send(request, 1024));
assertThrows(IllegalArgumentException.class, () -> transport.send(request, Integer.MAX_VALUE));
}
}
@@ -298,6 +298,23 @@ class AsyncProviderClientTest {
assertEquals("INVALID_CREDENTIAL_FINGERPRINT", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "test-key")).getCode());
}
@ParameterizedTest
@CsvSource({"200,UNKNOWN", "401,REJECTED"})
void responseLimitKeepsTheObservedAcceptanceBoundary(int status, ProviderRequestException.Acceptance expected) throws Exception {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
var calls = new java.util.concurrent.atomic.AtomicInteger();
server.createContext("/", exchange -> {
calls.incrementAndGet(); exchange.getRequestBody().readAllBytes();
exchange.sendResponseHeaders(status, 9 * 1024 * 1024); exchange.close();
}); server.start();
try {
var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort());
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "test"));
assertEquals(expected, failure.acceptance());
assertEquals(1, calls.get());
} finally { server.stop(0); }
}
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
var client = clientFor("http://127.0.0.1:1");
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
@@ -127,6 +127,16 @@ class AsyncTaskPostgresTest {
verifyNoInteractions(provider);
}
@Test void taskReadsAreOwnerScopedAndCannotExposeAnotherUsersResult() {
UUID anotherUser = UUID.randomUUID(), anotherKey = UUID.randomUUID();
when(catalog.keyOwnerId(anotherKey)).thenReturn(anotherUser);
var foreignKey = Map.<String,Object>of("id", anotherKey.toString());
assertEquals("TASK_NOT_FOUND", assertThrows(ApiException.class, () -> tasks.getForKey(foreignKey, taskId)).getCode());
when(catalog.authenticateKey("foreign-token")).thenReturn(foreignKey);
assertTrue(tasks.list("Bearer foreign-token", 100).isEmpty());
assertEquals(taskId, tasks.getForKey(Map.of("id", keyId.toString()), taskId).get("id"));
}
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
var first = create("client-operation-1", input);