fix: retain async holds for unknown provider outcomes
This commit is contained in:
1 parent
76265845dc
commit
5be06a97c1
18 files changed
+575
-109
No files matched your search
@@ -15,7 +15,9 @@
|
|||||||
|
|
||||||
协议字段映射留在各自 Meter/Adapter;`ApiRequestRiskRule` 只能读取最小化上下文并返回决策,不能调用 Provider 或改账。新增协议时应复用准入、风控和账务边界,而不是在 Controller 中另写一套钱包/重试流程。默认 `GATEWAY_RISK_MODE=shadow` 仅记录规则 ID,不改变调用;只有显式部署规则并设置 `enforce` 才会拒绝或限速。请求来源由服务端路由确定,不能信任 `X-Moyi-Source` 作为风控分类。
|
协议字段映射留在各自 Meter/Adapter;`ApiRequestRiskRule` 只能读取最小化上下文并返回决策,不能调用 Provider 或改账。新增协议时应复用准入、风控和账务边界,而不是在 Controller 中另写一套钱包/重试流程。默认 `GATEWAY_RISK_MODE=shadow` 仅记录规则 ID,不改变调用;只有显式部署规则并设置 `enforce` 才会拒绝或限速。请求来源由服务端路由确定,不能信任 `X-Moyi-Source` 作为风控分类。
|
||||||
|
|
||||||
账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;失败释放和成功结算沿用现有事务处理;不完整流或无效用量不得结算。上游提交结果未知时的异步核账状态机与异步创建幂等将在后续阶段单独处理。
|
账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;已确认失败的释放和成功结算沿用原子事务;不完整流或无效用量不得估算扣费。
|
||||||
|
|
||||||
|
异步创建支持用户级 `Idempotency-Key`(`V45`),同键同请求返回原任务,同键不同请求返回 409。异步上游受理未知、回执丢失、worker 中断或轮询期限到期时进入 `reconciling`(`V46`),保留冻结与预算占用,不自动重发/退款。已观察的生成结果先持久化再结算,账本失败只重试本地结算。发布时须先停旧 worker,不能混跑旧的未知结果退款逻辑。详见 `docs/22-async-task-safety.md`;受审计的人工核账入口仍属下一阶段。
|
||||||
|
|
||||||
同步和原生调用在结算前单独提交精简用量快照(Flyway `V44`)。快照已持久化而账本事务失败时,接口返回 `SETTLEMENT_PENDING`,冻结额度保留,由后台恢复任务重试结算。该快照不保存提示词、生成内容或密钥;不能将其当作响应重放存储。若数据库在快照写入前已不可用,则尚不能保证结果的持久恢复,必须结合上游记录核对。
|
同步和原生调用在结算前单独提交精简用量快照(Flyway `V44`)。快照已持久化而账本事务失败时,接口返回 `SETTLEMENT_PENDING`,冻结额度保留,由后台恢复任务重试结算。该快照不保存提示词、生成内容或密钥;不能将其当作响应重放存储。若数据库在快照写入前已不可用,则尚不能保证结果的持久恢复,必须结合上游记录核对。
|
||||||
|
|
||||||
|
|||||||
@@ -383,6 +383,7 @@ public class CatalogService {
|
|||||||
result.put("tasks", Map.of(
|
result.put("tasks", Map.of(
|
||||||
"queued", taskCounts.getOrDefault("queued", 0L),
|
"queued", taskCounts.getOrDefault("queued", 0L),
|
||||||
"running", taskCounts.getOrDefault("running", 0L),
|
"running", taskCounts.getOrDefault("running", 0L),
|
||||||
|
"reconciling", taskCounts.getOrDefault("reconciling", 0L),
|
||||||
"succeeded", taskCounts.getOrDefault("succeeded", 0L),
|
"succeeded", taskCounts.getOrDefault("succeeded", 0L),
|
||||||
"failed", taskCounts.getOrDefault("failed", 0L)
|
"failed", taskCounts.getOrDefault("failed", 0L)
|
||||||
));
|
));
|
||||||
|
|||||||
+44
@@ -0,0 +1,44 @@
|
|||||||
|
package com.moyi.controlplane.provider.transport;
|
||||||
|
|
||||||
|
import com.moyi.controlplane.common.ApiException;
|
||||||
|
|
||||||
|
/** HTTP boundary facts, not a retry policy. Unknown acceptance must never authorize replay/refund. */
|
||||||
|
public final class ProviderRequestException extends ApiException {
|
||||||
|
public enum Acceptance { NOT_SENT, REJECTED, UNKNOWN }
|
||||||
|
|
||||||
|
private final Acceptance acceptance;
|
||||||
|
private final Integer upstreamStatus;
|
||||||
|
|
||||||
|
private ProviderRequestException(String code, String message, int status, Acceptance acceptance, Integer upstreamStatus) {
|
||||||
|
super(code, message, status);
|
||||||
|
this.acceptance = acceptance;
|
||||||
|
this.upstreamStatus = upstreamStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ProviderRequestException notSent(String code, String message, int status) {
|
||||||
|
return new ProviderRequestException(code, message, status, Acceptance.NOT_SENT, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ProviderRequestException unknown(String code, String message) {
|
||||||
|
return new ProviderRequestException(code, message, 502, Acceptance.UNKNOWN, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ProviderRequestException http(int status, String message) {
|
||||||
|
String code = switch (status) {
|
||||||
|
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||||
|
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||||
|
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||||
|
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||||
|
};
|
||||||
|
// Timeout/conflict/5xx responses do not prove a generation was never accepted.
|
||||||
|
Acceptance acceptance = switch (status) {
|
||||||
|
case 400, 401, 402, 403, 404, 405, 413, 415, 422, 429 -> Acceptance.REJECTED;
|
||||||
|
default -> Acceptance.UNKNOWN;
|
||||||
|
};
|
||||||
|
return new ProviderRequestException(code, message + "(HTTP " + status + ")", 502, acceptance, status);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Acceptance acceptance() { return acceptance; }
|
||||||
|
public Integer upstreamStatus() { return upstreamStatus; }
|
||||||
|
public boolean definitelyNotAccepted() { return acceptance != Acceptance.UNKNOWN; }
|
||||||
|
}
|
||||||
+43
-35
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import com.moyi.controlplane.common.ApiException;
|
import com.moyi.controlplane.common.ApiException;
|
||||||
import com.moyi.controlplane.security.SecretCipher;
|
import com.moyi.controlplane.security.SecretCipher;
|
||||||
import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver;
|
import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver;
|
||||||
|
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.http.HttpClient;
|
import java.net.http.HttpClient;
|
||||||
@@ -60,6 +61,22 @@ public class AsyncProviderClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public Map<String,Object> submit(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
public Map<String,Object> submit(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
||||||
|
try {
|
||||||
|
return submitConfigured(providerId, upstreamModel, routeId, taskType, input);
|
||||||
|
} catch (ProviderRequestException exception) {
|
||||||
|
throw exception;
|
||||||
|
} catch (ApiException exception) {
|
||||||
|
// Only these local validation/configuration errors are known to precede generation submission.
|
||||||
|
if (List.of("PROVIDER_NOT_CONFIGURED", "UPSTREAM_KEY_NOT_AVAILABLE", "ASYNC_PROVIDER_NOT_IMPLEMENTED",
|
||||||
|
"MEDIA_INPUT_UNSUPPORTED", "INVALID_MEDIA_URL", "MEDIA_FETCH_FAILED", "MEDIA_TOO_LARGE",
|
||||||
|
"MEDIA_TYPE_UNSUPPORTED", "INVALID_TASK_INPUT").contains(exception.getCode())) {
|
||||||
|
throw ProviderRequestException.notSent(exception.getCode(), exception.getMessage(), exception.getStatus());
|
||||||
|
}
|
||||||
|
throw ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", "上游任务提交结果尚未确认");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String,Object> submitConfigured(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
||||||
if (input == null) input = Map.of();
|
if (input == null) input = Map.of();
|
||||||
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
||||||
Provider provider = provider(providerId, routeId, upstreamModel);
|
Provider provider = provider(providerId, routeId, upstreamModel);
|
||||||
@@ -261,8 +278,8 @@ public class AsyncProviderClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private boolean isNotFound(ApiException exception) {
|
private boolean isNotFound(ApiException exception) {
|
||||||
String message = exception.getMessage();
|
return exception instanceof ProviderRequestException failure
|
||||||
return message != null && (message.contains("HTTP 404") || message.contains("HTTP 405"));
|
&& (Integer.valueOf(404).equals(failure.upstreamStatus()) || Integer.valueOf(405).equals(failure.upstreamStatus()));
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Normalize OpenAI image/audio response fields to the task result shape. */
|
/** Normalize OpenAI image/audio response fields to the task result shape. */
|
||||||
@@ -287,19 +304,14 @@ public class AsyncProviderClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private Map<String,Object> requestAudio(String method, String url, String key, Map<String,Object> body, String message) {
|
private Map<String,Object> requestAudio(String method, String url, String key, Map<String,Object> body, String message) {
|
||||||
|
boolean sending = false;
|
||||||
try {
|
try {
|
||||||
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
||||||
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
||||||
HttpResponse<byte[]> response = client.send(builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(), HttpResponse.BodyHandlers.ofByteArray());
|
HttpRequest request = builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build();
|
||||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
sending = true;
|
||||||
String code = switch (response.statusCode()) {
|
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
|
||||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
|
||||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
|
||||||
};
|
|
||||||
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
|
|
||||||
}
|
|
||||||
byte[] bytes = response.body();
|
byte[] bytes = response.body();
|
||||||
if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502);
|
if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502);
|
||||||
String contentType = response.headers().firstValue("Content-Type").orElse("audio/mpeg").split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT);
|
String contentType = response.headers().firstValue("Content-Type").orElse("audio/mpeg").split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT);
|
||||||
@@ -309,8 +321,7 @@ public class AsyncProviderClient {
|
|||||||
result.put("mimeType", contentType);
|
result.put("mimeType", contentType);
|
||||||
result.put("format", String.valueOf(body.getOrDefault("response_format", "mp3")));
|
result.put("format", String.valueOf(body.getOrDefault("response_format", "mp3")));
|
||||||
return result;
|
return result;
|
||||||
} catch (ApiException exception) { throw exception; }
|
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -322,6 +333,7 @@ public class AsyncProviderClient {
|
|||||||
*/
|
*/
|
||||||
private Map<String,Object> requestMultipart(String method, String url, String key, Map<String,Object> fields,
|
private Map<String,Object> requestMultipart(String method, String url, String key, Map<String,Object> fields,
|
||||||
String fileField, String media, String message) {
|
String fileField, String media, String message) {
|
||||||
|
boolean sending = false;
|
||||||
try {
|
try {
|
||||||
String boundary = "----moyi-" + java.util.UUID.randomUUID();
|
String boundary = "----moyi-" + java.util.UUID.randomUUID();
|
||||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||||
@@ -337,6 +349,7 @@ public class AsyncProviderClient {
|
|||||||
.header("Authorization", "Bearer " + key)
|
.header("Authorization", "Bearer " + key)
|
||||||
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
|
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
|
||||||
.POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build();
|
.POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build();
|
||||||
|
sending = true;
|
||||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||||
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||||
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
||||||
@@ -345,18 +358,19 @@ public class AsyncProviderClient {
|
|||||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
||||||
}
|
}
|
||||||
return parsed;
|
return parsed;
|
||||||
} catch (ApiException exception) { throw exception; }
|
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private ApiException upstreamStatus(int status, String message) {
|
private ApiException upstreamStatus(int status, String message) {
|
||||||
String code = switch (status) {
|
return ProviderRequestException.http(status, message);
|
||||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
}
|
||||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
|
||||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
private ProviderRequestException requestFailure(Exception exception, boolean sending, String message) {
|
||||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
if (exception instanceof InterruptedException) Thread.currentThread().interrupt();
|
||||||
};
|
if (exception instanceof ProviderRequestException failure) return failure;
|
||||||
return new ApiException(code, message + "(HTTP " + status + ")", 502);
|
if (sending) return ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", message);
|
||||||
|
if (exception instanceof ApiException error) return ProviderRequestException.notSent(error.getCode(), error.getMessage(), error.getStatus());
|
||||||
|
return ProviderRequestException.notSent("UPSTREAM_REQUEST_NOT_SENT", "请求准备失败,尚未提交上游生成", 502);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void multipartField(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException {
|
private void multipartField(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException {
|
||||||
@@ -414,21 +428,16 @@ public class AsyncProviderClient {
|
|||||||
private record MediaBytes(byte[] bytes, String contentType, String filename) {}
|
private record MediaBytes(byte[] bytes, String contentType, String filename) {}
|
||||||
|
|
||||||
private Map<String,Object> request(String method, String url, String key, Map<String,Object> body, String message) {
|
private Map<String,Object> request(String method, String url, String key, Map<String,Object> body, String message) {
|
||||||
|
boolean sending = false;
|
||||||
try {
|
try {
|
||||||
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
||||||
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
||||||
if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body)));
|
if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body)));
|
||||||
else builder.GET();
|
else builder.GET();
|
||||||
HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
|
HttpRequest request = builder.build();
|
||||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
sending = true;
|
||||||
String code = switch (response.statusCode()) {
|
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
|
||||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
|
||||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
|
||||||
};
|
|
||||||
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
|
|
||||||
}
|
|
||||||
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
||||||
Object code = parsed.get("code");
|
Object code = parsed.get("code");
|
||||||
if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) {
|
if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) {
|
||||||
@@ -437,8 +446,7 @@ public class AsyncProviderClient {
|
|||||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
||||||
}
|
}
|
||||||
return parsed;
|
return parsed;
|
||||||
} catch (ApiException exception) { throw exception; }
|
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Provider provider(String id) {
|
private Provider provider(String id) {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import com.moyi.controlplane.gateway.request.ApiRequestProtocol;
|
|||||||
import com.moyi.controlplane.gateway.request.ApiRequestRiskEngine;
|
import com.moyi.controlplane.gateway.request.ApiRequestRiskEngine;
|
||||||
import com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy;
|
import com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy;
|
||||||
import com.moyi.controlplane.usage.UsageAuditService;
|
import com.moyi.controlplane.usage.UsageAuditService;
|
||||||
|
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
@@ -35,6 +36,7 @@ import org.springframework.transaction.annotation.Transactional;
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class AsyncTaskService {
|
public class AsyncTaskService {
|
||||||
|
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(AsyncTaskService.class);
|
||||||
private final JdbcTemplate jdbc;
|
private final JdbcTemplate jdbc;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final CatalogService catalogService;
|
private final CatalogService catalogService;
|
||||||
@@ -49,6 +51,7 @@ public class AsyncTaskService {
|
|||||||
private final TransactionTemplate transactions;
|
private final TransactionTemplate transactions;
|
||||||
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
|
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
|
||||||
private final ApiRequestRiskPolicy requestRisk;
|
private final ApiRequestRiskPolicy requestRisk;
|
||||||
|
private final TaskOutcomeJournal outcomes;
|
||||||
|
|
||||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||||
@@ -104,6 +107,7 @@ public class AsyncTaskService {
|
|||||||
this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager);
|
this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager);
|
||||||
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
|
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
|
||||||
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
|
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
|
||||||
|
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
@@ -177,7 +181,7 @@ public class AsyncTaskService {
|
|||||||
// model. Without the transaction advisory lock, concurrent creates
|
// model. Without the transaction advisory lock, concurrent creates
|
||||||
// could all observe the same count and exceed maxQueuedPerModel.
|
// could all observe the same count and exceed maxQueuedPerModel.
|
||||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId);
|
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId);
|
||||||
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId);
|
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running', 'reconciling')", Integer.class, modelId);
|
||||||
if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429);
|
if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429);
|
||||||
@SuppressWarnings("unchecked") Map<String,Object> frozenPricing=(Map<String,Object>) model.get("pricing");
|
@SuppressWarnings("unchecked") Map<String,Object> frozenPricing=(Map<String,Object>) model.get("pricing");
|
||||||
if(AsyncPriceSchedule.tokens(frozenPricing) && !"image".equals(taskType)) throw new ApiException("ASYNC_BILLING_UNSUPPORTED","该媒体类型的 Token 用量结算尚未接入",503);
|
if(AsyncPriceSchedule.tokens(frozenPricing) && !"image".equals(taskType)) throw new ApiException("ASYNC_BILLING_UNSUPPORTED","该媒体类型的 Token 用量结算尚未接入",503);
|
||||||
@@ -237,6 +241,11 @@ public class AsyncTaskService {
|
|||||||
|
|
||||||
@Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}")
|
@Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}")
|
||||||
public void completeMockTasks() {
|
public void completeMockTasks() {
|
||||||
|
// Retry local settlement from durable observations, never the upstream generation request.
|
||||||
|
for (var completion : outcomes.pendingCompletions()) {
|
||||||
|
try { completeObserved(completion); }
|
||||||
|
catch (RuntimeException exception) { log.warn("Async task {} settlement remains pending", completion.taskId()); }
|
||||||
|
}
|
||||||
List<String> taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id"));
|
List<String> taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id"));
|
||||||
jdbc.update(
|
jdbc.update(
|
||||||
"UPDATE model_tasks SET status = 'succeeded', progress = 100, result = CAST(CASE WHEN task_type = '3d' THEN ? ELSE ? END AS jsonb), updated_at = ?, completed_at = ? WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'",
|
"UPDATE model_tasks SET status = 'succeeded', progress = 100, result = CAST(CASE WHEN task_type = '3d' THEN ? ELSE ? END AS jsonb), updated_at = ?, completed_at = ? WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'",
|
||||||
@@ -244,7 +253,7 @@ public class AsyncTaskService {
|
|||||||
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
|
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
|
||||||
);
|
);
|
||||||
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
||||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||||
for (Map<String,Object> row : pending) pollProviderTask(row);
|
for (Map<String,Object> row : pending) pollProviderTask(row);
|
||||||
dispatchQueuedTasks();
|
dispatchQueuedTasks();
|
||||||
}
|
}
|
||||||
@@ -257,36 +266,43 @@ public class AsyncTaskService {
|
|||||||
*/
|
*/
|
||||||
private void dispatchQueuedTasks() {
|
private void dispatchQueuedTasks() {
|
||||||
if (providerClient == null) return;
|
if (providerClient == null) return;
|
||||||
// A process can die after claiming a task but before receiving an
|
// A lost submit response is ambiguous, not a failed generation.
|
||||||
// upstream task id. Fail those claims after a grace period. Retrying
|
// Quarantine stale claims without retrying or releasing financial holds.
|
||||||
// them blindly could submit the same generation twice when the
|
|
||||||
// upstream accepted the request just before the process died.
|
|
||||||
recoverInterruptedClaims();
|
recoverInterruptedClaims();
|
||||||
for (int i = 0; i < 50; i++) {
|
for (int i = 0; i < 50; i++) {
|
||||||
Map<String,Object> task = claimQueuedTask();
|
Map<String,Object> task = claimQueuedTask();
|
||||||
if (task == null) return;
|
if (task == null) return;
|
||||||
String taskId = String.valueOf(task.get("id"));
|
String taskId = String.valueOf(task.get("id"));
|
||||||
usage.updateTask(taskId, "running", null, null, null);
|
Map<String,Object> upstream;
|
||||||
try {
|
try {
|
||||||
Map<String,Object> input = taskInput(task);
|
Map<String,Object> input = taskInput(task);
|
||||||
Map<String,Object> upstream = submitProviderTask(task, input);
|
upstream = submitProviderTask(task, input);
|
||||||
// Some OpenAI-compatible image/audio endpoints return the
|
} catch (RuntimeException exception) {
|
||||||
// generated media in the submit response rather than creating
|
if (exception instanceof ProviderRequestException failure && failure.definitelyNotAccepted()) {
|
||||||
// a pollable job. Persist that result and settle the task in
|
completeObserved(outcomes.observeTerminal(taskId, "failed", null, failure.getCode(), failure.getMessage()));
|
||||||
// the same worker transaction.
|
} else {
|
||||||
|
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_ACCEPTANCE_UNKNOWN", "上游任务是否已受理尚未确认;冻结额度保留,请联系管理员核账,勿重复提交", false);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
TaskOutcomeJournal.Completion completion = null;
|
||||||
|
try {
|
||||||
|
// Journal a received result before touching the ledger. A failed settlement
|
||||||
|
// must only retry locally, never generate again or release its reservation.
|
||||||
if (upstream.get("_moyiInlineResult") instanceof Map<?,?> inline) {
|
if (upstream.get("_moyiInlineResult") instanceof Map<?,?> inline) {
|
||||||
Map<String,Object> result = new LinkedHashMap<>();
|
Map<String,Object> result = new LinkedHashMap<>();
|
||||||
inline.forEach((key, value) -> result.put(String.valueOf(key), value));
|
inline.forEach((key, value) -> result.put(String.valueOf(key), value));
|
||||||
finishTask(taskId, "succeeded", 100, result, null, null);
|
completion = outcomes.observeTerminal(taskId, "succeeded", result, null, null);
|
||||||
continue;
|
} else {
|
||||||
}
|
|
||||||
OffsetDateTime submittedAt = now();
|
OffsetDateTime submittedAt = now();
|
||||||
jdbc.update("UPDATE model_tasks SET provider_task_id=?, provider_key_fingerprint=?, provider_deadline_at=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"), submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)), submittedAt, taskId);
|
outcomes.accepted(taskId, providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"),
|
||||||
} catch (RuntimeException exception) {
|
submittedAt, submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)));
|
||||||
String code = exception instanceof ApiException error ? error.getCode() : "UPSTREAM_TASK_SUBMIT_FAILED";
|
|
||||||
String message = exception instanceof ApiException error ? error.getMessage() : "上游异步任务创建失败";
|
|
||||||
failTask(taskId, code, message);
|
|
||||||
}
|
}
|
||||||
|
} catch (RuntimeException exception) {
|
||||||
|
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_RECEIPT_UNKNOWN", "已调用上游,但任务回执无法确认或保存;冻结额度保留,等待核账,勿重复提交", false);
|
||||||
|
}
|
||||||
|
// Deliberately outside the provider exception handler.
|
||||||
|
completeObserved(completion);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -306,18 +322,18 @@ public class AsyncTaskService {
|
|||||||
AND t.provider_task_id IS NULL
|
AND t.provider_task_id IS NULL
|
||||||
AND m.status='active' AND m.deleted_at IS NULL
|
AND m.status='active' AND m.deleted_at IS NULL
|
||||||
AND p.status='active' AND p.deleted_at IS NULL
|
AND p.status='active' AND p.deleted_at IS NULL
|
||||||
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status='running') < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
|
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status IN ('running','reconciling')) < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
|
||||||
AND (
|
AND (
|
||||||
? <= 0
|
? <= 0
|
||||||
OR COALESCE(m.max_concurrency, ?) <= 0
|
OR COALESCE(m.max_concurrency, ?) <= 0
|
||||||
OR (SELECT COUNT(*) FROM model_tasks running_model
|
OR (SELECT COUNT(*) FROM model_tasks running_model
|
||||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
WHERE running_model.model_id=t.model_id AND running_model.status IN ('running','reconciling'))
|
||||||
< COALESCE(m.max_concurrency, ?)
|
< COALESCE(m.max_concurrency, ?)
|
||||||
)
|
)
|
||||||
AND (
|
AND (
|
||||||
? <= 0
|
? <= 0
|
||||||
OR (SELECT COUNT(*) FROM model_tasks running_key
|
OR (SELECT COUNT(*) FROM model_tasks running_key
|
||||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
WHERE running_key.api_key_id=t.api_key_id AND running_key.status IN ('running','reconciling'))
|
||||||
< ?
|
< ?
|
||||||
)
|
)
|
||||||
ORDER BY t.created_at
|
ORDER BY t.created_at
|
||||||
@@ -332,19 +348,26 @@ public class AsyncTaskService {
|
|||||||
if(!gatewayGuard.tryTaskUserSlot((UUID)row.get("api_key_id"),row.get("id").toString(),providerTimeoutMs+180000)) return null;
|
if(!gatewayGuard.tryTaskUserSlot((UUID)row.get("api_key_id"),row.get("id").toString(),providerTimeoutMs+180000)) return null;
|
||||||
try {jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));}
|
try {jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));}
|
||||||
catch(RuntimeException e) {gatewayGuard.releaseTaskUserSlot((UUID)row.get("user_id"),row.get("id").toString());throw e;}
|
catch(RuntimeException e) {gatewayGuard.releaseTaskUserSlot((UUID)row.get("user_id"),row.get("id").toString());throw e;}
|
||||||
|
usage.updateTask(row.get("id").toString(), "running", null, null, null);
|
||||||
return row;
|
return row;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private void recoverInterruptedClaims() {
|
private void recoverInterruptedClaims() {
|
||||||
List<String> taskIds = inTransaction(() -> jdbc.queryForList(
|
List<String> taskIds = inTransaction(() -> jdbc.queryForList(
|
||||||
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
|
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND completion_status IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
|
||||||
String.class));
|
String.class));
|
||||||
for (String taskId : taskIds) {
|
for (String taskId : taskIds) {
|
||||||
failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交");
|
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游提交回执丢失或进程中断;冻结额度保留,等待核账,勿重复提交", true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void completeObserved(TaskOutcomeJournal.Completion completion) {
|
||||||
|
if (completion == null) return;
|
||||||
|
finishTask(completion.taskId(), completion.status(), "succeeded".equals(completion.status()) ? 100 : 0,
|
||||||
|
completion.result(), completion.code(), completion.message());
|
||||||
|
}
|
||||||
|
|
||||||
private void failTask(String taskId, String code, String message) {
|
private void failTask(String taskId, String code, String message) {
|
||||||
finishTask(taskId, "failed", 0, null, code, message);
|
finishTask(taskId, "failed", 0, null, code, message);
|
||||||
}
|
}
|
||||||
@@ -354,8 +377,10 @@ public class AsyncTaskService {
|
|||||||
// crash after marking failed/succeeded leaves a hold that polling never
|
// crash after marking failed/succeeded leaves a hold that polling never
|
||||||
// revisits. The conditional update also prevents duplicate settlement.
|
// revisits. The conditional update also prevents duplicate settlement.
|
||||||
inTransaction(() -> {
|
inTransaction(() -> {
|
||||||
Map<String,Object> task=jdbc.queryForMap("SELECT status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
|
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
|
||||||
if(!List.of("queued","running").contains(task.get("status"))) return null;
|
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
|
||||||
|
// A late failure must not override a durably observed success whose settlement is pending.
|
||||||
|
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
|
||||||
AsyncPriceSchedule.Bill bill=null;
|
AsyncPriceSchedule.Bill bill=null;
|
||||||
String finalCode=code,finalMessage=message;
|
String finalCode=code,finalMessage=message;
|
||||||
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
|
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
|
||||||
@@ -363,7 +388,7 @@ public class AsyncTaskService {
|
|||||||
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
|
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
|
||||||
}
|
}
|
||||||
int changed = jdbc.update(
|
int changed = jdbc.update(
|
||||||
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
|
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
|
||||||
status, progress, result == null ? null : writeJson(result),
|
status, progress, result == null ? null : writeJson(result),
|
||||||
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
|
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
|
||||||
if (changed == 0) return null;
|
if (changed == 0) return null;
|
||||||
@@ -397,6 +422,7 @@ public class AsyncTaskService {
|
|||||||
task.put("progress", resultSet.getInt("progress"));
|
task.put("progress", resultSet.getInt("progress"));
|
||||||
task.put("createdAt", resultSet.getObject("created_at", OffsetDateTime.class));
|
task.put("createdAt", resultSet.getObject("created_at", OffsetDateTime.class));
|
||||||
task.put("updatedAt", resultSet.getObject("updated_at", OffsetDateTime.class));
|
task.put("updatedAt", resultSet.getObject("updated_at", OffsetDateTime.class));
|
||||||
|
task.put("requiresReconciliation", "reconciling".equals(resultSet.getString("status")));
|
||||||
putIfPresent(task, "costAmount", resultSet, "cost_amount");
|
putIfPresent(task, "costAmount", resultSet, "cost_amount");
|
||||||
String result = resultSet.getString("result");
|
String result = resultSet.getString("result");
|
||||||
if (result != null) task.put("result", readMap(result));
|
if (result != null) task.put("result", readMap(result));
|
||||||
@@ -440,9 +466,12 @@ public class AsyncTaskService {
|
|||||||
|
|
||||||
private String providerTaskId(Map<String,Object> response) {
|
private String providerTaskId(Map<String,Object> response) {
|
||||||
Object direct = response.get("id");
|
Object direct = response.get("id");
|
||||||
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id");
|
if (direct == null) direct = response.get("task_id");
|
||||||
if (direct == null) throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回异步任务编号", 502);
|
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id") == null ? data.get("id") : data.get("task_id");
|
||||||
return String.valueOf(direct);
|
if (!(direct instanceof String value) || value.isBlank() || value.length() > 160 || value.chars().anyMatch(Character::isISOControl)) {
|
||||||
|
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回有效的异步任务编号", 502);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void pollProviderTask(Map<String,Object> row) {
|
private void pollProviderTask(Map<String,Object> row) {
|
||||||
@@ -456,7 +485,7 @@ public class AsyncTaskService {
|
|||||||
deadline = createdAt.plus(java.time.Duration.ofMillis(providerTimeoutMs));
|
deadline = createdAt.plus(java.time.Duration.ofMillis(providerTimeoutMs));
|
||||||
}
|
}
|
||||||
if (!now().isBefore(deadline)) {
|
if (!now().isBefore(deadline)) {
|
||||||
failTask(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;平台冻结额度已释放。上游可能仍在执行,请先核实后再提交");
|
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;冻结额度保留,等待管理员核账,勿重复提交", false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Map<String,Object> response;
|
Map<String,Object> response;
|
||||||
@@ -464,7 +493,7 @@ public class AsyncTaskService {
|
|||||||
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("model_id")), String.valueOf(row.get("upstream_model")), String.valueOf(row.get("provider_task_id")), row.get("provider_key_fingerprint") == null ? null : String.valueOf(row.get("provider_key_fingerprint")));
|
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("model_id")), String.valueOf(row.get("upstream_model")), String.valueOf(row.get("provider_task_id")), row.get("provider_key_fingerprint") == null ? null : String.valueOf(row.get("provider_key_fingerprint")));
|
||||||
} catch (RuntimeException exception) {
|
} catch (RuntimeException exception) {
|
||||||
if (exception instanceof ApiException error && "UPSTREAM_KEY_NOT_AVAILABLE".equals(error.getCode())) {
|
if (exception instanceof ApiException error && "UPSTREAM_KEY_NOT_AVAILABLE".equals(error.getCode())) {
|
||||||
failTask(taskId, error.getCode(), error.getMessage());
|
outcomes.requireReconciliation(taskId, error.getCode(), "提交任务使用的上游凭据已不可用,执行结果未知;冻结额度保留,等待核账", false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Even a permanent authentication/configuration error only proves
|
// Even a permanent authentication/configuration error only proves
|
||||||
@@ -487,9 +516,9 @@ public class AsyncTaskService {
|
|||||||
if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) {
|
if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) {
|
||||||
// Persistence failures must roll back and remain pollable, rather
|
// Persistence failures must roll back and remain pollable, rather
|
||||||
// than being mistaken for upstream failures and refunded.
|
// than being mistaken for upstream failures and refunded.
|
||||||
finishTask(taskId, status, progress, result,
|
completeObserved(outcomes.observeTerminal(taskId, status, result,
|
||||||
"failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null,
|
"failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null,
|
||||||
"failed".equals(status) ? "上游异步任务失败" : null);
|
"failed".equals(status) ? "上游异步任务失败" : null));
|
||||||
} else {
|
} else {
|
||||||
updatePollingState(taskId, progress, null, null);
|
updatePollingState(taskId, progress, null, null);
|
||||||
}
|
}
|
||||||
@@ -497,7 +526,7 @@ public class AsyncTaskService {
|
|||||||
|
|
||||||
private void updatePollingState(String taskId, Integer progress, String code, String message) {
|
private void updatePollingState(String taskId, Integer progress, String code, String message) {
|
||||||
inTransaction(() -> {
|
inTransaction(() -> {
|
||||||
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running')",
|
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL",
|
||||||
progress, code == null ? null : writeJson(Map.of("code", code, "message", message, "retryable", true)), now(), taskId);
|
progress, code == null ? null : writeJson(Map.of("code", code, "message", message, "retryable", true)), now(), taskId);
|
||||||
if (changed > 0) usage.updateTask(taskId, "running", null, code, message);
|
if (changed > 0) usage.updateTask(taskId, "running", null, code, message);
|
||||||
return null;
|
return null;
|
||||||
@@ -515,10 +544,12 @@ public class AsyncTaskService {
|
|||||||
Object raw = response.get("status");
|
Object raw = response.get("status");
|
||||||
if (raw == null && response.get("data") instanceof Map<?,?> data) raw = data.get("status");
|
if (raw == null && response.get("data") instanceof Map<?,?> data) raw = data.get("status");
|
||||||
String value = String.valueOf(raw == null ? "running" : raw).toLowerCase(java.util.Locale.ROOT);
|
String value = String.valueOf(raw == null ? "running" : raw).toLowerCase(java.util.Locale.ROOT);
|
||||||
if (value.contains("success") || value.equals("completed")) return "succeeded";
|
return switch (value) {
|
||||||
if (value.contains("fail") || value.equals("error") || value.equals("banned")) return "failed";
|
case "success", "succeeded", "completed" -> "succeeded";
|
||||||
if (value.contains("cancel")) return "cancelled";
|
case "failed", "failure", "error", "banned" -> "failed";
|
||||||
return "running";
|
case "cancelled", "canceled" -> "cancelled";
|
||||||
|
default -> "running";
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private int progress(Map<String,Object> response, String status) {
|
private int progress(Map<String,Object> response, String status) {
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package com.moyi.controlplane.task;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.moyi.controlplane.usage.UsageAuditService;
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
|
||||||
|
/** Durable provider observations only. No upstream calls, wallet mutation, retry or refund policy. */
|
||||||
|
final class TaskOutcomeJournal {
|
||||||
|
record Completion(String taskId, String status, Map<String,Object> result, String code, String message) { }
|
||||||
|
|
||||||
|
private final JdbcTemplate jdbc;
|
||||||
|
private final ObjectMapper mapper;
|
||||||
|
private final TransactionTemplate transactions;
|
||||||
|
private final UsageAuditService usage;
|
||||||
|
|
||||||
|
TaskOutcomeJournal(JdbcTemplate jdbc, ObjectMapper mapper, TransactionTemplate transactions, UsageAuditService usage) {
|
||||||
|
this.jdbc = jdbc;
|
||||||
|
this.mapper = mapper;
|
||||||
|
this.transactions = transactions;
|
||||||
|
this.usage = usage;
|
||||||
|
}
|
||||||
|
|
||||||
|
Completion observeTerminal(String taskId, String status, Map<String,Object> result, String code, String message) {
|
||||||
|
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal outcome required");
|
||||||
|
return transaction(() -> {
|
||||||
|
var row = jdbc.queryForMap("SELECT status,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||||
|
if (!List.of("queued", "running", "reconciling").contains(row.get("status"))) return null;
|
||||||
|
if (row.get("completion_status") != null) return completion(taskId, row);
|
||||||
|
jdbc.update("UPDATE model_tasks SET completion_status=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),updated_at=now() WHERE id=?",
|
||||||
|
status, json(result), code == null ? null : json(Map.of("code", code, "message", message)), taskId);
|
||||||
|
return new Completion(taskId, status, result, code, message);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Completion> pendingCompletions() {
|
||||||
|
return jdbc.queryForList("SELECT id,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE status IN ('running','reconciling') AND completion_status IS NOT NULL ORDER BY updated_at LIMIT 50")
|
||||||
|
.stream().map(row -> completion(String.valueOf(row.get("id")), row)).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
void accepted(String taskId, String providerTaskId, Object keyFingerprint, OffsetDateTime submittedAt, OffsetDateTime deadline) {
|
||||||
|
transaction(() -> {
|
||||||
|
int changed = jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=? WHERE id=? AND status IN ('running','reconciling') AND provider_task_id IS NULL AND completion_status IS NULL",
|
||||||
|
providerTaskId, keyFingerprint, deadline, submittedAt, taskId);
|
||||||
|
if (changed > 0) usage.updateTask(taskId, "running", null, null, null);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
void requireReconciliation(String taskId, String code, String message, boolean interruptedClaim) {
|
||||||
|
transaction(() -> {
|
||||||
|
String claimGuard = interruptedClaim
|
||||||
|
? " AND provider_task_id IS NULL AND updated_at < now()-interval '2 minutes'" : "";
|
||||||
|
int changed = jdbc.update("UPDATE model_tasks SET status='reconciling',error=CAST(? AS jsonb),reconciliation_required_at=now(),updated_at=now() WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL" + claimGuard,
|
||||||
|
json(Map.of("code", code, "message", message, "retryable", false, "requiresReconciliation", true)), taskId);
|
||||||
|
if (changed > 0) {
|
||||||
|
usage.updateTask(taskId, "reconciling", null, code, message);
|
||||||
|
jdbc.update("UPDATE usage_events SET cost_status='pending',cost_amount=NULL WHERE provider_task_id=? AND event_kind='task'", taskId);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private Completion completion(String taskId, Map<String,Object> row) {
|
||||||
|
Map<String,Object> error = map(row.get("error"));
|
||||||
|
return new Completion(taskId, String.valueOf(row.get("completion_status")), map(row.get("result")),
|
||||||
|
error == null ? null : (String) error.get("code"), error == null ? null : (String) error.get("message"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String,Object> map(Object value) {
|
||||||
|
if (value == null) return null;
|
||||||
|
try { return mapper.readValue(String.valueOf(value), new TypeReference<>() {}); }
|
||||||
|
catch (JsonProcessingException exception) { throw new IllegalStateException("Invalid persisted provider observation"); }
|
||||||
|
}
|
||||||
|
|
||||||
|
private String json(Object value) {
|
||||||
|
if (value == null) return null;
|
||||||
|
try { return mapper.writeValueAsString(value); }
|
||||||
|
catch (JsonProcessingException exception) { throw new IllegalStateException("Provider observation cannot be encoded"); }
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T> T transaction(Supplier<T> work) {
|
||||||
|
return transactions == null ? work.get() : transactions.execute(status -> work.get());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -157,15 +157,17 @@ public class UsageAuditService {
|
|||||||
WITH pending AS (
|
WITH pending AS (
|
||||||
SELECT model_id,
|
SELECT model_id,
|
||||||
COUNT(*) FILTER (WHERE status='running') AS running,
|
COUNT(*) FILTER (WHERE status='running') AS running,
|
||||||
COUNT(*) FILTER (WHERE status='queued') AS queued
|
COUNT(*) FILTER (WHERE status='queued') AS queued,
|
||||||
FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id
|
COUNT(*) FILTER (WHERE status='reconciling') AS reconciling
|
||||||
|
FROM model_tasks WHERE status IN ('queued','running','reconciling') GROUP BY model_id
|
||||||
)
|
)
|
||||||
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
|
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
|
||||||
CASE WHEN m.max_concurrency = 0 THEN 0
|
CASE WHEN m.max_concurrency = 0 THEN 0
|
||||||
ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency,
|
ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency,
|
||||||
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
|
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
|
||||||
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
|
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
|
||||||
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued
|
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued,
|
||||||
|
COALESCE(pending.reconciling, 0) AS reconciling
|
||||||
FROM model_catalog m JOIN model_providers p ON p.id=m.provider_id
|
FROM model_catalog m JOIN model_providers p ON p.id=m.provider_id
|
||||||
JOIN model_definitions d ON d.id=m.logical_model_id
|
JOIN model_definitions d ON d.id=m.logical_model_id
|
||||||
LEFT JOIN pending ON pending.model_id=m.id
|
LEFT JOIN pending ON pending.model_id=m.id
|
||||||
@@ -182,6 +184,7 @@ public class UsageAuditService {
|
|||||||
group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0);
|
group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0);
|
||||||
group.put("running", rs.getLong("running"));
|
group.put("running", rs.getLong("running"));
|
||||||
group.put("queued", rs.getLong("queued"));
|
group.put("queued", rs.getLong("queued"));
|
||||||
|
group.put("reconciling", rs.getLong("reconciling"));
|
||||||
return group;
|
return group;
|
||||||
}, defaultModelLimit);
|
}, defaultModelLimit);
|
||||||
List<String> synchronousIds = groups.stream().filter(group -> !"async".equals(group.get("mode"))).map(group -> group.get("id").toString()).toList();
|
List<String> synchronousIds = groups.stream().filter(group -> !"async".equals(group.get("mode"))).map(group -> group.get("id").toString()).toList();
|
||||||
@@ -193,12 +196,13 @@ public class UsageAuditService {
|
|||||||
group.put("running", current == null ? null : current.running());
|
group.put("running", current == null ? null : current.running());
|
||||||
group.put("queued", current == null ? null : current.queued());
|
group.put("queued", current == null ? null : current.queued());
|
||||||
}
|
}
|
||||||
// Keep disabled groups visible while their queued or running work still exists.
|
// Keep disabled groups visible while any work or unresolved financial hold still exists.
|
||||||
groups = groups.stream().filter(group -> Boolean.TRUE.equals(group.get("routeEnabled"))
|
groups = groups.stream().filter(group -> Boolean.TRUE.equals(group.get("routeEnabled"))
|
||||||
|| positive(group.get("running")) || positive(group.get("queued"))).toList();
|
|| positive(group.get("running")) || positive(group.get("queued")) || positive(group.get("reconciling"))).toList();
|
||||||
result.put("capacityGroups", groups);
|
result.put("capacityGroups", groups);
|
||||||
result.put("running", totalCapacity(groups, "running"));
|
result.put("running", totalCapacity(groups, "running"));
|
||||||
result.put("waiting", totalCapacity(groups, "queued"));
|
result.put("waiting", totalCapacity(groups, "queued"));
|
||||||
|
result.put("reconciling", totalCapacity(groups, "reconciling"));
|
||||||
result.put("capacityConfig", Map.of(
|
result.put("capacityConfig", Map.of(
|
||||||
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
|
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
|
||||||
"defaultModelConcurrency", defaultModelLimit,
|
"defaultModelConcurrency", defaultModelLimit,
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
-- Unknown provider acceptance is not a failed generation and must retain its financial holds.
|
||||||
|
ALTER TABLE model_tasks DROP CONSTRAINT model_tasks_status_check;
|
||||||
|
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_status_check
|
||||||
|
CHECK (status IN ('queued','running','reconciling','succeeded','failed','cancelled'));
|
||||||
|
ALTER TABLE usage_events DROP CONSTRAINT usage_events_status_check;
|
||||||
|
ALTER TABLE usage_events ADD CONSTRAINT usage_events_status_check
|
||||||
|
CHECK (status IN ('queued','running','reconciling','succeeded','failed','cancelled'));
|
||||||
|
ALTER TABLE model_tasks ADD COLUMN reconciliation_required_at timestamptz;
|
||||||
|
-- A normalized observed terminal outcome survives a crash or transient ledger/audit write failure.
|
||||||
|
-- The result itself uses the existing result column; no second copy of generated media is stored.
|
||||||
|
ALTER TABLE model_tasks ADD COLUMN completion_status varchar(16)
|
||||||
|
CHECK (completion_status IN ('succeeded','failed','cancelled'));
|
||||||
|
CREATE INDEX model_tasks_pending_completion_idx ON model_tasks(updated_at)
|
||||||
|
WHERE completion_status IS NOT NULL AND status IN ('running','reconciling');
|
||||||
|
CREATE INDEX model_tasks_reconciliation_idx ON model_tasks(reconciliation_required_at)
|
||||||
|
WHERE status='reconciling';
|
||||||
+68
-1
@@ -11,6 +11,7 @@ import static org.mockito.Mockito.when;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.moyi.controlplane.common.ApiException;
|
import com.moyi.controlplane.common.ApiException;
|
||||||
import com.moyi.controlplane.security.SecretCipher;
|
import com.moyi.controlplane.security.SecretCipher;
|
||||||
|
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||||
import com.sun.net.httpserver.HttpServer;
|
import com.sun.net.httpserver.HttpServer;
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
@@ -207,7 +208,7 @@ class AsyncProviderClientTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@ParameterizedTest
|
||||||
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "503,UPSTREAM_PROVIDER_ERROR"})
|
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "408,UPSTREAM_PROVIDER_ERROR", "409,UPSTREAM_PROVIDER_ERROR", "503,UPSTREAM_PROVIDER_ERROR"})
|
||||||
void httpErrorsExposeOnlySafeCodes(int status, String code) throws Exception {
|
void httpErrorsExposeOnlySafeCodes(int status, String code) throws Exception {
|
||||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
server.createContext("/", exchange -> {
|
server.createContext("/", exchange -> {
|
||||||
@@ -221,6 +222,7 @@ class AsyncProviderClientTest {
|
|||||||
ApiException submit = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
|
ApiException submit = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
|
||||||
assertEquals(code, submit.getCode());
|
assertEquals(code, submit.getCode());
|
||||||
assertEquals("上游 3D 任务创建失败(HTTP " + status + ")", submit.getMessage());
|
assertEquals("上游 3D 任务创建失败(HTTP " + status + ")", submit.getMessage());
|
||||||
|
assertEquals(status != 408 && status != 409 && status < 500, ((ProviderRequestException) submit).definitelyNotAccepted());
|
||||||
ApiException poll = assertThrows(ApiException.class, () -> client.poll("provider-vast", "task-test"));
|
ApiException poll = assertThrows(ApiException.class, () -> client.poll("provider-vast", "task-test"));
|
||||||
assertEquals(code, poll.getCode());
|
assertEquals(code, poll.getCode());
|
||||||
assertEquals("上游异步任务查询失败(HTTP " + status + ")", poll.getMessage());
|
assertEquals("上游异步任务查询失败(HTTP " + status + ")", poll.getMessage());
|
||||||
@@ -229,6 +231,71 @@ class AsyncProviderClientTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test void successfulHttpResponseWithMalformedBodyHasUnknownAcceptance() throws Exception {
|
||||||
|
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
|
server.createContext("/", exchange -> {
|
||||||
|
byte[] body = "not-json; private upstream detail".getBytes(StandardCharsets.UTF_8);
|
||||||
|
exchange.sendResponseHeaders(200, body.length);
|
||||||
|
try (var output = exchange.getResponseBody()) { output.write(body); }
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
try {
|
||||||
|
var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort());
|
||||||
|
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair"));
|
||||||
|
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
|
||||||
|
org.junit.jupiter.api.Assertions.assertFalse(failure.getMessage().contains("private upstream detail"));
|
||||||
|
} finally { server.stop(0); }
|
||||||
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@CsvSource({"image,gpt-image-2", "video,sora-2", "audio,tts-1"})
|
||||||
|
void compatibleSubmit5xxRemainsUnknownAndNeverUsesFallbackRoute(String type, String model) throws Exception {
|
||||||
|
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
|
var calls = new java.util.concurrent.atomic.AtomicInteger();
|
||||||
|
server.createContext("/", exchange -> {
|
||||||
|
calls.incrementAndGet();
|
||||||
|
exchange.getRequestBody().readAllBytes();
|
||||||
|
byte[] body = "private account details".getBytes(StandardCharsets.UTF_8);
|
||||||
|
exchange.sendResponseHeaders(503, body.length);
|
||||||
|
try (var output = exchange.getResponseBody()) { output.write(body); }
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
try {
|
||||||
|
var jdbc = mock(JdbcTemplate.class);
|
||||||
|
var cipher = new SecretCipher("test-encryption");
|
||||||
|
when(jdbc.queryForMap(any(String.class), eq("relay"))).thenReturn(Map.of(
|
||||||
|
"code", "relay", "protocol", "openai-compatible", "base_url", "http://127.0.0.1:" + server.getAddress().getPort(), "credential_ciphertext", cipher.encrypt("test-key")));
|
||||||
|
var client = new AsyncProviderClient(jdbc, cipher, new ObjectMapper(), mock(Environment.class));
|
||||||
|
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("relay", model, type, "chair"));
|
||||||
|
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
|
||||||
|
assertEquals(1, calls.get());
|
||||||
|
} finally { server.stop(0); }
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void acceptedPostWithDroppedResponseIsUnknownAndIsNotTransparentlyRepeated() throws Exception {
|
||||||
|
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
|
var calls = new java.util.concurrent.atomic.AtomicInteger();
|
||||||
|
server.createContext("/", exchange -> {
|
||||||
|
exchange.getRequestBody().readAllBytes();
|
||||||
|
calls.incrementAndGet();
|
||||||
|
exchange.close(); // Simulate acceptance followed by loss of the HTTP response.
|
||||||
|
});
|
||||||
|
server.start();
|
||||||
|
try {
|
||||||
|
var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort());
|
||||||
|
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair"));
|
||||||
|
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
|
||||||
|
assertEquals(1, calls.get());
|
||||||
|
} finally { server.stop(0); }
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
|
||||||
|
var client = clientFor("http://127.0.0.1:1");
|
||||||
|
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
|
||||||
|
assertEquals(ProviderRequestException.Acceptance.NOT_SENT, failure.acceptance());
|
||||||
|
assertEquals("INVALID_MEDIA_URL", failure.getCode());
|
||||||
|
}
|
||||||
|
|
||||||
private AsyncProviderClient clientFor(String baseUrl) {
|
private AsyncProviderClient clientFor(String baseUrl) {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||||
SecretCipher cipher = new SecretCipher("test-encryption");
|
SecretCipher cipher = new SecretCipher("test-encryption");
|
||||||
|
|||||||
+159
-6
@@ -15,6 +15,7 @@ import java.util.concurrent.CountDownLatch;
|
|||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
|
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
|
||||||
|
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||||
import org.flywaydb.core.Flyway;
|
import org.flywaydb.core.Flyway;
|
||||||
import org.junit.jupiter.api.AfterEach;
|
import org.junit.jupiter.api.AfterEach;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
@@ -42,6 +43,7 @@ class AsyncTaskPostgresTest {
|
|||||||
private AsyncProviderClient provider;
|
private AsyncProviderClient provider;
|
||||||
private AsyncTaskService tasks;
|
private AsyncTaskService tasks;
|
||||||
private CatalogService catalog;
|
private CatalogService catalog;
|
||||||
|
private GatewayGuard guard;
|
||||||
|
|
||||||
@BeforeEach void setup() {
|
@BeforeEach void setup() {
|
||||||
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
|
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
|
||||||
@@ -52,7 +54,7 @@ class AsyncTaskPostgresTest {
|
|||||||
jdbc = new JdbcTemplate(source);
|
jdbc = new JdbcTemplate(source);
|
||||||
usage = spy(new UsageAuditService(jdbc, json));
|
usage = spy(new UsageAuditService(jdbc, json));
|
||||||
provider = mock(AsyncProviderClient.class);
|
provider = mock(AsyncProviderClient.class);
|
||||||
GatewayGuard guard=mock(GatewayGuard.class);
|
guard=mock(GatewayGuard.class);
|
||||||
when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true);
|
when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true);
|
||||||
catalog = mock(CatalogService.class);
|
catalog = mock(CatalogService.class);
|
||||||
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000,
|
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000,
|
||||||
@@ -152,7 +154,10 @@ class AsyncTaskPostgresTest {
|
|||||||
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
|
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
|
||||||
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
|
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
|
||||||
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId);
|
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId);
|
||||||
when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(new ApiException(code,"上游任务未接受",503));
|
when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(
|
||||||
|
"PROVIDER_NOT_CONFIGURED".equals(code)
|
||||||
|
? ProviderRequestException.notSent(code, "上游任务未接受", 503)
|
||||||
|
: ProviderRequestException.http(401, "上游任务未接受"));
|
||||||
tasks.completeMockTasks();
|
tasks.completeMockTasks();
|
||||||
tasks.completeMockTasks();
|
tasks.completeMockTasks();
|
||||||
assertEquals("failed",status());
|
assertEquals("failed",status());
|
||||||
@@ -216,19 +221,167 @@ class AsyncTaskPostgresTest {
|
|||||||
assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId));
|
assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed() {
|
@Test void deadlineRetainsHoldWithoutClaimingTheUpstreamFailed() {
|
||||||
jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId);
|
jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId);
|
||||||
poll(); poll();
|
poll(); poll();
|
||||||
assertEquals("failed",status());
|
assertEquals("reconciling",status());
|
||||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||||
assertEquals(0,wallet("reserved_balance").signum());
|
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
|
||||||
assertEquals(0,cost().signum());
|
assertNull(cost());
|
||||||
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
|
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
|
||||||
assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT"));
|
assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT"));
|
||||||
assertTrue(error.contains("执行结果未知"));
|
assertTrue(error.contains("执行结果未知"));
|
||||||
verifyNoInteractions(provider);
|
verifyNoInteractions(provider);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@ValueSource(ints={408,409,500,502,503,504})
|
||||||
|
void uncertainSubmitResponseNeverRefundsOrResubmits(int httpStatus) {
|
||||||
|
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
|
||||||
|
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||||
|
.thenThrow(ProviderRequestException.http(httpStatus, "safe local message"));
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
assertEquals("reconciling", status());
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||||
|
assertNull(cost());
|
||||||
|
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||||
|
assertEquals("pending", jdbc.queryForObject("SELECT cost_status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||||
|
assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId));
|
||||||
|
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void lostSubmitResponseAndInterruptedClaimRequireReconciliationWithoutReplay() {
|
||||||
|
jdbc.update("UPDATE model_tasks SET provider_task_id=NULL,updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
assertEquals("reconciling", status());
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
assertNull(cost());
|
||||||
|
verifyNoInteractions(provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void missingTaskReceiptRetainsFinancialAndBudgetHolds() {
|
||||||
|
var created = create("missing-receipt", Map.of("model", "model-mock-3d", "prompt", "chair"));
|
||||||
|
String id = String.valueOf(created.get("id"));
|
||||||
|
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap())).thenReturn(Map.of("accepted", true));
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id));
|
||||||
|
assertEquals(new BigDecimal("2.88000000"), wallet("reserved_balance"));
|
||||||
|
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id));
|
||||||
|
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void genericSubmitFailureIsNotTreatedAsEvidenceOfRejection() {
|
||||||
|
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
|
||||||
|
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||||
|
.thenThrow(new IllegalStateException("network connection lost; private details"));
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
assertEquals("reconciling", status());
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
assertFalse(jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?", String.class, taskId).contains("private details"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void missingPollingCredentialDoesNotRefundAnAlreadyAcceptedJob() {
|
||||||
|
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
|
||||||
|
.thenThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "sensitive detail", 503));
|
||||||
|
poll();
|
||||||
|
assertEquals("reconciling", status());
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
assertNull(cost());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void inlineSuccessSurvivesSettlementFailureAndRecoversWithoutRegenerating() {
|
||||||
|
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
|
||||||
|
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||||
|
.thenReturn(Map.of("_moyiInlineResult", Map.of("url", "https://example.test/generated.glb")));
|
||||||
|
doThrow(new IllegalStateException("simulated audit failure")).when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull());
|
||||||
|
assertThrows(IllegalStateException.class, () -> tasks.completeMockTasks());
|
||||||
|
assertEquals("running", status());
|
||||||
|
assertEquals("succeeded", jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId));
|
||||||
|
fail(); // A stale error cannot refund an already observed success before the retry settles it.
|
||||||
|
assertEquals("running", status());
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||||
|
assertNull(cost());
|
||||||
|
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
|
||||||
|
doCallRealMethod().when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull());
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
assertEquals("succeeded", status());
|
||||||
|
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||||
|
assertEquals(0, wallet("reserved_balance").signum());
|
||||||
|
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId));
|
||||||
|
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@ValueSource(strings={"unsuccessful","not_failed","cancelling","unknown"})
|
||||||
|
void ambiguousProviderStatusCannotTriggerSettlementOrRefund(String state) {
|
||||||
|
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
|
||||||
|
.thenReturn(Map.of("status", state));
|
||||||
|
poll();
|
||||||
|
assertEquals("running", status());
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
assertNull(cost());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void interruptedClaimCheckCannotQuarantineALaterAcceptedReceipt() {
|
||||||
|
var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage);
|
||||||
|
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
|
||||||
|
journal.requireReconciliation(taskId, "STALE_WORKER", "stale claim check", true);
|
||||||
|
assertEquals("running", status(), "An upstream task ID proves this is no longer an interrupted claim");
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void reconciliationAndUsageAuditRollBackTogetherWithoutTouchingTheHold() {
|
||||||
|
var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage);
|
||||||
|
doThrow(new IllegalStateException("audit offline")).when(usage).updateTask(eq(taskId), eq("reconciling"), isNull(), anyString(), anyString());
|
||||||
|
assertThrows(IllegalStateException.class, () -> journal.requireReconciliation(taskId, "UNKNOWN", "result unknown", false));
|
||||||
|
assertEquals("running", status());
|
||||||
|
assertEquals("running", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||||
|
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||||
|
assertNull(cost());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void failedTaskReplayDoesNotRequeueOrReserveAgain() {
|
||||||
|
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "will be rejected");
|
||||||
|
var first = create("rejected-task", input);
|
||||||
|
String id = String.valueOf(first.get("id"));
|
||||||
|
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap()))
|
||||||
|
.thenThrow(ProviderRequestException.http(429, "provider rejected"));
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
BigDecimal held = wallet("reserved_balance");
|
||||||
|
var replay = create("rejected-task", input);
|
||||||
|
assertEquals(id, replay.get("id"));
|
||||||
|
assertEquals("failed", replay.get("status"));
|
||||||
|
assertEquals(held, wallet("reserved_balance"));
|
||||||
|
assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id));
|
||||||
|
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void pendingReconciliationConsumesAsyncConcurrencyRatherThanOpeningAnotherSlot() {
|
||||||
|
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000, false,
|
||||||
|
guard, usage, provider, new DataSourceTransactionManager(jdbc.getDataSource()),
|
||||||
|
new com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy(com.moyi.controlplane.gateway.request.ApiRequestRiskEngine.allowAll()));
|
||||||
|
jdbc.update("UPDATE model_tasks SET status='reconciling' WHERE id=?", taskId);
|
||||||
|
jdbc.update("UPDATE model_catalog SET max_concurrency=1 WHERE id='model-mock-3d'");
|
||||||
|
String id = String.valueOf(create("next-while-unknown", Map.of("model", "model-mock-3d", "prompt", "another chair")).get("id"));
|
||||||
|
tasks.completeMockTasks();
|
||||||
|
assertEquals("queued", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id));
|
||||||
|
verifyNoInteractions(provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test void documentedSucceededStatusSettlesInsteadOfPollingForever() {
|
||||||
|
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
|
||||||
|
.thenReturn(Map.of("status", "succeeded", "url", "https://example.test/generated.glb"));
|
||||||
|
poll();
|
||||||
|
assertEquals("succeeded", status());
|
||||||
|
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||||
|
}
|
||||||
|
|
||||||
@Test void confirmedUpstreamFailureReleasesHoldImmediately() {
|
@Test void confirmedUpstreamFailureReleasesHoldImmediately() {
|
||||||
when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)).thenReturn(Map.of("data",Map.of("status","failed")));
|
when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)).thenReturn(Map.of("data",Map.of("status","failed")));
|
||||||
poll();
|
poll();
|
||||||
|
|||||||
+3
-1
@@ -39,7 +39,7 @@ class UsageCapacityPostgresTest {
|
|||||||
UUID user = UUID.randomUUID(), key = UUID.randomUUID();
|
UUID user = UUID.randomUUID(), key = UUID.randomUUID();
|
||||||
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'capacity-user','username','not-loginable','Capacity user')", user);
|
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'capacity-user','username','not-loginable','Capacity user')", user);
|
||||||
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Capacity test','local-test',?)", key, user, "c".repeat(64));
|
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Capacity test','local-test',?)", key, user, "c".repeat(64));
|
||||||
for (String state : List.of("running", "queued", "succeeded")) {
|
for (String state : List.of("running", "queued", "reconciling", "succeeded")) {
|
||||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,created_at) VALUES (?,?,?,'capacity-route-105','3d',?,now()-interval '1 day')", "task-" + state, user, key, state);
|
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,created_at) VALUES (?,?,?,'capacity-route-105','3d',?,now()-interval '1 day')", "task-" + state, user, key, state);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -67,9 +67,11 @@ class UsageCapacityPostgresTest {
|
|||||||
Map<String, Object> async = groups.stream().filter(group -> group.get("id").equals("capacity-route-105")).findFirst().orElseThrow();
|
Map<String, Object> async = groups.stream().filter(group -> group.get("id").equals("capacity-route-105")).findFirst().orElseThrow();
|
||||||
assertEquals(1L, async.get("running"), "A task older than ten minutes must remain in live capacity");
|
assertEquals(1L, async.get("running"), "A task older than ten minutes must remain in live capacity");
|
||||||
assertEquals(1L, async.get("queued"), "Running tasks must not also count as queued");
|
assertEquals(1L, async.get("queued"), "Running tasks must not also count as queued");
|
||||||
|
assertEquals(1L, async.get("reconciling"), "Unknown upstream outcomes must remain visible, not be counted as failed or idle");
|
||||||
assertEquals(7, async.get("limit"), "Null route limits use the real configured model default");
|
assertEquals(7, async.get("limit"), "Null route limits use the real configured model default");
|
||||||
assertEquals(3L, summary.get("running"));
|
assertEquals(3L, summary.get("running"));
|
||||||
assertEquals(4L, summary.get("waiting"));
|
assertEquals(4L, summary.get("waiting"));
|
||||||
|
assertEquals(1L, summary.get("reconciling"));
|
||||||
Map<?, ?> config = (Map<?, ?>) summary.get("capacityConfig");
|
Map<?, ?> config = (Map<?, ?>) summary.get("capacityConfig");
|
||||||
assertEquals(4500L, config.get("queueWaitMs"));
|
assertEquals(4500L, config.get("queueWaitMs"));
|
||||||
assertEquals(3, config.get("defaultKeyConcurrency"));
|
assertEquals(3, config.get("defaultKeyConcurrency"));
|
||||||
|
|||||||
@@ -63,8 +63,8 @@ PostgreSQL 集成测试由 `MOYI_AUTH_TEST_JDBC_URL` 启用,当前测试限制
|
|||||||
|
|
||||||
## 下一阶段优先级
|
## 下一阶段优先级
|
||||||
|
|
||||||
1. 异步创建 `Idempotency-Key`、请求哈希和数据库唯一性/锁语义,保证客户端重试不重复排队或预留。
|
1. 已完成 `V45`:异步创建 `Idempotency-Key`、请求哈希和数据库唯一性/锁语义,客户端同键重试不重复排队或预留。
|
||||||
2. 异步上游提交结果未知的持久状态与核账入口:区分明确未受理、已受理可轮询、结果未知,禁止统一退款后自动重试。
|
2. 已完成 `V46` 的未知受理持久状态与冻结保护;审计化人工核账入口仍待实现。详情及本阶段 190 项后端验证结果见 `22-async-task-safety.md`。
|
||||||
3. 执行中元数据和计量快照的崩溃窗口、重试退避、人工对账与待结算可视化。
|
3. 已增加异步已观察终态的结算恢复与待核账提示;继续完善执行路由快照、重试退避、人工对账和不可原子提交窗口。
|
||||||
4. HTTP Transport 生命周期继续拆分;真实 Redis 并发、租约回收与队列公平性回归。
|
4. HTTP Transport 生命周期继续拆分;真实 Redis 并发、租约回收与队列公平性回归。
|
||||||
5. 再进入用户/角色/会话、管理操作与密钥生命周期的后续安全优化;本轮模块改造不替代完整审计。
|
5. 再进入用户/角色/会话、管理操作与密钥生命周期的后续安全优化;本轮模块改造不替代完整审计。
|
||||||
@@ -12,6 +12,50 @@
|
|||||||
|
|
||||||
验证:Java 21 / Maven,隔离 PostgreSQL 15.18(不连接业务数据库),26 项针对性测试通过,0 失败、0 跳过。包含 8 路并发同键请求、重放、请求冲突、完成后路由下线重放、键摘要规范化与风控顺序。
|
验证:Java 21 / Maven,隔离 PostgreSQL 15.18(不连接业务数据库),26 项针对性测试通过,0 失败、0 跳过。包含 8 路并发同键请求、重放、请求冲突、完成后路由下线重放、键摘要规范化与风控顺序。
|
||||||
|
|
||||||
## 接续项
|
## 阶段二:未知受理与可恢复结算(2026-10-21)
|
||||||
|
|
||||||
接续修复提交响应丢失、进程中断、轮询超时以及结算数据库失败的语义。未知结果不得自动重发生成请求或释放冻结余额;只有已证实的拒绝或终态失败才能进入正常释放流程。核账操作必须有证据和审计,不允许用直接修改钱包余额替代结算事务。
|
### 模块边界
|
||||||
|
|
||||||
|
- `provider.transport.ProviderRequestException` 只描述 HTTP 边界事实:`NOT_SENT`、`REJECTED`、`UNKNOWN`,保留可安全展示的错误码及 HTTP 状态,不附带上游响应体、凭据或异常原文;不决定重试和退款。
|
||||||
|
- `task.TaskOutcomeJournal` 持久化已观察的终态和待核账状态,不发送请求、不修改钱包、不制定退款策略。
|
||||||
|
- `AsyncTaskService` 编排提交、只读轮询和本地结算。收到生成结果后先提交日志,再在原有钱包/任务/审计事务中结算;账本失败不能被提交请求异常处理器误认为上游生成失败。
|
||||||
|
|
||||||
|
### 状态与账务
|
||||||
|
|
||||||
|
| 观察到的事实 | 任务状态 / 后续动作 | 钱包与预算 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| 本地输入/配置校验失败、请求尚未发出 | `failed` | 原子释放预留,不扣费 |
|
||||||
|
| HTTP 400/401/402/403/404/405/413/415/422/429 明确拒绝 | `failed` | 原子释放预留,不自动重发 |
|
||||||
|
| HTTP 408/409、5xx、网络异常、成功响应不可解析 | `reconciling`,不自动重发 | 保留冻结及预算占用,等待证据核账 |
|
||||||
|
| 回执缺少合法任务 ID,或回执无法保存 | `reconciling` | 保留冻结,不假定生成失败 |
|
||||||
|
| worker 已认领、两分钟后仍无 ID 且无终态日志 | `reconciling` | 保留冻结,不重新派发 |
|
||||||
|
| 已受理任务的轮询期限到期,或原凭据已移除 | `reconciling`,停止自动轮询 | 保留冻结,不将查询失败视为生成失败 |
|
||||||
|
| 已收到上游终态,但本地结算失败 | 保存 `completion_status` 和归一化结果,下轮只重试本地结算 | 保留冻结,成功后只扣一次 |
|
||||||
|
| 已确认上游失败/取消 | 原子提交终态与审计 | 释放预留,只执行一次 |
|
||||||
|
|
||||||
|
兼容视频路径的备用接口只在可结构化识别的 HTTP 404/405 拒绝后尝试,不再解析错误文案判断重试;5xx 或结果未知不换接口。轮询是只读操作,瞬时鉴权/限流/网络异常可继续尝试至持久化期限,但不能由此推导退款。
|
||||||
|
|
||||||
|
`V46` 扩展 `model_tasks` / `usage_events` 状态约束并增加终态观察日志。生成内容复用已有 `result` 列,不重复存两份。日志一旦观察到终态,晚到的冲突状态不能覆盖它;过期认领检查也不能覆盖已经保存的任务编号。
|
||||||
|
|
||||||
|
`reconciling` 仍计入异步模型/API Key/用户的数据库并发与队列容量,避免未知任务被无限替换为新任务。用户任务页展示“待核账”,提示额度仍冻结、勿重复提交;使用记录可筛选,管理员容量分组保留待核账计数,即使渠道已停用也不隐藏。
|
||||||
|
|
||||||
|
状态解析改为明确白名单:`success` / `succeeded` / `completed` 才能结算;不再把 `unsuccessful` 当成功,也不把 `cancelling` 当取消完成。其他未知状态继续只读查询,期限到后进入核账状态。
|
||||||
|
|
||||||
|
### 上线与核账边界
|
||||||
|
|
||||||
|
1. 本阶段只验证隔离测试数据库,没有迁移业务数据库或部署线上服务。发布时先停旧 worker,再迁移 `V45` / `V46` 并启新 worker,避免混跑旧的自动退款逻辑。
|
||||||
|
2. 不回溯改写历史失败任务或重新冻结曾经释放的资金。历史的提交中断、超时、凭据丢失记录须独立审计,不能自动扣款。
|
||||||
|
3. 待核账状态不会按 TTL 自动退款,不会自动重发。收集本地任务 ID、用户、时间、模型路由、预留金额、上游任务/账单记录;不要把完整凭据放进工单或日志。
|
||||||
|
4. 暂不提供“无证据一键退款/重新生成”的管理接口。受审计的人工核账入口、关联上游任务编号、人工确认终态以及差额结算是下一阶段产品工作;在该入口完成前,只允许按受控运维流程核对证据,不直接修改钱包余额替代结算事务。
|
||||||
|
5. 成功的 token 图片任务缺少可核验用量时,仍沿用既有 `TASK_USAGE_PENDING` 规则:保留结果、暂不扣费、释放该次预留并标记待核账。本阶段没有把“已知完成但计量缺失”与“是否受理未知”混为一类。
|
||||||
|
6. HTTP 受理至观察日志提交之间仍有无法跨外部系统原子提交的窗口。数据库完全不可用时也不能承诺保存回执;恢复后转入待核账,而非声称精确恢复所有生成结果。
|
||||||
|
|
||||||
|
后续重点:审计化人工核账入口、异步路由/上游模型的执行快照、共享有界 HTTP Transport 与媒体下载安全、上游有证明的幂等能力适配。
|
||||||
|
|
||||||
|
### 最终验证
|
||||||
|
|
||||||
|
2026-10-21:Docker / Java 21 / Maven 3.9.9 执行 `mvn -B -o clean verify`,后端 190 项测试全部通过(0 失败、0 错误、0 跳过),Spring Boot JAR 构建成功。使用独立 PostgreSQL 15.18,Maven 容器共享测试数据库的 network namespace;未访问业务数据库或真实供应商 API。
|
||||||
|
|
||||||
|
前端 `npm run build`(含 `vue-tsc --noEmit`)和 `npm test` 通过,22 项测试全部通过。前端“待核账”提示已通过类型检查和构建,未将这些测试描述为完整浏览器交互验收。`git diff --check` 无错误。
|
||||||
|
|
||||||
|
重点回归覆盖:8 路同键并发、终态/失败后重放、不同请求冲突、HTTP 408/409/5xx、收到 POST 后断开响应且不重发、成功响应 JSON 无法解析、无任务 ID、旧认领检查竞态、凭据丢失、轮询期限、待核账并发占位、审计事务回滚、成功结果先保存后结算、晚到失败不得退款、结算恢复一次扣费,以及模糊状态白名单。
|
||||||
@@ -23,7 +23,7 @@ const number = (n: unknown) => n == null ? '—' : Number(n).toLocaleString('zh-
|
|||||||
const money = (n: unknown) => n == null ? '—' : `¥ ${Number(n).toLocaleString('zh-CN', { minimumFractionDigits: 2, maximumFractionDigits: 8 })}`
|
const money = (n: unknown) => n == null ? '—' : `¥ ${Number(n).toLocaleString('zh-CN', { minimumFractionDigits: 2, maximumFractionDigits: 8 })}`
|
||||||
const duration = (n: unknown) => n == null ? '—' : `${(Number(n) / 1000).toFixed(2)}s`
|
const duration = (n: unknown) => n == null ? '—' : `${(Number(n) / 1000).toFixed(2)}s`
|
||||||
const time = (value: string) => new Date(value).toLocaleString('zh-CN', { hour12: false })
|
const time = (value: string) => new Date(value).toLocaleString('zh-CN', { hour12: false })
|
||||||
const statusName = (v: string) => ({ succeeded: '成功', failed: '失败', queued: '排队中', running: '执行中', cancelled: '已取消' })[v] || v
|
const statusName = (v: string) => ({ succeeded: '成功', failed: '失败', queued: '排队中', running: '执行中', reconciling: '待核账(额度仍冻结)', cancelled: '已取消' })[v] || v
|
||||||
const modeName = (v?: string) => ({ sync: '同步', stream: '流式', async: '异步' })[v || ''] || '—'
|
const modeName = (v?: string) => ({ sync: '同步', stream: '流式', async: '异步' })[v || ''] || '—'
|
||||||
const sourceName = (v?: string) => ({ playground: '在线调试', api: 'API', unknown: '历史调用' })[v || 'unknown'] || '历史调用'
|
const sourceName = (v?: string) => ({ playground: '在线调试', api: 'API', unknown: '历史调用' })[v || 'unknown'] || '历史调用'
|
||||||
const pretty = (v: unknown) => v == null ? '未保存内容' : JSON.stringify(v, null, 2)
|
const pretty = (v: unknown) => v == null ? '未保存内容' : JSON.stringify(v, null, 2)
|
||||||
@@ -108,7 +108,7 @@ onUnmounted(() => { requestSequence++; selectedId.value = ''; if (timer) window.
|
|||||||
<label>分组<select v-model="filters.groupId" class="text-input"><option value="">全部分组</option><option v-for="group in groups" :key="group.id" :value="group.id">{{ group.name }} · {{ group.channel }}</option></select></label>
|
<label>分组<select v-model="filters.groupId" class="text-input"><option value="">全部分组</option><option v-for="group in groups" :key="group.id" :value="group.id">{{ group.name }} · {{ group.channel }}</option></select></label>
|
||||||
<label>调用类型<select v-model="filters.callMode" class="text-input"><option value="">全部类型</option><option value="sync">同步</option><option value="stream">流式</option><option value="async">异步</option></select></label>
|
<label>调用类型<select v-model="filters.callMode" class="text-input"><option value="">全部类型</option><option value="sync">同步</option><option value="stream">流式</option><option value="async">异步</option></select></label>
|
||||||
<label>请求类别<select v-model="filters.eventKind" class="text-input"><option value="">全部请求</option><option value="call">模型调用</option><option value="task">异步任务</option></select></label>
|
<label>请求类别<select v-model="filters.eventKind" class="text-input"><option value="">全部请求</option><option value="call">模型调用</option><option value="task">异步任务</option></select></label>
|
||||||
<label>状态<select v-model="filters.status" class="text-input"><option value="">全部状态</option><option v-for="state in ['succeeded','failed','queued','running','cancelled']" :key="state" :value="state">{{ statusName(state) }}</option></select></label>
|
<label>状态<select v-model="filters.status" class="text-input"><option value="">全部状态</option><option v-for="state in ['succeeded','failed','queued','running','reconciling','cancelled']" :key="state" :value="state">{{ statusName(state) }}</option></select></label>
|
||||||
<label>费用记录<select v-model="filters.costState" class="text-input"><option value="">全部</option><option value="recorded">已记录</option><option value="pending">未记录</option></select></label>
|
<label>费用记录<select v-model="filters.costState" class="text-input"><option value="">全部</option><option value="recorded">已记录</option><option value="pending">未记录</option></select></label>
|
||||||
<label>来源<select v-model="filters.source" class="text-input"><option value="">全部来源</option><option value="playground">在线调试</option><option value="api">API</option><option value="unknown">历史调用(未标记来源)</option></select></label>
|
<label>来源<select v-model="filters.source" class="text-input"><option value="">全部来源</option><option value="playground">在线调试</option><option value="api">API</option><option value="unknown">历史调用(未标记来源)</option></select></label>
|
||||||
<label>User-Agent<input v-model="filters.userAgent" class="text-input" maxlength="200" placeholder="模糊搜索客户端" /></label>
|
<label>User-Agent<input v-model="filters.userAgent" class="text-input" maxlength="200" placeholder="模糊搜索客户端" /></label>
|
||||||
|
|||||||
@@ -122,7 +122,7 @@ export interface PlatformOverview {
|
|||||||
availabilityPercent?: number
|
availabilityPercent?: number
|
||||||
asyncTasks: number
|
asyncTasks: number
|
||||||
asyncSuccessRate?: number
|
asyncSuccessRate?: number
|
||||||
tasks: { queued: number; running: number; succeeded: number; failed: number }
|
tasks: { queued: number; running: number; reconciling?: number; succeeded: number; failed: number }
|
||||||
series: Array<{ date: string; count: number }>
|
series: Array<{ date: string; count: number }>
|
||||||
ledger: Array<{ id: string; title: string; amount: number; createdAt: string }>
|
ledger: Array<{ id: string; title: string; amount: number; createdAt: string }>
|
||||||
}
|
}
|
||||||
@@ -172,8 +172,9 @@ export interface AdminUsageSummary {
|
|||||||
costToday: number
|
costToday: number
|
||||||
costThisMonth: number
|
costThisMonth: number
|
||||||
series: Array<{ date: string; count: number }>
|
series: Array<{ date: string; count: number }>
|
||||||
capacityGroups: Array<{ id: string; name: string; sourceLabel?: string; providerName?: string; mode?: string; routeEnabled?: boolean; running: number | null; limit: number; queued: number | null }>
|
capacityGroups: Array<{ id: string; name: string; sourceLabel?: string; providerName?: string; mode?: string; routeEnabled?: boolean; running: number | null; limit: number; queued: number | null; reconciling?: number }>
|
||||||
running?: number | null
|
running?: number | null
|
||||||
|
reconciling?: number | null
|
||||||
capacityConfig?: {
|
capacityConfig?: {
|
||||||
queueWaitMs: number
|
queueWaitMs: number
|
||||||
defaultModelConcurrency: number
|
defaultModelConcurrency: number
|
||||||
@@ -231,7 +232,8 @@ export interface AsyncTaskResult {
|
|||||||
id: string
|
id: string
|
||||||
model: string
|
model: string
|
||||||
provider?: string
|
provider?: string
|
||||||
status: 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled'
|
status: 'queued' | 'running' | 'reconciling' | 'succeeded' | 'failed' | 'cancelled'
|
||||||
|
requiresReconciliation?: boolean
|
||||||
progress?: number
|
progress?: number
|
||||||
created_at?: string
|
created_at?: string
|
||||||
updated_at?: string
|
updated_at?: string
|
||||||
@@ -240,7 +242,7 @@ export interface AsyncTaskResult {
|
|||||||
result?: { format?: string; url?: string; modelUrl?: string; previewUrl?: string; videoUrl?: string; imageUrl?: string; audioUrl?: string; mimeType?: string; creditsConsumed?: number; [key: string]: unknown }
|
result?: { format?: string; url?: string; modelUrl?: string; previewUrl?: string; videoUrl?: string; imageUrl?: string; audioUrl?: string; mimeType?: string; creditsConsumed?: number; [key: string]: unknown }
|
||||||
output?: { modelUrl?: string; previewUrl?: string; videoUrl?: string; imageUrl?: string; audioUrl?: string; url?: string; format?: string; mimeType?: string }
|
output?: { modelUrl?: string; previewUrl?: string; videoUrl?: string; imageUrl?: string; audioUrl?: string; url?: string; format?: string; mimeType?: string }
|
||||||
creditsConsumed?: number
|
creditsConsumed?: number
|
||||||
error?: { code?: number; message?: string }
|
error?: { code?: string | number; message?: string; retryable?: boolean; requiresReconciliation?: boolean }
|
||||||
}
|
}
|
||||||
|
|
||||||
/** OpenAI-compatible multimodal content. Keep text-only calls byte-for-byte compatible. */
|
/** OpenAI-compatible multimodal content. Keep text-only calls byte-for-byte compatible. */
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import type { AuthRole } from './services/auth'
|
|||||||
export type Portal = 'user' | 'admin'
|
export type Portal = 'user' | 'admin'
|
||||||
export type ModelType = '文本' | '多模态' | '图像' | '3D' | '视频' | '音频'
|
export type ModelType = '文本' | '多模态' | '图像' | '3D' | '视频' | '音频'
|
||||||
export type CallMode = '同步' | '流式' | '异步'
|
export type CallMode = '同步' | '流式' | '异步'
|
||||||
export type TaskStatus = '排队中' | '执行中' | '已完成' | '失败'
|
export type TaskStatus = '排队中' | '执行中' | '待核账' | '已完成' | '失败'
|
||||||
|
|
||||||
export interface NavItem {
|
export interface NavItem {
|
||||||
id: string
|
id: string
|
||||||
|
|||||||
@@ -202,8 +202,8 @@ function formatUsagePayload(value: unknown) {
|
|||||||
const queueGroups = computed(() => (adminSummary.value?.capacityGroups || []).map((group) => ({
|
const queueGroups = computed(() => (adminSummary.value?.capacityGroups || []).map((group) => ({
|
||||||
...group,
|
...group,
|
||||||
type: `${group.providerName || '未知渠道'} · ${group.sourceLabel || '模型分组'}`,
|
type: `${group.providerName || '未知渠道'} · ${group.sourceLabel || '模型分组'}`,
|
||||||
status: group.running === null ? '未读取' : group.routeEnabled === false ? '已停用,仍有任务' : group.queued ? '排队中' : group.running ? '执行中' : '空闲',
|
status: group.reconciling ? `待核账 ${group.reconciling} 笔(额度仍冻结)` : group.running === null ? '未读取' : group.routeEnabled === false ? '已停用,仍有任务' : group.queued ? '排队中' : group.running ? '执行中' : '空闲',
|
||||||
warning: group.running === null || group.routeEnabled === false || !!group.queued,
|
warning: group.running === null || group.routeEnabled === false || !!group.queued || !!group.reconciling,
|
||||||
})))
|
})))
|
||||||
const { page: capacityPage, pageSize: capacityPageSize, pagedItems: capacityPageGroups } = usePagination(queueGroups)
|
const { page: capacityPage, pageSize: capacityPageSize, pagedItems: capacityPageGroups } = usePagination(queueGroups)
|
||||||
const configuredCapacityGroups = computed(() => adminSummary.value?.capacityGroups.filter(group => group.routeEnabled !== false) || [])
|
const configuredCapacityGroups = computed(() => adminSummary.value?.capacityGroups.filter(group => group.routeEnabled !== false) || [])
|
||||||
|
|||||||
@@ -235,6 +235,7 @@ function normalizeOverview(value: unknown): PlatformOverview {
|
|||||||
tasks: {
|
tasks: {
|
||||||
queued: asNumber(rawTasks.queued),
|
queued: asNumber(rawTasks.queued),
|
||||||
running: asNumber(rawTasks.running),
|
running: asNumber(rawTasks.running),
|
||||||
|
reconciling: asNumber(rawTasks.reconciling),
|
||||||
succeeded: asNumber(rawTasks.succeeded),
|
succeeded: asNumber(rawTasks.succeeded),
|
||||||
failed: asNumber(rawTasks.failed),
|
failed: asNumber(rawTasks.failed),
|
||||||
},
|
},
|
||||||
@@ -504,6 +505,7 @@ function mapAsyncTask(task: AsyncTaskResult, model: ModelItem, taskPrompt?: stri
|
|||||||
const statusMap: Record<AsyncTaskResult['status'], TaskStatus> = {
|
const statusMap: Record<AsyncTaskResult['status'], TaskStatus> = {
|
||||||
queued: '排队中',
|
queued: '排队中',
|
||||||
running: '执行中',
|
running: '执行中',
|
||||||
|
reconciling: '待核账',
|
||||||
succeeded: '已完成',
|
succeeded: '已完成',
|
||||||
failed: '失败',
|
failed: '失败',
|
||||||
cancelled: '失败',
|
cancelled: '失败',
|
||||||
@@ -516,7 +518,7 @@ function mapAsyncTask(task: AsyncTaskResult, model: ModelItem, taskPrompt?: stri
|
|||||||
status: statusMap[task.status],
|
status: statusMap[task.status],
|
||||||
progress: task.status === 'succeeded' ? 100 : Math.round(task.progress || 0),
|
progress: task.status === 'succeeded' ? 100 : Math.round(task.progress || 0),
|
||||||
createdAt: createdAt ? formatTaskTime(createdAt) : '刚刚',
|
createdAt: createdAt ? formatTaskTime(createdAt) : '刚刚',
|
||||||
duration: task.completedAt ? '已完成' : '计算中',
|
duration: task.completedAt ? '已完成' : task.status === 'reconciling' ? '等待核账' : '计算中',
|
||||||
result: task.error?.message || (videoUrl ? '视频已生成' : imageUrl ? '图片已生成' : audioUrl ? '音频已生成' : modelUrl ? 'GLB 模型已生成' : resultUrl ? '文件已生成' : task.status === 'succeeded' ? '异步任务已完成' : task.status === 'running' ? `正在生成 · ${Math.round(task.progress || 0)}%` : '等待平台调度'),
|
result: task.error?.message || (videoUrl ? '视频已生成' : imageUrl ? '图片已生成' : audioUrl ? '音频已生成' : modelUrl ? 'GLB 模型已生成' : resultUrl ? '文件已生成' : task.status === 'succeeded' ? '异步任务已完成' : task.status === 'running' ? `正在生成 · ${Math.round(task.progress || 0)}%` : '等待平台调度'),
|
||||||
provider: task.provider,
|
provider: task.provider,
|
||||||
prompt: taskPrompt,
|
prompt: taskPrompt,
|
||||||
@@ -557,13 +559,11 @@ async function pollTask(taskId: string, apiKey: string) {
|
|||||||
scheduleTaskPoll(taskId, apiKey, 2500)
|
scheduleTaskPoll(taskId, apiKey, 2500)
|
||||||
} else {
|
} else {
|
||||||
taskPollers.delete(taskId)
|
taskPollers.delete(taskId)
|
||||||
// The async endpoint settles or releases the wallet only when the
|
// Reconciliation stops automatic polling, but it is NOT a failure or a refund.
|
||||||
// provider task reaches a terminal state. Refresh the overview here so
|
// Refresh financial state without encouraging another generation request.
|
||||||
// the balance card and usage counters reflect that settlement immediately
|
|
||||||
// instead of waiting for a full page reload.
|
|
||||||
await refreshOverview()
|
await refreshOverview()
|
||||||
historyRevision.value++
|
historyRevision.value++
|
||||||
emit('notify', latest.status === 'succeeded' ? `${previous.name}完成,可以查看和保存结果` : `${previous.name}未能完成,请查看错误说明`)
|
emit('notify', latest.status === 'reconciling' ? `${previous.name}结果尚未确认,额度仍冻结,请联系管理员核账,勿重复提交` : latest.status === 'succeeded' ? `${previous.name}完成,可以查看和保存结果` : `${previous.name}未能完成,请查看错误说明`)
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
taskPollers.delete(taskId)
|
taskPollers.delete(taskId)
|
||||||
@@ -1010,6 +1010,7 @@ $env:OPENAI_API_KEY="你的 MOYI 用户 API Key"
|
|||||||
.token-limit { display: grid; gap: 8px; }
|
.token-limit { display: grid; gap: 8px; }
|
||||||
.response-result > p, .response-content { white-space: pre-wrap; overflow-wrap: anywhere; max-width: 100%; }
|
.response-result > p, .response-content { white-space: pre-wrap; overflow-wrap: anywhere; max-width: 100%; }
|
||||||
.response-result .status-chip.failed { color: #b42318; background: #fee4e2; }
|
.response-result .status-chip.failed { color: #b42318; background: #fee4e2; }
|
||||||
|
.status-chip.待核账 { color: #b54708; background: #fffaeb; }
|
||||||
.response-result p[role="alert"] { color: #b42318; }
|
.response-result p[role="alert"] { color: #b42318; }
|
||||||
.playground-selectors { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; }
|
.playground-selectors { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; }
|
||||||
.playground-selectors label { display: grid; gap: 8px; font-size: 13px; min-width: 0; }
|
.playground-selectors label { display: grid; gap: 8px; font-size: 13px; min-width: 0; }
|
||||||
|
|||||||
Reference in new issue
Block a user