fix: retain async holds for unknown provider outcomes
This commit is contained in:
1 parent
76265845dc
commit
5be06a97c1
18 files changed
+574
-108
No files matched your search
@@ -15,7 +15,9 @@
|
||||
|
||||
协议字段映射留在各自 Meter/Adapter;`ApiRequestRiskRule` 只能读取最小化上下文并返回决策,不能调用 Provider 或改账。新增协议时应复用准入、风控和账务边界,而不是在 Controller 中另写一套钱包/重试流程。默认 `GATEWAY_RISK_MODE=shadow` 仅记录规则 ID,不改变调用;只有显式部署规则并设置 `enforce` 才会拒绝或限速。请求来源由服务端路由确定,不能信任 `X-Moyi-Source` 作为风控分类。
|
||||
|
||||
账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;失败释放和成功结算沿用现有事务处理;不完整流或无效用量不得结算。上游提交结果未知时的异步核账状态机与异步创建幂等将在后续阶段单独处理。
|
||||
账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;已确认失败的释放和成功结算沿用原子事务;不完整流或无效用量不得估算扣费。
|
||||
|
||||
异步创建支持用户级 `Idempotency-Key`(`V45`),同键同请求返回原任务,同键不同请求返回 409。异步上游受理未知、回执丢失、worker 中断或轮询期限到期时进入 `reconciling`(`V46`),保留冻结与预算占用,不自动重发/退款。已观察的生成结果先持久化再结算,账本失败只重试本地结算。发布时须先停旧 worker,不能混跑旧的未知结果退款逻辑。详见 `docs/22-async-task-safety.md`;受审计的人工核账入口仍属下一阶段。
|
||||
|
||||
同步和原生调用在结算前单独提交精简用量快照(Flyway `V44`)。快照已持久化而账本事务失败时,接口返回 `SETTLEMENT_PENDING`,冻结额度保留,由后台恢复任务重试结算。该快照不保存提示词、生成内容或密钥;不能将其当作响应重放存储。若数据库在快照写入前已不可用,则尚不能保证结果的持久恢复,必须结合上游记录核对。
|
||||
|
||||
|
||||
@@ -383,6 +383,7 @@ public class CatalogService {
|
||||
result.put("tasks", Map.of(
|
||||
"queued", taskCounts.getOrDefault("queued", 0L),
|
||||
"running", taskCounts.getOrDefault("running", 0L),
|
||||
"reconciling", taskCounts.getOrDefault("reconciling", 0L),
|
||||
"succeeded", taskCounts.getOrDefault("succeeded", 0L),
|
||||
"failed", taskCounts.getOrDefault("failed", 0L)
|
||||
));
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package com.moyi.controlplane.provider.transport;
|
||||
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
|
||||
/** HTTP boundary facts, not a retry policy. Unknown acceptance must never authorize replay/refund. */
|
||||
public final class ProviderRequestException extends ApiException {
|
||||
public enum Acceptance { NOT_SENT, REJECTED, UNKNOWN }
|
||||
|
||||
private final Acceptance acceptance;
|
||||
private final Integer upstreamStatus;
|
||||
|
||||
private ProviderRequestException(String code, String message, int status, Acceptance acceptance, Integer upstreamStatus) {
|
||||
super(code, message, status);
|
||||
this.acceptance = acceptance;
|
||||
this.upstreamStatus = upstreamStatus;
|
||||
}
|
||||
|
||||
public static ProviderRequestException notSent(String code, String message, int status) {
|
||||
return new ProviderRequestException(code, message, status, Acceptance.NOT_SENT, null);
|
||||
}
|
||||
|
||||
public static ProviderRequestException unknown(String code, String message) {
|
||||
return new ProviderRequestException(code, message, 502, Acceptance.UNKNOWN, null);
|
||||
}
|
||||
|
||||
public static ProviderRequestException http(int status, String message) {
|
||||
String code = switch (status) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
// Timeout/conflict/5xx responses do not prove a generation was never accepted.
|
||||
Acceptance acceptance = switch (status) {
|
||||
case 400, 401, 402, 403, 404, 405, 413, 415, 422, 429 -> Acceptance.REJECTED;
|
||||
default -> Acceptance.UNKNOWN;
|
||||
};
|
||||
return new ProviderRequestException(code, message + "(HTTP " + status + ")", 502, acceptance, status);
|
||||
}
|
||||
|
||||
public Acceptance acceptance() { return acceptance; }
|
||||
public Integer upstreamStatus() { return upstreamStatus; }
|
||||
public boolean definitelyNotAccepted() { return acceptance != Acceptance.UNKNOWN; }
|
||||
}
|
||||
+43
-35
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver;
|
||||
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||
import java.net.URI;
|
||||
import java.net.InetAddress;
|
||||
import java.net.http.HttpClient;
|
||||
@@ -60,6 +61,22 @@ public class AsyncProviderClient {
|
||||
}
|
||||
|
||||
public Map<String,Object> submit(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
||||
try {
|
||||
return submitConfigured(providerId, upstreamModel, routeId, taskType, input);
|
||||
} catch (ProviderRequestException exception) {
|
||||
throw exception;
|
||||
} catch (ApiException exception) {
|
||||
// Only these local validation/configuration errors are known to precede generation submission.
|
||||
if (List.of("PROVIDER_NOT_CONFIGURED", "UPSTREAM_KEY_NOT_AVAILABLE", "ASYNC_PROVIDER_NOT_IMPLEMENTED",
|
||||
"MEDIA_INPUT_UNSUPPORTED", "INVALID_MEDIA_URL", "MEDIA_FETCH_FAILED", "MEDIA_TOO_LARGE",
|
||||
"MEDIA_TYPE_UNSUPPORTED", "INVALID_TASK_INPUT").contains(exception.getCode())) {
|
||||
throw ProviderRequestException.notSent(exception.getCode(), exception.getMessage(), exception.getStatus());
|
||||
}
|
||||
throw ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", "上游任务提交结果尚未确认");
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String,Object> submitConfigured(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
||||
if (input == null) input = Map.of();
|
||||
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
||||
Provider provider = provider(providerId, routeId, upstreamModel);
|
||||
@@ -261,8 +278,8 @@ public class AsyncProviderClient {
|
||||
}
|
||||
|
||||
private boolean isNotFound(ApiException exception) {
|
||||
String message = exception.getMessage();
|
||||
return message != null && (message.contains("HTTP 404") || message.contains("HTTP 405"));
|
||||
return exception instanceof ProviderRequestException failure
|
||||
&& (Integer.valueOf(404).equals(failure.upstreamStatus()) || Integer.valueOf(405).equals(failure.upstreamStatus()));
|
||||
}
|
||||
|
||||
/** Normalize OpenAI image/audio response fields to the task result shape. */
|
||||
@@ -287,19 +304,14 @@ public class AsyncProviderClient {
|
||||
}
|
||||
|
||||
private Map<String,Object> requestAudio(String method, String url, String key, Map<String,Object> body, String message) {
|
||||
boolean sending = false;
|
||||
try {
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
||||
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
||||
HttpResponse<byte[]> response = client.send(builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(), HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String code = switch (response.statusCode()) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
|
||||
}
|
||||
HttpRequest request = builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build();
|
||||
sending = true;
|
||||
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||
byte[] bytes = response.body();
|
||||
if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502);
|
||||
String contentType = response.headers().firstValue("Content-Type").orElse("audio/mpeg").split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT);
|
||||
@@ -309,8 +321,7 @@ public class AsyncProviderClient {
|
||||
result.put("mimeType", contentType);
|
||||
result.put("format", String.valueOf(body.getOrDefault("response_format", "mp3")));
|
||||
return result;
|
||||
} catch (ApiException exception) { throw exception; }
|
||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
||||
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -322,6 +333,7 @@ public class AsyncProviderClient {
|
||||
*/
|
||||
private Map<String,Object> requestMultipart(String method, String url, String key, Map<String,Object> fields,
|
||||
String fileField, String media, String message) {
|
||||
boolean sending = false;
|
||||
try {
|
||||
String boundary = "----moyi-" + java.util.UUID.randomUUID();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
@@ -337,6 +349,7 @@ public class AsyncProviderClient {
|
||||
.header("Authorization", "Bearer " + key)
|
||||
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
|
||||
.POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build();
|
||||
sending = true;
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
||||
@@ -345,18 +358,19 @@ public class AsyncProviderClient {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
||||
}
|
||||
return parsed;
|
||||
} catch (ApiException exception) { throw exception; }
|
||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
||||
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||
}
|
||||
|
||||
private ApiException upstreamStatus(int status, String message) {
|
||||
String code = switch (status) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
return new ApiException(code, message + "(HTTP " + status + ")", 502);
|
||||
return ProviderRequestException.http(status, message);
|
||||
}
|
||||
|
||||
private ProviderRequestException requestFailure(Exception exception, boolean sending, String message) {
|
||||
if (exception instanceof InterruptedException) Thread.currentThread().interrupt();
|
||||
if (exception instanceof ProviderRequestException failure) return failure;
|
||||
if (sending) return ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", message);
|
||||
if (exception instanceof ApiException error) return ProviderRequestException.notSent(error.getCode(), error.getMessage(), error.getStatus());
|
||||
return ProviderRequestException.notSent("UPSTREAM_REQUEST_NOT_SENT", "请求准备失败,尚未提交上游生成", 502);
|
||||
}
|
||||
|
||||
private void multipartField(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException {
|
||||
@@ -414,21 +428,16 @@ public class AsyncProviderClient {
|
||||
private record MediaBytes(byte[] bytes, String contentType, String filename) {}
|
||||
|
||||
private Map<String,Object> request(String method, String url, String key, Map<String,Object> body, String message) {
|
||||
boolean sending = false;
|
||||
try {
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
||||
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
||||
if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body)));
|
||||
else builder.GET();
|
||||
HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String code = switch (response.statusCode()) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
|
||||
}
|
||||
HttpRequest request = builder.build();
|
||||
sending = true;
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
||||
Object code = parsed.get("code");
|
||||
if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) {
|
||||
@@ -437,8 +446,7 @@ public class AsyncProviderClient {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
||||
}
|
||||
return parsed;
|
||||
} catch (ApiException exception) { throw exception; }
|
||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
||||
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||
}
|
||||
|
||||
private Provider provider(String id) {
|
||||
|
||||
@@ -13,6 +13,7 @@ import com.moyi.controlplane.gateway.request.ApiRequestProtocol;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestRiskEngine;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.time.OffsetDateTime;
|
||||
@@ -35,6 +36,7 @@ import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Service
|
||||
public class AsyncTaskService {
|
||||
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(AsyncTaskService.class);
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final CatalogService catalogService;
|
||||
@@ -49,6 +51,7 @@ public class AsyncTaskService {
|
||||
private final TransactionTemplate transactions;
|
||||
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
|
||||
private final ApiRequestRiskPolicy requestRisk;
|
||||
private final TaskOutcomeJournal outcomes;
|
||||
|
||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
@@ -104,6 +107,7 @@ public class AsyncTaskService {
|
||||
this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager);
|
||||
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
|
||||
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
|
||||
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -177,7 +181,7 @@ public class AsyncTaskService {
|
||||
// model. Without the transaction advisory lock, concurrent creates
|
||||
// could all observe the same count and exceed maxQueuedPerModel.
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId);
|
||||
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId);
|
||||
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running', 'reconciling')", Integer.class, modelId);
|
||||
if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429);
|
||||
@SuppressWarnings("unchecked") Map<String,Object> frozenPricing=(Map<String,Object>) model.get("pricing");
|
||||
if(AsyncPriceSchedule.tokens(frozenPricing) && !"image".equals(taskType)) throw new ApiException("ASYNC_BILLING_UNSUPPORTED","该媒体类型的 Token 用量结算尚未接入",503);
|
||||
@@ -237,6 +241,11 @@ public class AsyncTaskService {
|
||||
|
||||
@Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}")
|
||||
public void completeMockTasks() {
|
||||
// Retry local settlement from durable observations, never the upstream generation request.
|
||||
for (var completion : outcomes.pendingCompletions()) {
|
||||
try { completeObserved(completion); }
|
||||
catch (RuntimeException exception) { log.warn("Async task {} settlement remains pending", completion.taskId()); }
|
||||
}
|
||||
List<String> taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id"));
|
||||
jdbc.update(
|
||||
"UPDATE model_tasks SET status = 'succeeded', progress = 100, result = CAST(CASE WHEN task_type = '3d' THEN ? ELSE ? END AS jsonb), updated_at = ?, completed_at = ? WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'",
|
||||
@@ -244,7 +253,7 @@ public class AsyncTaskService {
|
||||
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
|
||||
);
|
||||
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||
for (Map<String,Object> row : pending) pollProviderTask(row);
|
||||
dispatchQueuedTasks();
|
||||
}
|
||||
@@ -257,36 +266,43 @@ public class AsyncTaskService {
|
||||
*/
|
||||
private void dispatchQueuedTasks() {
|
||||
if (providerClient == null) return;
|
||||
// A process can die after claiming a task but before receiving an
|
||||
// upstream task id. Fail those claims after a grace period. Retrying
|
||||
// them blindly could submit the same generation twice when the
|
||||
// upstream accepted the request just before the process died.
|
||||
// A lost submit response is ambiguous, not a failed generation.
|
||||
// Quarantine stale claims without retrying or releasing financial holds.
|
||||
recoverInterruptedClaims();
|
||||
for (int i = 0; i < 50; i++) {
|
||||
Map<String,Object> task = claimQueuedTask();
|
||||
if (task == null) return;
|
||||
String taskId = String.valueOf(task.get("id"));
|
||||
usage.updateTask(taskId, "running", null, null, null);
|
||||
Map<String,Object> upstream;
|
||||
try {
|
||||
Map<String,Object> input = taskInput(task);
|
||||
Map<String,Object> upstream = submitProviderTask(task, input);
|
||||
// Some OpenAI-compatible image/audio endpoints return the
|
||||
// generated media in the submit response rather than creating
|
||||
// a pollable job. Persist that result and settle the task in
|
||||
// the same worker transaction.
|
||||
upstream = submitProviderTask(task, input);
|
||||
} catch (RuntimeException exception) {
|
||||
if (exception instanceof ProviderRequestException failure && failure.definitelyNotAccepted()) {
|
||||
completeObserved(outcomes.observeTerminal(taskId, "failed", null, failure.getCode(), failure.getMessage()));
|
||||
} else {
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_ACCEPTANCE_UNKNOWN", "上游任务是否已受理尚未确认;冻结额度保留,请联系管理员核账,勿重复提交", false);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
TaskOutcomeJournal.Completion completion = null;
|
||||
try {
|
||||
// Journal a received result before touching the ledger. A failed settlement
|
||||
// must only retry locally, never generate again or release its reservation.
|
||||
if (upstream.get("_moyiInlineResult") instanceof Map<?,?> inline) {
|
||||
Map<String,Object> result = new LinkedHashMap<>();
|
||||
inline.forEach((key, value) -> result.put(String.valueOf(key), value));
|
||||
finishTask(taskId, "succeeded", 100, result, null, null);
|
||||
continue;
|
||||
completion = outcomes.observeTerminal(taskId, "succeeded", result, null, null);
|
||||
} else {
|
||||
OffsetDateTime submittedAt = now();
|
||||
outcomes.accepted(taskId, providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"),
|
||||
submittedAt, submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)));
|
||||
}
|
||||
OffsetDateTime submittedAt = now();
|
||||
jdbc.update("UPDATE model_tasks SET provider_task_id=?, provider_key_fingerprint=?, provider_deadline_at=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"), submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)), submittedAt, taskId);
|
||||
} catch (RuntimeException exception) {
|
||||
String code = exception instanceof ApiException error ? error.getCode() : "UPSTREAM_TASK_SUBMIT_FAILED";
|
||||
String message = exception instanceof ApiException error ? error.getMessage() : "上游异步任务创建失败";
|
||||
failTask(taskId, code, message);
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_RECEIPT_UNKNOWN", "已调用上游,但任务回执无法确认或保存;冻结额度保留,等待核账,勿重复提交", false);
|
||||
}
|
||||
// Deliberately outside the provider exception handler.
|
||||
completeObserved(completion);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,18 +322,18 @@ public class AsyncTaskService {
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status='running') < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
|
||||
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status IN ('running','reconciling')) < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
|
||||
AND (
|
||||
? <= 0
|
||||
OR COALESCE(m.max_concurrency, ?) <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_model
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status IN ('running','reconciling'))
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
)
|
||||
AND (
|
||||
? <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_key
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status IN ('running','reconciling'))
|
||||
< ?
|
||||
)
|
||||
ORDER BY t.created_at
|
||||
@@ -332,19 +348,26 @@ public class AsyncTaskService {
|
||||
if(!gatewayGuard.tryTaskUserSlot((UUID)row.get("api_key_id"),row.get("id").toString(),providerTimeoutMs+180000)) return null;
|
||||
try {jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));}
|
||||
catch(RuntimeException e) {gatewayGuard.releaseTaskUserSlot((UUID)row.get("user_id"),row.get("id").toString());throw e;}
|
||||
usage.updateTask(row.get("id").toString(), "running", null, null, null);
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
private void recoverInterruptedClaims() {
|
||||
List<String> taskIds = inTransaction(() -> jdbc.queryForList(
|
||||
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
|
||||
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND completion_status IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
|
||||
String.class));
|
||||
for (String taskId : taskIds) {
|
||||
failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交");
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游提交回执丢失或进程中断;冻结额度保留,等待核账,勿重复提交", true);
|
||||
}
|
||||
}
|
||||
|
||||
private void completeObserved(TaskOutcomeJournal.Completion completion) {
|
||||
if (completion == null) return;
|
||||
finishTask(completion.taskId(), completion.status(), "succeeded".equals(completion.status()) ? 100 : 0,
|
||||
completion.result(), completion.code(), completion.message());
|
||||
}
|
||||
|
||||
private void failTask(String taskId, String code, String message) {
|
||||
finishTask(taskId, "failed", 0, null, code, message);
|
||||
}
|
||||
@@ -354,8 +377,10 @@ public class AsyncTaskService {
|
||||
// crash after marking failed/succeeded leaves a hold that polling never
|
||||
// revisits. The conditional update also prevents duplicate settlement.
|
||||
inTransaction(() -> {
|
||||
Map<String,Object> task=jdbc.queryForMap("SELECT status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
|
||||
if(!List.of("queued","running").contains(task.get("status"))) return null;
|
||||
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
|
||||
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
|
||||
// A late failure must not override a durably observed success whose settlement is pending.
|
||||
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
|
||||
AsyncPriceSchedule.Bill bill=null;
|
||||
String finalCode=code,finalMessage=message;
|
||||
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
|
||||
@@ -363,7 +388,7 @@ public class AsyncTaskService {
|
||||
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
|
||||
}
|
||||
int changed = jdbc.update(
|
||||
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
|
||||
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
|
||||
status, progress, result == null ? null : writeJson(result),
|
||||
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
|
||||
if (changed == 0) return null;
|
||||
@@ -397,6 +422,7 @@ public class AsyncTaskService {
|
||||
task.put("progress", resultSet.getInt("progress"));
|
||||
task.put("createdAt", resultSet.getObject("created_at", OffsetDateTime.class));
|
||||
task.put("updatedAt", resultSet.getObject("updated_at", OffsetDateTime.class));
|
||||
task.put("requiresReconciliation", "reconciling".equals(resultSet.getString("status")));
|
||||
putIfPresent(task, "costAmount", resultSet, "cost_amount");
|
||||
String result = resultSet.getString("result");
|
||||
if (result != null) task.put("result", readMap(result));
|
||||
@@ -440,9 +466,12 @@ public class AsyncTaskService {
|
||||
|
||||
private String providerTaskId(Map<String,Object> response) {
|
||||
Object direct = response.get("id");
|
||||
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id");
|
||||
if (direct == null) throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回异步任务编号", 502);
|
||||
return String.valueOf(direct);
|
||||
if (direct == null) direct = response.get("task_id");
|
||||
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id") == null ? data.get("id") : data.get("task_id");
|
||||
if (!(direct instanceof String value) || value.isBlank() || value.length() > 160 || value.chars().anyMatch(Character::isISOControl)) {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回有效的异步任务编号", 502);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private void pollProviderTask(Map<String,Object> row) {
|
||||
@@ -456,7 +485,7 @@ public class AsyncTaskService {
|
||||
deadline = createdAt.plus(java.time.Duration.ofMillis(providerTimeoutMs));
|
||||
}
|
||||
if (!now().isBefore(deadline)) {
|
||||
failTask(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;平台冻结额度已释放。上游可能仍在执行,请先核实后再提交");
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;冻结额度保留,等待管理员核账,勿重复提交", false);
|
||||
return;
|
||||
}
|
||||
Map<String,Object> response;
|
||||
@@ -464,7 +493,7 @@ public class AsyncTaskService {
|
||||
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("model_id")), String.valueOf(row.get("upstream_model")), String.valueOf(row.get("provider_task_id")), row.get("provider_key_fingerprint") == null ? null : String.valueOf(row.get("provider_key_fingerprint")));
|
||||
} catch (RuntimeException exception) {
|
||||
if (exception instanceof ApiException error && "UPSTREAM_KEY_NOT_AVAILABLE".equals(error.getCode())) {
|
||||
failTask(taskId, error.getCode(), error.getMessage());
|
||||
outcomes.requireReconciliation(taskId, error.getCode(), "提交任务使用的上游凭据已不可用,执行结果未知;冻结额度保留,等待核账", false);
|
||||
return;
|
||||
}
|
||||
// Even a permanent authentication/configuration error only proves
|
||||
@@ -487,9 +516,9 @@ public class AsyncTaskService {
|
||||
if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) {
|
||||
// Persistence failures must roll back and remain pollable, rather
|
||||
// than being mistaken for upstream failures and refunded.
|
||||
finishTask(taskId, status, progress, result,
|
||||
completeObserved(outcomes.observeTerminal(taskId, status, result,
|
||||
"failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null,
|
||||
"failed".equals(status) ? "上游异步任务失败" : null);
|
||||
"failed".equals(status) ? "上游异步任务失败" : null));
|
||||
} else {
|
||||
updatePollingState(taskId, progress, null, null);
|
||||
}
|
||||
@@ -497,7 +526,7 @@ public class AsyncTaskService {
|
||||
|
||||
private void updatePollingState(String taskId, Integer progress, String code, String message) {
|
||||
inTransaction(() -> {
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running')",
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL",
|
||||
progress, code == null ? null : writeJson(Map.of("code", code, "message", message, "retryable", true)), now(), taskId);
|
||||
if (changed > 0) usage.updateTask(taskId, "running", null, code, message);
|
||||
return null;
|
||||
@@ -515,10 +544,12 @@ public class AsyncTaskService {
|
||||
Object raw = response.get("status");
|
||||
if (raw == null && response.get("data") instanceof Map<?,?> data) raw = data.get("status");
|
||||
String value = String.valueOf(raw == null ? "running" : raw).toLowerCase(java.util.Locale.ROOT);
|
||||
if (value.contains("success") || value.equals("completed")) return "succeeded";
|
||||
if (value.contains("fail") || value.equals("error") || value.equals("banned")) return "failed";
|
||||
if (value.contains("cancel")) return "cancelled";
|
||||
return "running";
|
||||
return switch (value) {
|
||||
case "success", "succeeded", "completed" -> "succeeded";
|
||||
case "failed", "failure", "error", "banned" -> "failed";
|
||||
case "cancelled", "canceled" -> "cancelled";
|
||||
default -> "running";
|
||||
};
|
||||
}
|
||||
|
||||
private int progress(Map<String,Object> response, String status) {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
/** Durable provider observations only. No upstream calls, wallet mutation, retry or refund policy. */
|
||||
final class TaskOutcomeJournal {
|
||||
record Completion(String taskId, String status, Map<String,Object> result, String code, String message) { }
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper mapper;
|
||||
private final TransactionTemplate transactions;
|
||||
private final UsageAuditService usage;
|
||||
|
||||
TaskOutcomeJournal(JdbcTemplate jdbc, ObjectMapper mapper, TransactionTemplate transactions, UsageAuditService usage) {
|
||||
this.jdbc = jdbc;
|
||||
this.mapper = mapper;
|
||||
this.transactions = transactions;
|
||||
this.usage = usage;
|
||||
}
|
||||
|
||||
Completion observeTerminal(String taskId, String status, Map<String,Object> result, String code, String message) {
|
||||
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal outcome required");
|
||||
return transaction(() -> {
|
||||
var row = jdbc.queryForMap("SELECT status,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
if (!List.of("queued", "running", "reconciling").contains(row.get("status"))) return null;
|
||||
if (row.get("completion_status") != null) return completion(taskId, row);
|
||||
jdbc.update("UPDATE model_tasks SET completion_status=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),updated_at=now() WHERE id=?",
|
||||
status, json(result), code == null ? null : json(Map.of("code", code, "message", message)), taskId);
|
||||
return new Completion(taskId, status, result, code, message);
|
||||
});
|
||||
}
|
||||
|
||||
List<Completion> pendingCompletions() {
|
||||
return jdbc.queryForList("SELECT id,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE status IN ('running','reconciling') AND completion_status IS NOT NULL ORDER BY updated_at LIMIT 50")
|
||||
.stream().map(row -> completion(String.valueOf(row.get("id")), row)).toList();
|
||||
}
|
||||
|
||||
void accepted(String taskId, String providerTaskId, Object keyFingerprint, OffsetDateTime submittedAt, OffsetDateTime deadline) {
|
||||
transaction(() -> {
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=? WHERE id=? AND status IN ('running','reconciling') AND provider_task_id IS NULL AND completion_status IS NULL",
|
||||
providerTaskId, keyFingerprint, deadline, submittedAt, taskId);
|
||||
if (changed > 0) usage.updateTask(taskId, "running", null, null, null);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
void requireReconciliation(String taskId, String code, String message, boolean interruptedClaim) {
|
||||
transaction(() -> {
|
||||
String claimGuard = interruptedClaim
|
||||
? " AND provider_task_id IS NULL AND updated_at < now()-interval '2 minutes'" : "";
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='reconciling',error=CAST(? AS jsonb),reconciliation_required_at=now(),updated_at=now() WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL" + claimGuard,
|
||||
json(Map.of("code", code, "message", message, "retryable", false, "requiresReconciliation", true)), taskId);
|
||||
if (changed > 0) {
|
||||
usage.updateTask(taskId, "reconciling", null, code, message);
|
||||
jdbc.update("UPDATE usage_events SET cost_status='pending',cost_amount=NULL WHERE provider_task_id=? AND event_kind='task'", taskId);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private Completion completion(String taskId, Map<String,Object> row) {
|
||||
Map<String,Object> error = map(row.get("error"));
|
||||
return new Completion(taskId, String.valueOf(row.get("completion_status")), map(row.get("result")),
|
||||
error == null ? null : (String) error.get("code"), error == null ? null : (String) error.get("message"));
|
||||
}
|
||||
|
||||
private Map<String,Object> map(Object value) {
|
||||
if (value == null) return null;
|
||||
try { return mapper.readValue(String.valueOf(value), new TypeReference<>() {}); }
|
||||
catch (JsonProcessingException exception) { throw new IllegalStateException("Invalid persisted provider observation"); }
|
||||
}
|
||||
|
||||
private String json(Object value) {
|
||||
if (value == null) return null;
|
||||
try { return mapper.writeValueAsString(value); }
|
||||
catch (JsonProcessingException exception) { throw new IllegalStateException("Provider observation cannot be encoded"); }
|
||||
}
|
||||
|
||||
private <T> T transaction(Supplier<T> work) {
|
||||
return transactions == null ? work.get() : transactions.execute(status -> work.get());
|
||||
}
|
||||
}
|
||||
@@ -157,15 +157,17 @@ public class UsageAuditService {
|
||||
WITH pending AS (
|
||||
SELECT model_id,
|
||||
COUNT(*) FILTER (WHERE status='running') AS running,
|
||||
COUNT(*) FILTER (WHERE status='queued') AS queued
|
||||
FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id
|
||||
COUNT(*) FILTER (WHERE status='queued') AS queued,
|
||||
COUNT(*) FILTER (WHERE status='reconciling') AS reconciling
|
||||
FROM model_tasks WHERE status IN ('queued','running','reconciling') GROUP BY model_id
|
||||
)
|
||||
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
|
||||
CASE WHEN m.max_concurrency = 0 THEN 0
|
||||
ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency,
|
||||
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
|
||||
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued
|
||||
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued,
|
||||
COALESCE(pending.reconciling, 0) AS reconciling
|
||||
FROM model_catalog m JOIN model_providers p ON p.id=m.provider_id
|
||||
JOIN model_definitions d ON d.id=m.logical_model_id
|
||||
LEFT JOIN pending ON pending.model_id=m.id
|
||||
@@ -182,6 +184,7 @@ public class UsageAuditService {
|
||||
group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0);
|
||||
group.put("running", rs.getLong("running"));
|
||||
group.put("queued", rs.getLong("queued"));
|
||||
group.put("reconciling", rs.getLong("reconciling"));
|
||||
return group;
|
||||
}, defaultModelLimit);
|
||||
List<String> synchronousIds = groups.stream().filter(group -> !"async".equals(group.get("mode"))).map(group -> group.get("id").toString()).toList();
|
||||
@@ -193,12 +196,13 @@ public class UsageAuditService {
|
||||
group.put("running", current == null ? null : current.running());
|
||||
group.put("queued", current == null ? null : current.queued());
|
||||
}
|
||||
// Keep disabled groups visible while their queued or running work still exists.
|
||||
// Keep disabled groups visible while any work or unresolved financial hold still exists.
|
||||
groups = groups.stream().filter(group -> Boolean.TRUE.equals(group.get("routeEnabled"))
|
||||
|| positive(group.get("running")) || positive(group.get("queued"))).toList();
|
||||
|| positive(group.get("running")) || positive(group.get("queued")) || positive(group.get("reconciling"))).toList();
|
||||
result.put("capacityGroups", groups);
|
||||
result.put("running", totalCapacity(groups, "running"));
|
||||
result.put("waiting", totalCapacity(groups, "queued"));
|
||||
result.put("reconciling", totalCapacity(groups, "reconciling"));
|
||||
result.put("capacityConfig", Map.of(
|
||||
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
|
||||
"defaultModelConcurrency", defaultModelLimit,
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Unknown provider acceptance is not a failed generation and must retain its financial holds.
|
||||
ALTER TABLE model_tasks DROP CONSTRAINT model_tasks_status_check;
|
||||
ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_status_check
|
||||
CHECK (status IN ('queued','running','reconciling','succeeded','failed','cancelled'));
|
||||
ALTER TABLE usage_events DROP CONSTRAINT usage_events_status_check;
|
||||
ALTER TABLE usage_events ADD CONSTRAINT usage_events_status_check
|
||||
CHECK (status IN ('queued','running','reconciling','succeeded','failed','cancelled'));
|
||||
ALTER TABLE model_tasks ADD COLUMN reconciliation_required_at timestamptz;
|
||||
-- A normalized observed terminal outcome survives a crash or transient ledger/audit write failure.
|
||||
-- The result itself uses the existing result column; no second copy of generated media is stored.
|
||||
ALTER TABLE model_tasks ADD COLUMN completion_status varchar(16)
|
||||
CHECK (completion_status IN ('succeeded','failed','cancelled'));
|
||||
CREATE INDEX model_tasks_pending_completion_idx ON model_tasks(updated_at)
|
||||
WHERE completion_status IS NOT NULL AND status IN ('running','reconciling');
|
||||
CREATE INDEX model_tasks_reconciliation_idx ON model_tasks(reconciliation_required_at)
|
||||
WHERE status='reconciling';
|
||||
+68
-1
@@ -11,6 +11,7 @@ import static org.mockito.Mockito.when;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -207,7 +208,7 @@ class AsyncProviderClientTest {
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "503,UPSTREAM_PROVIDER_ERROR"})
|
||||
@CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "408,UPSTREAM_PROVIDER_ERROR", "409,UPSTREAM_PROVIDER_ERROR", "503,UPSTREAM_PROVIDER_ERROR"})
|
||||
void httpErrorsExposeOnlySafeCodes(int status, String code) throws Exception {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/", exchange -> {
|
||||
@@ -221,6 +222,7 @@ class AsyncProviderClientTest {
|
||||
ApiException submit = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair"));
|
||||
assertEquals(code, submit.getCode());
|
||||
assertEquals("上游 3D 任务创建失败(HTTP " + status + ")", submit.getMessage());
|
||||
assertEquals(status != 408 && status != 409 && status < 500, ((ProviderRequestException) submit).definitelyNotAccepted());
|
||||
ApiException poll = assertThrows(ApiException.class, () -> client.poll("provider-vast", "task-test"));
|
||||
assertEquals(code, poll.getCode());
|
||||
assertEquals("上游异步任务查询失败(HTTP " + status + ")", poll.getMessage());
|
||||
@@ -229,6 +231,71 @@ class AsyncProviderClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test void successfulHttpResponseWithMalformedBodyHasUnknownAcceptance() throws Exception {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/", exchange -> {
|
||||
byte[] body = "not-json; private upstream detail".getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(200, body.length);
|
||||
try (var output = exchange.getResponseBody()) { output.write(body); }
|
||||
});
|
||||
server.start();
|
||||
try {
|
||||
var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair"));
|
||||
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
|
||||
org.junit.jupiter.api.Assertions.assertFalse(failure.getMessage().contains("private upstream detail"));
|
||||
} finally { server.stop(0); }
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@CsvSource({"image,gpt-image-2", "video,sora-2", "audio,tts-1"})
|
||||
void compatibleSubmit5xxRemainsUnknownAndNeverUsesFallbackRoute(String type, String model) throws Exception {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
var calls = new java.util.concurrent.atomic.AtomicInteger();
|
||||
server.createContext("/", exchange -> {
|
||||
calls.incrementAndGet();
|
||||
exchange.getRequestBody().readAllBytes();
|
||||
byte[] body = "private account details".getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(503, body.length);
|
||||
try (var output = exchange.getResponseBody()) { output.write(body); }
|
||||
});
|
||||
server.start();
|
||||
try {
|
||||
var jdbc = mock(JdbcTemplate.class);
|
||||
var cipher = new SecretCipher("test-encryption");
|
||||
when(jdbc.queryForMap(any(String.class), eq("relay"))).thenReturn(Map.of(
|
||||
"code", "relay", "protocol", "openai-compatible", "base_url", "http://127.0.0.1:" + server.getAddress().getPort(), "credential_ciphertext", cipher.encrypt("test-key")));
|
||||
var client = new AsyncProviderClient(jdbc, cipher, new ObjectMapper(), mock(Environment.class));
|
||||
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("relay", model, type, "chair"));
|
||||
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
|
||||
assertEquals(1, calls.get());
|
||||
} finally { server.stop(0); }
|
||||
}
|
||||
|
||||
@Test void acceptedPostWithDroppedResponseIsUnknownAndIsNotTransparentlyRepeated() throws Exception {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
var calls = new java.util.concurrent.atomic.AtomicInteger();
|
||||
server.createContext("/", exchange -> {
|
||||
exchange.getRequestBody().readAllBytes();
|
||||
calls.incrementAndGet();
|
||||
exchange.close(); // Simulate acceptance followed by loss of the HTTP response.
|
||||
});
|
||||
server.start();
|
||||
try {
|
||||
var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair"));
|
||||
assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance());
|
||||
assertEquals(1, calls.get());
|
||||
} finally { server.stop(0); }
|
||||
}
|
||||
|
||||
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
|
||||
var client = clientFor("http://127.0.0.1:1");
|
||||
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
|
||||
assertEquals(ProviderRequestException.Acceptance.NOT_SENT, failure.acceptance());
|
||||
assertEquals("INVALID_MEDIA_URL", failure.getCode());
|
||||
}
|
||||
|
||||
private AsyncProviderClient clientFor(String baseUrl) {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
SecretCipher cipher = new SecretCipher("test-encryption");
|
||||
|
||||
+159
-6
@@ -15,6 +15,7 @@ import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
|
||||
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -42,6 +43,7 @@ class AsyncTaskPostgresTest {
|
||||
private AsyncProviderClient provider;
|
||||
private AsyncTaskService tasks;
|
||||
private CatalogService catalog;
|
||||
private GatewayGuard guard;
|
||||
|
||||
@BeforeEach void setup() {
|
||||
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
|
||||
@@ -52,7 +54,7 @@ class AsyncTaskPostgresTest {
|
||||
jdbc = new JdbcTemplate(source);
|
||||
usage = spy(new UsageAuditService(jdbc, json));
|
||||
provider = mock(AsyncProviderClient.class);
|
||||
GatewayGuard guard=mock(GatewayGuard.class);
|
||||
guard=mock(GatewayGuard.class);
|
||||
when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true);
|
||||
catalog = mock(CatalogService.class);
|
||||
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000,
|
||||
@@ -152,7 +154,10 @@ class AsyncTaskPostgresTest {
|
||||
@ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"})
|
||||
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) {
|
||||
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId);
|
||||
when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(new ApiException(code,"上游任务未接受",503));
|
||||
when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(
|
||||
"PROVIDER_NOT_CONFIGURED".equals(code)
|
||||
? ProviderRequestException.notSent(code, "上游任务未接受", 503)
|
||||
: ProviderRequestException.http(401, "上游任务未接受"));
|
||||
tasks.completeMockTasks();
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("failed",status());
|
||||
@@ -216,19 +221,167 @@ class AsyncTaskPostgresTest {
|
||||
assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId));
|
||||
}
|
||||
|
||||
@Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed() {
|
||||
@Test void deadlineRetainsHoldWithoutClaimingTheUpstreamFailed() {
|
||||
jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId);
|
||||
poll(); poll();
|
||||
assertEquals("failed",status());
|
||||
assertEquals("reconciling",status());
|
||||
assertEquals(new BigDecimal("10.00000000"),wallet("balance"));
|
||||
assertEquals(0,wallet("reserved_balance").signum());
|
||||
assertEquals(0,cost().signum());
|
||||
assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance"));
|
||||
assertNull(cost());
|
||||
String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId);
|
||||
assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT"));
|
||||
assertTrue(error.contains("执行结果未知"));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints={408,409,500,502,503,504})
|
||||
void uncertainSubmitResponseNeverRefundsOrResubmits(int httpStatus) {
|
||||
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
|
||||
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||
.thenThrow(ProviderRequestException.http(httpStatus, "safe local message"));
|
||||
tasks.completeMockTasks();
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("reconciling", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertNull(cost());
|
||||
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||
assertEquals("pending", jdbc.queryForObject("SELECT cost_status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||
assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId));
|
||||
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test void lostSubmitResponseAndInterruptedClaimRequireReconciliationWithoutReplay() {
|
||||
jdbc.update("UPDATE model_tasks SET provider_task_id=NULL,updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
|
||||
tasks.completeMockTasks();
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("reconciling", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertNull(cost());
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void missingTaskReceiptRetainsFinancialAndBudgetHolds() {
|
||||
var created = create("missing-receipt", Map.of("model", "model-mock-3d", "prompt", "chair"));
|
||||
String id = String.valueOf(created.get("id"));
|
||||
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap())).thenReturn(Map.of("accepted", true));
|
||||
tasks.completeMockTasks();
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id));
|
||||
assertEquals(new BigDecimal("2.88000000"), wallet("reserved_balance"));
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id));
|
||||
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap());
|
||||
}
|
||||
|
||||
@Test void genericSubmitFailureIsNotTreatedAsEvidenceOfRejection() {
|
||||
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
|
||||
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||
.thenThrow(new IllegalStateException("network connection lost; private details"));
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("reconciling", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertFalse(jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?", String.class, taskId).contains("private details"));
|
||||
}
|
||||
|
||||
@Test void missingPollingCredentialDoesNotRefundAnAlreadyAcceptedJob() {
|
||||
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
|
||||
.thenThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "sensitive detail", 503));
|
||||
poll();
|
||||
assertEquals("reconciling", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertNull(cost());
|
||||
}
|
||||
|
||||
@Test void inlineSuccessSurvivesSettlementFailureAndRecoversWithoutRegenerating() {
|
||||
jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId);
|
||||
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||
.thenReturn(Map.of("_moyiInlineResult", Map.of("url", "https://example.test/generated.glb")));
|
||||
doThrow(new IllegalStateException("simulated audit failure")).when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull());
|
||||
assertThrows(IllegalStateException.class, () -> tasks.completeMockTasks());
|
||||
assertEquals("running", status());
|
||||
assertEquals("succeeded", jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId));
|
||||
fail(); // A stale error cannot refund an already observed success before the retry settles it.
|
||||
assertEquals("running", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertNull(cost());
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
|
||||
doCallRealMethod().when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull());
|
||||
tasks.completeMockTasks();
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("succeeded", status());
|
||||
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||
assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId));
|
||||
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings={"unsuccessful","not_failed","cancelling","unknown"})
|
||||
void ambiguousProviderStatusCannotTriggerSettlementOrRefund(String state) {
|
||||
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
|
||||
.thenReturn(Map.of("status", state));
|
||||
poll();
|
||||
assertEquals("running", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertNull(cost());
|
||||
}
|
||||
|
||||
@Test void interruptedClaimCheckCannotQuarantineALaterAcceptedReceipt() {
|
||||
var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage);
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId);
|
||||
journal.requireReconciliation(taskId, "STALE_WORKER", "stale claim check", true);
|
||||
assertEquals("running", status(), "An upstream task ID proves this is no longer an interrupted claim");
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
}
|
||||
|
||||
@Test void reconciliationAndUsageAuditRollBackTogetherWithoutTouchingTheHold() {
|
||||
var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage);
|
||||
doThrow(new IllegalStateException("audit offline")).when(usage).updateTask(eq(taskId), eq("reconciling"), isNull(), anyString(), anyString());
|
||||
assertThrows(IllegalStateException.class, () -> journal.requireReconciliation(taskId, "UNKNOWN", "result unknown", false));
|
||||
assertEquals("running", status());
|
||||
assertEquals("running", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertNull(cost());
|
||||
}
|
||||
|
||||
@Test void failedTaskReplayDoesNotRequeueOrReserveAgain() {
|
||||
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "will be rejected");
|
||||
var first = create("rejected-task", input);
|
||||
String id = String.valueOf(first.get("id"));
|
||||
when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap()))
|
||||
.thenThrow(ProviderRequestException.http(429, "provider rejected"));
|
||||
tasks.completeMockTasks();
|
||||
BigDecimal held = wallet("reserved_balance");
|
||||
var replay = create("rejected-task", input);
|
||||
assertEquals(id, replay.get("id"));
|
||||
assertEquals("failed", replay.get("status"));
|
||||
assertEquals(held, wallet("reserved_balance"));
|
||||
assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id));
|
||||
verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap());
|
||||
}
|
||||
|
||||
@Test void pendingReconciliationConsumesAsyncConcurrencyRatherThanOpeningAnotherSlot() {
|
||||
tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000, false,
|
||||
guard, usage, provider, new DataSourceTransactionManager(jdbc.getDataSource()),
|
||||
new com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy(com.moyi.controlplane.gateway.request.ApiRequestRiskEngine.allowAll()));
|
||||
jdbc.update("UPDATE model_tasks SET status='reconciling' WHERE id=?", taskId);
|
||||
jdbc.update("UPDATE model_catalog SET max_concurrency=1 WHERE id='model-mock-3d'");
|
||||
String id = String.valueOf(create("next-while-unknown", Map.of("model", "model-mock-3d", "prompt", "another chair")).get("id"));
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("queued", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void documentedSucceededStatusSettlesInsteadOfPollingForever() {
|
||||
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null))
|
||||
.thenReturn(Map.of("status", "succeeded", "url", "https://example.test/generated.glb"));
|
||||
poll();
|
||||
assertEquals("succeeded", status());
|
||||
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||
}
|
||||
|
||||
@Test void confirmedUpstreamFailureReleasesHoldImmediately() {
|
||||
when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)).thenReturn(Map.of("data",Map.of("status","failed")));
|
||||
poll();
|
||||
|
||||
+3
-1
@@ -39,7 +39,7 @@ class UsageCapacityPostgresTest {
|
||||
UUID user = UUID.randomUUID(), key = UUID.randomUUID();
|
||||
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'capacity-user','username','not-loginable','Capacity user')", user);
|
||||
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Capacity test','local-test',?)", key, user, "c".repeat(64));
|
||||
for (String state : List.of("running", "queued", "succeeded")) {
|
||||
for (String state : List.of("running", "queued", "reconciling", "succeeded")) {
|
||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,created_at) VALUES (?,?,?,'capacity-route-105','3d',?,now()-interval '1 day')", "task-" + state, user, key, state);
|
||||
}
|
||||
}
|
||||
@@ -67,9 +67,11 @@ class UsageCapacityPostgresTest {
|
||||
Map<String, Object> async = groups.stream().filter(group -> group.get("id").equals("capacity-route-105")).findFirst().orElseThrow();
|
||||
assertEquals(1L, async.get("running"), "A task older than ten minutes must remain in live capacity");
|
||||
assertEquals(1L, async.get("queued"), "Running tasks must not also count as queued");
|
||||
assertEquals(1L, async.get("reconciling"), "Unknown upstream outcomes must remain visible, not be counted as failed or idle");
|
||||
assertEquals(7, async.get("limit"), "Null route limits use the real configured model default");
|
||||
assertEquals(3L, summary.get("running"));
|
||||
assertEquals(4L, summary.get("waiting"));
|
||||
assertEquals(1L, summary.get("reconciling"));
|
||||
Map<?, ?> config = (Map<?, ?>) summary.get("capacityConfig");
|
||||
assertEquals(4500L, config.get("queueWaitMs"));
|
||||
assertEquals(3, config.get("defaultKeyConcurrency"));
|
||||
|
||||
Reference in new issue
Block a user