fix: retain async holds for unknown provider outcomes

This commit is contained in:
xpp committed 2026-10-21 18:57:35 +08:00
1 parent 76265845dc
commit 5be06a97c1
18 files changed
+574 -108

No files matched your search

@@ -383,6 +383,7 @@ public class CatalogService {
result.put("tasks", Map.of(
"queued", taskCounts.getOrDefault("queued", 0L),
"running", taskCounts.getOrDefault("running", 0L),
"reconciling", taskCounts.getOrDefault("reconciling", 0L),
"succeeded", taskCounts.getOrDefault("succeeded", 0L),
"failed", taskCounts.getOrDefault("failed", 0L)
));
@@ -0,0 +1,44 @@
package com.moyi.controlplane.provider.transport;
import com.moyi.controlplane.common.ApiException;
/** HTTP boundary facts, not a retry policy. Unknown acceptance must never authorize replay/refund. */
public final class ProviderRequestException extends ApiException {
public enum Acceptance { NOT_SENT, REJECTED, UNKNOWN }
private final Acceptance acceptance;
private final Integer upstreamStatus;
private ProviderRequestException(String code, String message, int status, Acceptance acceptance, Integer upstreamStatus) {
super(code, message, status);
this.acceptance = acceptance;
this.upstreamStatus = upstreamStatus;
}
public static ProviderRequestException notSent(String code, String message, int status) {
return new ProviderRequestException(code, message, status, Acceptance.NOT_SENT, null);
}
public static ProviderRequestException unknown(String code, String message) {
return new ProviderRequestException(code, message, 502, Acceptance.UNKNOWN, null);
}
public static ProviderRequestException http(int status, String message) {
String code = switch (status) {
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
case 429 -> "UPSTREAM_RATE_LIMITED";
default -> "UPSTREAM_PROVIDER_ERROR";
};
// Timeout/conflict/5xx responses do not prove a generation was never accepted.
Acceptance acceptance = switch (status) {
case 400, 401, 402, 403, 404, 405, 413, 415, 422, 429 -> Acceptance.REJECTED;
default -> Acceptance.UNKNOWN;
};
return new ProviderRequestException(code, message + "(HTTP " + status + ")", 502, acceptance, status);
}
public Acceptance acceptance() { return acceptance; }
public Integer upstreamStatus() { return upstreamStatus; }
public boolean definitelyNotAccepted() { return acceptance != Acceptance.UNKNOWN; }
}
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.security.SecretCipher;
import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver;
import com.moyi.controlplane.provider.transport.ProviderRequestException;
import java.net.URI;
import java.net.InetAddress;
import java.net.http.HttpClient;
@@ -60,6 +61,22 @@ public class AsyncProviderClient {
}
public Map<String,Object> submit(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
try {
return submitConfigured(providerId, upstreamModel, routeId, taskType, input);
} catch (ProviderRequestException exception) {
throw exception;
} catch (ApiException exception) {
// Only these local validation/configuration errors are known to precede generation submission.
if (List.of("PROVIDER_NOT_CONFIGURED", "UPSTREAM_KEY_NOT_AVAILABLE", "ASYNC_PROVIDER_NOT_IMPLEMENTED",
"MEDIA_INPUT_UNSUPPORTED", "INVALID_MEDIA_URL", "MEDIA_FETCH_FAILED", "MEDIA_TOO_LARGE",
"MEDIA_TYPE_UNSUPPORTED", "INVALID_TASK_INPUT").contains(exception.getCode())) {
throw ProviderRequestException.notSent(exception.getCode(), exception.getMessage(), exception.getStatus());
}
throw ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", "上游任务提交结果尚未确认");
}
}
private Map<String,Object> submitConfigured(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
if (input == null) input = Map.of();
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
Provider provider = provider(providerId, routeId, upstreamModel);
@@ -261,8 +278,8 @@ public class AsyncProviderClient {
}
private boolean isNotFound(ApiException exception) {
String message = exception.getMessage();
return message != null && (message.contains("HTTP 404") || message.contains("HTTP 405"));
return exception instanceof ProviderRequestException failure
&& (Integer.valueOf(404).equals(failure.upstreamStatus()) || Integer.valueOf(405).equals(failure.upstreamStatus()));
}
/** Normalize OpenAI image/audio response fields to the task result shape. */
@@ -287,19 +304,14 @@ public class AsyncProviderClient {
}
private Map<String,Object> requestAudio(String method, String url, String key, Map<String,Object> body, String message) {
boolean sending = false;
try {
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
HttpResponse<byte[]> response = client.send(builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(), HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
String code = switch (response.statusCode()) {
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
case 429 -> "UPSTREAM_RATE_LIMITED";
default -> "UPSTREAM_PROVIDER_ERROR";
};
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
}
HttpRequest request = builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build();
sending = true;
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
byte[] bytes = response.body();
if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502);
String contentType = response.headers().firstValue("Content-Type").orElse("audio/mpeg").split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT);
@@ -309,8 +321,7 @@ public class AsyncProviderClient {
result.put("mimeType", contentType);
result.put("format", String.valueOf(body.getOrDefault("response_format", "mp3")));
return result;
} catch (ApiException exception) { throw exception; }
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
}
/**
@@ -322,6 +333,7 @@ public class AsyncProviderClient {
*/
private Map<String,Object> requestMultipart(String method, String url, String key, Map<String,Object> fields,
String fileField, String media, String message) {
boolean sending = false;
try {
String boundary = "----moyi-" + java.util.UUID.randomUUID();
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -337,6 +349,7 @@ public class AsyncProviderClient {
.header("Authorization", "Bearer " + key)
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
.POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build();
sending = true;
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
@@ -345,18 +358,19 @@ public class AsyncProviderClient {
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
}
return parsed;
} catch (ApiException exception) { throw exception; }
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
}
private ApiException upstreamStatus(int status, String message) {
String code = switch (status) {
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
case 429 -> "UPSTREAM_RATE_LIMITED";
default -> "UPSTREAM_PROVIDER_ERROR";
};
return new ApiException(code, message + "(HTTP " + status + ")", 502);
return ProviderRequestException.http(status, message);
}
private ProviderRequestException requestFailure(Exception exception, boolean sending, String message) {
if (exception instanceof InterruptedException) Thread.currentThread().interrupt();
if (exception instanceof ProviderRequestException failure) return failure;
if (sending) return ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", message);
if (exception instanceof ApiException error) return ProviderRequestException.notSent(error.getCode(), error.getMessage(), error.getStatus());
return ProviderRequestException.notSent("UPSTREAM_REQUEST_NOT_SENT", "请求准备失败,尚未提交上游生成", 502);
}
private void multipartField(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException {
@@ -414,21 +428,16 @@ public class AsyncProviderClient {
private record MediaBytes(byte[] bytes, String contentType, String filename) {}
private Map<String,Object> request(String method, String url, String key, Map<String,Object> body, String message) {
boolean sending = false;
try {
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body)));
else builder.GET();
HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
String code = switch (response.statusCode()) {
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
case 429 -> "UPSTREAM_RATE_LIMITED";
default -> "UPSTREAM_PROVIDER_ERROR";
};
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
}
HttpRequest request = builder.build();
sending = true;
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
Object code = parsed.get("code");
if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) {
@@ -437,8 +446,7 @@ public class AsyncProviderClient {
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
}
return parsed;
} catch (ApiException exception) { throw exception; }
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
}
private Provider provider(String id) {
@@ -13,6 +13,7 @@ import com.moyi.controlplane.gateway.request.ApiRequestProtocol;
import com.moyi.controlplane.gateway.request.ApiRequestRiskEngine;
import com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy;
import com.moyi.controlplane.usage.UsageAuditService;
import com.moyi.controlplane.provider.transport.ProviderRequestException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.OffsetDateTime;
@@ -35,6 +36,7 @@ import org.springframework.transaction.annotation.Transactional;
@Service
public class AsyncTaskService {
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(AsyncTaskService.class);
private final JdbcTemplate jdbc;
private final ObjectMapper objectMapper;
private final CatalogService catalogService;
@@ -49,6 +51,7 @@ public class AsyncTaskService {
private final TransactionTemplate transactions;
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
private final ApiRequestRiskPolicy requestRisk;
private final TaskOutcomeJournal outcomes;
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
@@ -104,6 +107,7 @@ public class AsyncTaskService {
this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager);
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
}
@Transactional
@@ -177,7 +181,7 @@ public class AsyncTaskService {
// model. Without the transaction advisory lock, concurrent creates
// could all observe the same count and exceed maxQueuedPerModel.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId);
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId);
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running', 'reconciling')", Integer.class, modelId);
if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429);
@SuppressWarnings("unchecked") Map<String,Object> frozenPricing=(Map<String,Object>) model.get("pricing");
if(AsyncPriceSchedule.tokens(frozenPricing) && !"image".equals(taskType)) throw new ApiException("ASYNC_BILLING_UNSUPPORTED","该媒体类型的 Token 用量结算尚未接入",503);
@@ -237,6 +241,11 @@ public class AsyncTaskService {
@Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}")
public void completeMockTasks() {
// Retry local settlement from durable observations, never the upstream generation request.
for (var completion : outcomes.pendingCompletions()) {
try { completeObserved(completion); }
catch (RuntimeException exception) { log.warn("Async task {} settlement remains pending", completion.taskId()); }
}
List<String> taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id"));
jdbc.update(
"UPDATE model_tasks SET status = 'succeeded', progress = 100, result = CAST(CASE WHEN task_type = '3d' THEN ? ELSE ? END AS jsonb), updated_at = ?, completed_at = ? WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'",
@@ -244,7 +253,7 @@ public class AsyncTaskService {
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
);
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
for (Map<String,Object> row : pending) pollProviderTask(row);
dispatchQueuedTasks();
}
@@ -257,36 +266,43 @@ public class AsyncTaskService {
*/
private void dispatchQueuedTasks() {
if (providerClient == null) return;
// A process can die after claiming a task but before receiving an
// upstream task id. Fail those claims after a grace period. Retrying
// them blindly could submit the same generation twice when the
// upstream accepted the request just before the process died.
// A lost submit response is ambiguous, not a failed generation.
// Quarantine stale claims without retrying or releasing financial holds.
recoverInterruptedClaims();
for (int i = 0; i < 50; i++) {
Map<String,Object> task = claimQueuedTask();
if (task == null) return;
String taskId = String.valueOf(task.get("id"));
usage.updateTask(taskId, "running", null, null, null);
Map<String,Object> upstream;
try {
Map<String,Object> input = taskInput(task);
Map<String,Object> upstream = submitProviderTask(task, input);
// Some OpenAI-compatible image/audio endpoints return the
// generated media in the submit response rather than creating
// a pollable job. Persist that result and settle the task in
// the same worker transaction.
upstream = submitProviderTask(task, input);
} catch (RuntimeException exception) {
if (exception instanceof ProviderRequestException failure && failure.definitelyNotAccepted()) {
completeObserved(outcomes.observeTerminal(taskId, "failed", null, failure.getCode(), failure.getMessage()));
} else {
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_ACCEPTANCE_UNKNOWN", "上游任务是否已受理尚未确认;冻结额度保留,请联系管理员核账,勿重复提交", false);
}
continue;
}
TaskOutcomeJournal.Completion completion = null;
try {
// Journal a received result before touching the ledger. A failed settlement
// must only retry locally, never generate again or release its reservation.
if (upstream.get("_moyiInlineResult") instanceof Map<?,?> inline) {
Map<String,Object> result = new LinkedHashMap<>();
inline.forEach((key, value) -> result.put(String.valueOf(key), value));
finishTask(taskId, "succeeded", 100, result, null, null);
continue;
completion = outcomes.observeTerminal(taskId, "succeeded", result, null, null);
} else {
OffsetDateTime submittedAt = now();
outcomes.accepted(taskId, providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"),
submittedAt, submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)));
}
OffsetDateTime submittedAt = now();
jdbc.update("UPDATE model_tasks SET provider_task_id=?, provider_key_fingerprint=?, provider_deadline_at=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"), submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)), submittedAt, taskId);
} catch (RuntimeException exception) {
String code = exception instanceof ApiException error ? error.getCode() : "UPSTREAM_TASK_SUBMIT_FAILED";
String message = exception instanceof ApiException error ? error.getMessage() : "上游异步任务创建失败";
failTask(taskId, code, message);
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_RECEIPT_UNKNOWN", "已调用上游,但任务回执无法确认或保存;冻结额度保留,等待核账,勿重复提交", false);
}
// Deliberately outside the provider exception handler.
completeObserved(completion);
}
}
@@ -306,18 +322,18 @@ public class AsyncTaskService {
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
AND p.status='active' AND p.deleted_at IS NULL
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status='running') < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status IN ('running','reconciling')) < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
AND (
? <= 0
OR COALESCE(m.max_concurrency, ?) <= 0
OR (SELECT COUNT(*) FROM model_tasks running_model
WHERE running_model.model_id=t.model_id AND running_model.status='running')
WHERE running_model.model_id=t.model_id AND running_model.status IN ('running','reconciling'))
< COALESCE(m.max_concurrency, ?)
)
AND (
? <= 0
OR (SELECT COUNT(*) FROM model_tasks running_key
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
WHERE running_key.api_key_id=t.api_key_id AND running_key.status IN ('running','reconciling'))
< ?
)
ORDER BY t.created_at
@@ -332,19 +348,26 @@ public class AsyncTaskService {
if(!gatewayGuard.tryTaskUserSlot((UUID)row.get("api_key_id"),row.get("id").toString(),providerTimeoutMs+180000)) return null;
try {jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));}
catch(RuntimeException e) {gatewayGuard.releaseTaskUserSlot((UUID)row.get("user_id"),row.get("id").toString());throw e;}
usage.updateTask(row.get("id").toString(), "running", null, null, null);
return row;
});
}
private void recoverInterruptedClaims() {
List<String> taskIds = inTransaction(() -> jdbc.queryForList(
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND completion_status IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
String.class));
for (String taskId : taskIds) {
failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交");
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游提交回执丢失或进程中断;冻结额度保留,等待核账,勿重复提交", true);
}
}
private void completeObserved(TaskOutcomeJournal.Completion completion) {
if (completion == null) return;
finishTask(completion.taskId(), completion.status(), "succeeded".equals(completion.status()) ? 100 : 0,
completion.result(), completion.code(), completion.message());
}
private void failTask(String taskId, String code, String message) {
finishTask(taskId, "failed", 0, null, code, message);
}
@@ -354,8 +377,10 @@ public class AsyncTaskService {
// crash after marking failed/succeeded leaves a hold that polling never
// revisits. The conditional update also prevents duplicate settlement.
inTransaction(() -> {
Map<String,Object> task=jdbc.queryForMap("SELECT status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
if(!List.of("queued","running").contains(task.get("status"))) return null;
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
// A late failure must not override a durably observed success whose settlement is pending.
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
AsyncPriceSchedule.Bill bill=null;
String finalCode=code,finalMessage=message;
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
@@ -363,7 +388,7 @@ public class AsyncTaskService {
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
}
int changed = jdbc.update(
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
status, progress, result == null ? null : writeJson(result),
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
if (changed == 0) return null;
@@ -397,6 +422,7 @@ public class AsyncTaskService {
task.put("progress", resultSet.getInt("progress"));
task.put("createdAt", resultSet.getObject("created_at", OffsetDateTime.class));
task.put("updatedAt", resultSet.getObject("updated_at", OffsetDateTime.class));
task.put("requiresReconciliation", "reconciling".equals(resultSet.getString("status")));
putIfPresent(task, "costAmount", resultSet, "cost_amount");
String result = resultSet.getString("result");
if (result != null) task.put("result", readMap(result));
@@ -440,9 +466,12 @@ public class AsyncTaskService {
private String providerTaskId(Map<String,Object> response) {
Object direct = response.get("id");
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id");
if (direct == null) throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回异步任务编号", 502);
return String.valueOf(direct);
if (direct == null) direct = response.get("task_id");
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id") == null ? data.get("id") : data.get("task_id");
if (!(direct instanceof String value) || value.isBlank() || value.length() > 160 || value.chars().anyMatch(Character::isISOControl)) {
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回有效的异步任务编号", 502);
}
return value;
}
private void pollProviderTask(Map<String,Object> row) {
@@ -456,7 +485,7 @@ public class AsyncTaskService {
deadline = createdAt.plus(java.time.Duration.ofMillis(providerTimeoutMs));
}
if (!now().isBefore(deadline)) {
failTask(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;平台冻结额度已释放。上游可能仍在执行,请先核实后再提交");
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;冻结额度保留,等待管理员核账,勿重复提交", false);
return;
}
Map<String,Object> response;
@@ -464,7 +493,7 @@ public class AsyncTaskService {
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("model_id")), String.valueOf(row.get("upstream_model")), String.valueOf(row.get("provider_task_id")), row.get("provider_key_fingerprint") == null ? null : String.valueOf(row.get("provider_key_fingerprint")));
} catch (RuntimeException exception) {
if (exception instanceof ApiException error && "UPSTREAM_KEY_NOT_AVAILABLE".equals(error.getCode())) {
failTask(taskId, error.getCode(), error.getMessage());
outcomes.requireReconciliation(taskId, error.getCode(), "提交任务使用的上游凭据已不可用,执行结果未知;冻结额度保留,等待核账", false);
return;
}
// Even a permanent authentication/configuration error only proves
@@ -487,9 +516,9 @@ public class AsyncTaskService {
if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) {
// Persistence failures must roll back and remain pollable, rather
// than being mistaken for upstream failures and refunded.
finishTask(taskId, status, progress, result,
completeObserved(outcomes.observeTerminal(taskId, status, result,
"failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null,
"failed".equals(status) ? "上游异步任务失败" : null);
"failed".equals(status) ? "上游异步任务失败" : null));
} else {
updatePollingState(taskId, progress, null, null);
}
@@ -497,7 +526,7 @@ public class AsyncTaskService {
private void updatePollingState(String taskId, Integer progress, String code, String message) {
inTransaction(() -> {
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running')",
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL",
progress, code == null ? null : writeJson(Map.of("code", code, "message", message, "retryable", true)), now(), taskId);
if (changed > 0) usage.updateTask(taskId, "running", null, code, message);
return null;
@@ -515,10 +544,12 @@ public class AsyncTaskService {
Object raw = response.get("status");
if (raw == null && response.get("data") instanceof Map<?,?> data) raw = data.get("status");
String value = String.valueOf(raw == null ? "running" : raw).toLowerCase(java.util.Locale.ROOT);
if (value.contains("success") || value.equals("completed")) return "succeeded";
if (value.contains("fail") || value.equals("error") || value.equals("banned")) return "failed";
if (value.contains("cancel")) return "cancelled";
return "running";
return switch (value) {
case "success", "succeeded", "completed" -> "succeeded";
case "failed", "failure", "error", "banned" -> "failed";
case "cancelled", "canceled" -> "cancelled";
default -> "running";
};
}
private int progress(Map<String,Object> response, String status) {
@@ -0,0 +1,91 @@
package com.moyi.controlplane.task;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.moyi.controlplane.usage.UsageAuditService;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.support.TransactionTemplate;
/** Durable provider observations only. No upstream calls, wallet mutation, retry or refund policy. */
final class TaskOutcomeJournal {
record Completion(String taskId, String status, Map<String,Object> result, String code, String message) { }
private final JdbcTemplate jdbc;
private final ObjectMapper mapper;
private final TransactionTemplate transactions;
private final UsageAuditService usage;
TaskOutcomeJournal(JdbcTemplate jdbc, ObjectMapper mapper, TransactionTemplate transactions, UsageAuditService usage) {
this.jdbc = jdbc;
this.mapper = mapper;
this.transactions = transactions;
this.usage = usage;
}
Completion observeTerminal(String taskId, String status, Map<String,Object> result, String code, String message) {
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal outcome required");
return transaction(() -> {
var row = jdbc.queryForMap("SELECT status,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE id=? FOR UPDATE", taskId);
if (!List.of("queued", "running", "reconciling").contains(row.get("status"))) return null;
if (row.get("completion_status") != null) return completion(taskId, row);
jdbc.update("UPDATE model_tasks SET completion_status=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),updated_at=now() WHERE id=?",
status, json(result), code == null ? null : json(Map.of("code", code, "message", message)), taskId);
return new Completion(taskId, status, result, code, message);
});
}
List<Completion> pendingCompletions() {
return jdbc.queryForList("SELECT id,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE status IN ('running','reconciling') AND completion_status IS NOT NULL ORDER BY updated_at LIMIT 50")
.stream().map(row -> completion(String.valueOf(row.get("id")), row)).toList();
}
void accepted(String taskId, String providerTaskId, Object keyFingerprint, OffsetDateTime submittedAt, OffsetDateTime deadline) {
transaction(() -> {
int changed = jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=? WHERE id=? AND status IN ('running','reconciling') AND provider_task_id IS NULL AND completion_status IS NULL",
providerTaskId, keyFingerprint, deadline, submittedAt, taskId);
if (changed > 0) usage.updateTask(taskId, "running", null, null, null);
return null;
});
}
void requireReconciliation(String taskId, String code, String message, boolean interruptedClaim) {
transaction(() -> {
String claimGuard = interruptedClaim
? " AND provider_task_id IS NULL AND updated_at < now()-interval '2 minutes'" : "";
int changed = jdbc.update("UPDATE model_tasks SET status='reconciling',error=CAST(? AS jsonb),reconciliation_required_at=now(),updated_at=now() WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL" + claimGuard,
json(Map.of("code", code, "message", message, "retryable", false, "requiresReconciliation", true)), taskId);
if (changed > 0) {
usage.updateTask(taskId, "reconciling", null, code, message);
jdbc.update("UPDATE usage_events SET cost_status='pending',cost_amount=NULL WHERE provider_task_id=? AND event_kind='task'", taskId);
}
return null;
});
}
private Completion completion(String taskId, Map<String,Object> row) {
Map<String,Object> error = map(row.get("error"));
return new Completion(taskId, String.valueOf(row.get("completion_status")), map(row.get("result")),
error == null ? null : (String) error.get("code"), error == null ? null : (String) error.get("message"));
}
private Map<String,Object> map(Object value) {
if (value == null) return null;
try { return mapper.readValue(String.valueOf(value), new TypeReference<>() {}); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Invalid persisted provider observation"); }
}
private String json(Object value) {
if (value == null) return null;
try { return mapper.writeValueAsString(value); }
catch (JsonProcessingException exception) { throw new IllegalStateException("Provider observation cannot be encoded"); }
}
private <T> T transaction(Supplier<T> work) {
return transactions == null ? work.get() : transactions.execute(status -> work.get());
}
}
@@ -157,15 +157,17 @@ public class UsageAuditService {
WITH pending AS (
SELECT model_id,
COUNT(*) FILTER (WHERE status='running') AS running,
COUNT(*) FILTER (WHERE status='queued') AS queued
FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id
COUNT(*) FILTER (WHERE status='queued') AS queued,
COUNT(*) FILTER (WHERE status='reconciling') AS reconciling
FROM model_tasks WHERE status IN ('queued','running','reconciling') GROUP BY model_id
)
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
CASE WHEN m.max_concurrency = 0 THEN 0
ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency,
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued,
COALESCE(pending.reconciling, 0) AS reconciling
FROM model_catalog m JOIN model_providers p ON p.id=m.provider_id
JOIN model_definitions d ON d.id=m.logical_model_id
LEFT JOIN pending ON pending.model_id=m.id
@@ -182,6 +184,7 @@ public class UsageAuditService {
group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0);
group.put("running", rs.getLong("running"));
group.put("queued", rs.getLong("queued"));
group.put("reconciling", rs.getLong("reconciling"));
return group;
}, defaultModelLimit);
List<String> synchronousIds = groups.stream().filter(group -> !"async".equals(group.get("mode"))).map(group -> group.get("id").toString()).toList();
@@ -193,12 +196,13 @@ public class UsageAuditService {
group.put("running", current == null ? null : current.running());
group.put("queued", current == null ? null : current.queued());
}
// Keep disabled groups visible while their queued or running work still exists.
// Keep disabled groups visible while any work or unresolved financial hold still exists.
groups = groups.stream().filter(group -> Boolean.TRUE.equals(group.get("routeEnabled"))
|| positive(group.get("running")) || positive(group.get("queued"))).toList();
|| positive(group.get("running")) || positive(group.get("queued")) || positive(group.get("reconciling"))).toList();
result.put("capacityGroups", groups);
result.put("running", totalCapacity(groups, "running"));
result.put("waiting", totalCapacity(groups, "queued"));
result.put("reconciling", totalCapacity(groups, "reconciling"));
result.put("capacityConfig", Map.of(
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
"defaultModelConcurrency", defaultModelLimit,