fix: classify upstream failures and prevent empty billed responses

This commit is contained in:
wlx committed 2026-09-28 04:55:00 +08:00
1 parent b455d8d79f
commit 9acc69d4c7
28 files changed
+1449 -131

No files matched your search

@@ -73,12 +73,21 @@ public class AsyncProviderClient {
if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body)));
else builder.GET();
HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) throw new ApiException("UPSTREAM_PROVIDER_ERROR", message + "(HTTP " + response.statusCode() + ")", 502);
if (response.statusCode() < 200 || response.statusCode() >= 300) {
String code = switch (response.statusCode()) {
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
case 429 -> "UPSTREAM_RATE_LIMITED";
default -> "UPSTREAM_PROVIDER_ERROR";
};
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
}
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
Object code = parsed.get("code");
if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) {
String detail = parsed.get("message") == null ? message : String.valueOf(parsed.get("message"));
throw new ApiException("UPSTREAM_PROVIDER_ERROR", detail, 502);
// Upstream messages can echo credentials or account details.
// Only locally defined diagnostics may reach task history/UI.
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
}
return parsed;
} catch (ApiException exception) { throw exception; }
@@ -35,6 +35,7 @@ public class AsyncTaskService {
private final int maxQueuedPerModel;
private final int defaultModelConcurrency;
private final int defaultKeyConcurrency;
private final long providerTimeoutMs;
private final GatewayGuard gatewayGuard;
private final UsageAuditService usage;
private final AsyncProviderClient providerClient;
@@ -43,7 +44,7 @@ public class AsyncTaskService {
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, gatewayGuard, usage, null, null);
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, 3_600_000, gatewayGuard, usage, null, null);
}
@org.springframework.beans.factory.annotation.Autowired
@@ -54,18 +55,21 @@ public class AsyncTaskService {
@Value("${moyi.tasks.max-queued-per-model:100}") int maxQueuedPerModel,
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
@Value("${moyi.tasks.provider-timeout-ms:3600000}") long providerTimeoutMs,
GatewayGuard gatewayGuard,
UsageAuditService usage,
AsyncProviderClient providerClient,
PlatformTransactionManager transactionManager
) {
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
if (providerTimeoutMs < 1_000) throw new IllegalArgumentException("Provider task timeout must be at least one second");
this.jdbc = jdbc;
this.objectMapper = objectMapper;
this.catalogService = catalogService;
this.maxQueuedPerModel = maxQueuedPerModel;
this.defaultModelConcurrency = defaultModelConcurrency;
this.defaultKeyConcurrency = defaultKeyConcurrency;
this.providerTimeoutMs = providerTimeoutMs;
this.gatewayGuard = gatewayGuard;
this.usage = usage;
this.providerClient = providerClient;
@@ -155,7 +159,7 @@ public class AsyncTaskService {
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
);
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.provider_task_id, m.provider_id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' ORDER BY t.created_at LIMIT 50");
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.provider_task_id, t.provider_deadline_at, t.created_at, m.provider_id FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
for (Map<String,Object> row : pending) pollProviderTask(row);
dispatchQueuedTasks();
}
@@ -185,9 +189,12 @@ public class AsyncTaskService {
String.valueOf(task.get("task_type")),
String.valueOf(task.get("prompt"))
);
jdbc.update("UPDATE model_tasks SET provider_task_id=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), now(), taskId);
OffsetDateTime submittedAt = now();
jdbc.update("UPDATE model_tasks SET provider_task_id=?, provider_deadline_at=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)), submittedAt, taskId);
} catch (RuntimeException exception) {
failTask(taskId, "UPSTREAM_TASK_SUBMIT_FAILED", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage());
String code = exception instanceof ApiException error ? error.getCode() : "UPSTREAM_TASK_SUBMIT_FAILED";
String message = exception instanceof ApiException error ? error.getMessage() : "上游异步任务创建失败";
failTask(taskId, code, message);
}
}
}
@@ -236,12 +243,24 @@ public class AsyncTaskService {
}
private void failTask(String taskId, String code, String message) {
int changed = jdbc.update(
"UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
writeJson(Map.of("code", code, "message", message)), now(), now(), taskId);
if (changed == 0) return;
releaseTask(taskId);
usage.updateTask(taskId, "failed", null, code, message);
finishTask(taskId, "failed", 0, null, code, message);
}
private void finishTask(String taskId, String status, int progress, Map<String,Object> result, String code, String message) {
// The terminal state and wallet change must commit together. Otherwise a
// crash after marking failed/succeeded leaves a hold that polling never
// revisits. The conditional update also prevents duplicate settlement.
inTransaction(() -> {
int changed = jdbc.update(
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
status, progress, result == null ? null : writeJson(result),
code == null ? null : writeJson(Map.of("code", code, "message", message)), now(), now(), taskId);
if (changed == 0) return null;
if ("succeeded".equals(status)) settleTaskInTransaction(taskId);
else releaseTaskInTransaction(taskId);
usage.updateTask(taskId, status, result, code, message);
return null;
});
}
private Map<String, Object> requireKey(String authorization) {
@@ -286,27 +305,59 @@ public class AsyncTaskService {
private void pollProviderTask(Map<String,Object> row) {
String taskId = String.valueOf(row.get("id"));
try {
Map<String,Object> response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("provider_task_id")));
String status = normalizedStatus(response);
int progress = progress(response, status);
Map<String,Object> result = result(response);
if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) {
jdbc.update("UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", status, progress, result == null ? null : writeJson(result), "failed".equals(status) ? writeJson(Map.of("message", "上游异步任务失败")) : null, now(), now(), taskId);
if ("succeeded".equals(status)) settleTask(taskId);
else releaseTask(taskId);
usage.updateTask(taskId, status, result, "failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null, "failed".equals(status) ? "上游异步任务失败" : null);
} else {
jdbc.update("UPDATE model_tasks SET status=?, progress=?, updated_at=? WHERE id=? AND status IN ('queued','running')", status, progress, now(), taskId);
}
} catch (Exception exception) {
jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", writeJson(Map.of("message", "上游异步任务查询失败")), now(), now(), taskId);
// A polling failure is terminal for this task. Release the
// reserved wallet amount as part of the same failure path so a
// transient/provider error cannot permanently lock user credit.
releaseTask(taskId);
usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_POLL_FAILED", "上游异步任务查询失败");
OffsetDateTime deadline = taskTimestamp(row.get("provider_deadline_at"));
if (deadline == null) {
// Defensive fallback for pre-migration tasks. The migration also
// persists deadlines so restarting cannot reset their timeout.
OffsetDateTime createdAt = taskTimestamp(row.get("created_at"));
if (createdAt == null) throw new IllegalStateException("Persisted task timestamp is missing");
deadline = createdAt.plus(java.time.Duration.ofMillis(providerTimeoutMs));
}
if (!now().isBefore(deadline)) {
failTask(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;平台冻结额度已释放。上游可能仍在执行,请先核实后再提交");
return;
}
Map<String,Object> response;
try {
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("provider_task_id")));
} catch (RuntimeException exception) {
// Even a permanent authentication/configuration error only proves
// that polling failed, not that an already accepted job stopped.
// Keep its hold and retry until the persisted deadline.
String message = exception instanceof ApiException error && "UPSTREAM_AUTH_FAILED".equals(error.getCode())
? "上游查询鉴权异常,执行结果尚未确认;正在重试"
: "暂时无法查询上游任务,执行结果尚未确认;正在重试";
updatePollingState(taskId, null, "UPSTREAM_TASK_POLL_RETRY", message);
return;
}
String status = normalizedStatus(response);
int progress = progress(response, status);
Map<String,Object> result = result(response);
if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) {
// Persistence failures must roll back and remain pollable, rather
// than being mistaken for upstream failures and refunded.
finishTask(taskId, status, progress, result,
"failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null,
"failed".equals(status) ? "上游异步任务失败" : null);
} else {
updatePollingState(taskId, progress, null, null);
}
}
private void updatePollingState(String taskId, Integer progress, String code, String message) {
inTransaction(() -> {
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running')",
progress, code == null ? null : writeJson(Map.of("code", code, "message", message, "retryable", true)), now(), taskId);
if (changed > 0) usage.updateTask(taskId, "running", null, code, message);
return null;
});
}
private OffsetDateTime taskTimestamp(Object value) {
if (value == null) return null;
if (value instanceof OffsetDateTime timestamp) return timestamp;
if (value instanceof java.sql.Timestamp timestamp) return timestamp.toInstant().atOffset(ZoneOffset.UTC);
throw new IllegalStateException("Persisted task timestamp is missing");
}
private String normalizedStatus(Map<String,Object> response) {
@@ -381,14 +432,6 @@ public class AsyncTaskService {
if (amount.signum() > 0) jdbc.update("UPDATE user_wallets SET reserved_balance=GREATEST(0,reserved_balance-?),updated_at=now() WHERE user_id=?", amount, userId);
}
private void settleTask(String taskId) {
if (transactions == null) {
settleTaskInTransaction(taskId);
return;
}
transactions.executeWithoutResult(status -> settleTaskInTransaction(taskId));
}
private void settleTaskInTransaction(String taskId) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
@@ -403,20 +446,13 @@ public class AsyncTaskService {
jdbc.update("UPDATE usage_events SET cost_amount=? WHERE provider_task_id=? AND event_kind='task'", amount, taskId);
}
private void releaseTask(String taskId) {
if (transactions == null) {
releaseTaskInTransaction(taskId);
return;
}
transactions.executeWithoutResult(status -> releaseTaskInTransaction(taskId));
}
private void releaseTaskInTransaction(String taskId) {
Map<String,Object> task = jdbc.queryForMap("SELECT user_id,reserved_amount,cost_amount FROM model_tasks WHERE id=? FOR UPDATE", taskId);
BigDecimal amount = (BigDecimal) task.get("reserved_amount");
if (amount != null && amount.signum() > 0 && task.get("cost_amount") == null) {
releaseWallet((UUID) task.get("user_id"), amount);
jdbc.update("UPDATE model_tasks SET cost_amount=0 WHERE id=?", taskId);
jdbc.update("UPDATE usage_events SET cost_amount=0 WHERE provider_task_id=? AND event_kind='task'", taskId);
}
}