fix: retain async holds for unknown provider outcomes
This commit is contained in:
1 parent
76265845dc
commit
5be06a97c1
18 files changed
+574
-108
No files matched your search
@@ -383,6 +383,7 @@ public class CatalogService {
|
||||
result.put("tasks", Map.of(
|
||||
"queued", taskCounts.getOrDefault("queued", 0L),
|
||||
"running", taskCounts.getOrDefault("running", 0L),
|
||||
"reconciling", taskCounts.getOrDefault("reconciling", 0L),
|
||||
"succeeded", taskCounts.getOrDefault("succeeded", 0L),
|
||||
"failed", taskCounts.getOrDefault("failed", 0L)
|
||||
));
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package com.moyi.controlplane.provider.transport;
|
||||
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
|
||||
/** HTTP boundary facts, not a retry policy. Unknown acceptance must never authorize replay/refund. */
|
||||
public final class ProviderRequestException extends ApiException {
|
||||
public enum Acceptance { NOT_SENT, REJECTED, UNKNOWN }
|
||||
|
||||
private final Acceptance acceptance;
|
||||
private final Integer upstreamStatus;
|
||||
|
||||
private ProviderRequestException(String code, String message, int status, Acceptance acceptance, Integer upstreamStatus) {
|
||||
super(code, message, status);
|
||||
this.acceptance = acceptance;
|
||||
this.upstreamStatus = upstreamStatus;
|
||||
}
|
||||
|
||||
public static ProviderRequestException notSent(String code, String message, int status) {
|
||||
return new ProviderRequestException(code, message, status, Acceptance.NOT_SENT, null);
|
||||
}
|
||||
|
||||
public static ProviderRequestException unknown(String code, String message) {
|
||||
return new ProviderRequestException(code, message, 502, Acceptance.UNKNOWN, null);
|
||||
}
|
||||
|
||||
public static ProviderRequestException http(int status, String message) {
|
||||
String code = switch (status) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
// Timeout/conflict/5xx responses do not prove a generation was never accepted.
|
||||
Acceptance acceptance = switch (status) {
|
||||
case 400, 401, 402, 403, 404, 405, 413, 415, 422, 429 -> Acceptance.REJECTED;
|
||||
default -> Acceptance.UNKNOWN;
|
||||
};
|
||||
return new ProviderRequestException(code, message + "(HTTP " + status + ")", 502, acceptance, status);
|
||||
}
|
||||
|
||||
public Acceptance acceptance() { return acceptance; }
|
||||
public Integer upstreamStatus() { return upstreamStatus; }
|
||||
public boolean definitelyNotAccepted() { return acceptance != Acceptance.UNKNOWN; }
|
||||
}
|
||||
+43
-35
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver;
|
||||
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||
import java.net.URI;
|
||||
import java.net.InetAddress;
|
||||
import java.net.http.HttpClient;
|
||||
@@ -60,6 +61,22 @@ public class AsyncProviderClient {
|
||||
}
|
||||
|
||||
public Map<String,Object> submit(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
||||
try {
|
||||
return submitConfigured(providerId, upstreamModel, routeId, taskType, input);
|
||||
} catch (ProviderRequestException exception) {
|
||||
throw exception;
|
||||
} catch (ApiException exception) {
|
||||
// Only these local validation/configuration errors are known to precede generation submission.
|
||||
if (List.of("PROVIDER_NOT_CONFIGURED", "UPSTREAM_KEY_NOT_AVAILABLE", "ASYNC_PROVIDER_NOT_IMPLEMENTED",
|
||||
"MEDIA_INPUT_UNSUPPORTED", "INVALID_MEDIA_URL", "MEDIA_FETCH_FAILED", "MEDIA_TOO_LARGE",
|
||||
"MEDIA_TYPE_UNSUPPORTED", "INVALID_TASK_INPUT").contains(exception.getCode())) {
|
||||
throw ProviderRequestException.notSent(exception.getCode(), exception.getMessage(), exception.getStatus());
|
||||
}
|
||||
throw ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", "上游任务提交结果尚未确认");
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String,Object> submitConfigured(String providerId, String upstreamModel, String routeId, String taskType, Map<String,Object> input) {
|
||||
if (input == null) input = Map.of();
|
||||
String prompt = String.valueOf(input.getOrDefault("prompt", ""));
|
||||
Provider provider = provider(providerId, routeId, upstreamModel);
|
||||
@@ -261,8 +278,8 @@ public class AsyncProviderClient {
|
||||
}
|
||||
|
||||
private boolean isNotFound(ApiException exception) {
|
||||
String message = exception.getMessage();
|
||||
return message != null && (message.contains("HTTP 404") || message.contains("HTTP 405"));
|
||||
return exception instanceof ProviderRequestException failure
|
||||
&& (Integer.valueOf(404).equals(failure.upstreamStatus()) || Integer.valueOf(405).equals(failure.upstreamStatus()));
|
||||
}
|
||||
|
||||
/** Normalize OpenAI image/audio response fields to the task result shape. */
|
||||
@@ -287,19 +304,14 @@ public class AsyncProviderClient {
|
||||
}
|
||||
|
||||
private Map<String,Object> requestAudio(String method, String url, String key, Map<String,Object> body, String message) {
|
||||
boolean sending = false;
|
||||
try {
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
||||
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
||||
HttpResponse<byte[]> response = client.send(builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(), HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String code = switch (response.statusCode()) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
|
||||
}
|
||||
HttpRequest request = builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build();
|
||||
sending = true;
|
||||
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||
byte[] bytes = response.body();
|
||||
if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502);
|
||||
String contentType = response.headers().firstValue("Content-Type").orElse("audio/mpeg").split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT);
|
||||
@@ -309,8 +321,7 @@ public class AsyncProviderClient {
|
||||
result.put("mimeType", contentType);
|
||||
result.put("format", String.valueOf(body.getOrDefault("response_format", "mp3")));
|
||||
return result;
|
||||
} catch (ApiException exception) { throw exception; }
|
||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
||||
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -322,6 +333,7 @@ public class AsyncProviderClient {
|
||||
*/
|
||||
private Map<String,Object> requestMultipart(String method, String url, String key, Map<String,Object> fields,
|
||||
String fileField, String media, String message) {
|
||||
boolean sending = false;
|
||||
try {
|
||||
String boundary = "----moyi-" + java.util.UUID.randomUUID();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
@@ -337,6 +349,7 @@ public class AsyncProviderClient {
|
||||
.header("Authorization", "Bearer " + key)
|
||||
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
|
||||
.POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build();
|
||||
sending = true;
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
||||
@@ -345,18 +358,19 @@ public class AsyncProviderClient {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
||||
}
|
||||
return parsed;
|
||||
} catch (ApiException exception) { throw exception; }
|
||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
||||
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||
}
|
||||
|
||||
private ApiException upstreamStatus(int status, String message) {
|
||||
String code = switch (status) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
return new ApiException(code, message + "(HTTP " + status + ")", 502);
|
||||
return ProviderRequestException.http(status, message);
|
||||
}
|
||||
|
||||
private ProviderRequestException requestFailure(Exception exception, boolean sending, String message) {
|
||||
if (exception instanceof InterruptedException) Thread.currentThread().interrupt();
|
||||
if (exception instanceof ProviderRequestException failure) return failure;
|
||||
if (sending) return ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", message);
|
||||
if (exception instanceof ApiException error) return ProviderRequestException.notSent(error.getCode(), error.getMessage(), error.getStatus());
|
||||
return ProviderRequestException.notSent("UPSTREAM_REQUEST_NOT_SENT", "请求准备失败,尚未提交上游生成", 502);
|
||||
}
|
||||
|
||||
private void multipartField(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException {
|
||||
@@ -414,21 +428,16 @@ public class AsyncProviderClient {
|
||||
private record MediaBytes(byte[] bytes, String contentType, String filename) {}
|
||||
|
||||
private Map<String,Object> request(String method, String url, String key, Map<String,Object> body, String message) {
|
||||
boolean sending = false;
|
||||
try {
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT)
|
||||
.header("Authorization", "Bearer " + key).header("Content-Type", "application/json");
|
||||
if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body)));
|
||||
else builder.GET();
|
||||
HttpResponse<String> response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String code = switch (response.statusCode()) {
|
||||
case 401, 403 -> "UPSTREAM_AUTH_FAILED";
|
||||
case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT";
|
||||
case 429 -> "UPSTREAM_RATE_LIMITED";
|
||||
default -> "UPSTREAM_PROVIDER_ERROR";
|
||||
};
|
||||
throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502);
|
||||
}
|
||||
HttpRequest request = builder.build();
|
||||
sending = true;
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message);
|
||||
Map<String,Object> parsed = mapper.readValue(response.body(), new TypeReference<>() {});
|
||||
Object code = parsed.get("code");
|
||||
if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) {
|
||||
@@ -437,8 +446,7 @@ public class AsyncProviderClient {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502);
|
||||
}
|
||||
return parsed;
|
||||
} catch (ApiException exception) { throw exception; }
|
||||
catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); }
|
||||
} catch (Exception exception) { throw requestFailure(exception, sending, message); }
|
||||
}
|
||||
|
||||
private Provider provider(String id) {
|
||||
|
||||
@@ -13,6 +13,7 @@ import com.moyi.controlplane.gateway.request.ApiRequestProtocol;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestRiskEngine;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import com.moyi.controlplane.provider.transport.ProviderRequestException;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.time.OffsetDateTime;
|
||||
@@ -35,6 +36,7 @@ import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Service
|
||||
public class AsyncTaskService {
|
||||
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(AsyncTaskService.class);
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final CatalogService catalogService;
|
||||
@@ -49,6 +51,7 @@ public class AsyncTaskService {
|
||||
private final TransactionTemplate transactions;
|
||||
private final com.moyi.controlplane.gateway.EnterprisePolicies policies;
|
||||
private final ApiRequestRiskPolicy requestRisk;
|
||||
private final TaskOutcomeJournal outcomes;
|
||||
|
||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
@@ -104,6 +107,7 @@ public class AsyncTaskService {
|
||||
this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager);
|
||||
this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper);
|
||||
this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk");
|
||||
this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -177,7 +181,7 @@ public class AsyncTaskService {
|
||||
// model. Without the transaction advisory lock, concurrent creates
|
||||
// could all observe the same count and exceed maxQueuedPerModel.
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId);
|
||||
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId);
|
||||
Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running', 'reconciling')", Integer.class, modelId);
|
||||
if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429);
|
||||
@SuppressWarnings("unchecked") Map<String,Object> frozenPricing=(Map<String,Object>) model.get("pricing");
|
||||
if(AsyncPriceSchedule.tokens(frozenPricing) && !"image".equals(taskType)) throw new ApiException("ASYNC_BILLING_UNSUPPORTED","该媒体类型的 Token 用量结算尚未接入",503);
|
||||
@@ -237,6 +241,11 @@ public class AsyncTaskService {
|
||||
|
||||
@Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}")
|
||||
public void completeMockTasks() {
|
||||
// Retry local settlement from durable observations, never the upstream generation request.
|
||||
for (var completion : outcomes.pendingCompletions()) {
|
||||
try { completeObserved(completion); }
|
||||
catch (RuntimeException exception) { log.warn("Async task {} settlement remains pending", completion.taskId()); }
|
||||
}
|
||||
List<String> taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id"));
|
||||
jdbc.update(
|
||||
"UPDATE model_tasks SET status = 'succeeded', progress = 100, result = CAST(CASE WHEN task_type = '3d' THEN ? ELSE ? END AS jsonb), updated_at = ?, completed_at = ? WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'",
|
||||
@@ -244,7 +253,7 @@ public class AsyncTaskService {
|
||||
"{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now()
|
||||
);
|
||||
taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null));
|
||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||
List<Map<String,Object>> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50");
|
||||
for (Map<String,Object> row : pending) pollProviderTask(row);
|
||||
dispatchQueuedTasks();
|
||||
}
|
||||
@@ -257,36 +266,43 @@ public class AsyncTaskService {
|
||||
*/
|
||||
private void dispatchQueuedTasks() {
|
||||
if (providerClient == null) return;
|
||||
// A process can die after claiming a task but before receiving an
|
||||
// upstream task id. Fail those claims after a grace period. Retrying
|
||||
// them blindly could submit the same generation twice when the
|
||||
// upstream accepted the request just before the process died.
|
||||
// A lost submit response is ambiguous, not a failed generation.
|
||||
// Quarantine stale claims without retrying or releasing financial holds.
|
||||
recoverInterruptedClaims();
|
||||
for (int i = 0; i < 50; i++) {
|
||||
Map<String,Object> task = claimQueuedTask();
|
||||
if (task == null) return;
|
||||
String taskId = String.valueOf(task.get("id"));
|
||||
usage.updateTask(taskId, "running", null, null, null);
|
||||
Map<String,Object> upstream;
|
||||
try {
|
||||
Map<String,Object> input = taskInput(task);
|
||||
Map<String,Object> upstream = submitProviderTask(task, input);
|
||||
// Some OpenAI-compatible image/audio endpoints return the
|
||||
// generated media in the submit response rather than creating
|
||||
// a pollable job. Persist that result and settle the task in
|
||||
// the same worker transaction.
|
||||
upstream = submitProviderTask(task, input);
|
||||
} catch (RuntimeException exception) {
|
||||
if (exception instanceof ProviderRequestException failure && failure.definitelyNotAccepted()) {
|
||||
completeObserved(outcomes.observeTerminal(taskId, "failed", null, failure.getCode(), failure.getMessage()));
|
||||
} else {
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_ACCEPTANCE_UNKNOWN", "上游任务是否已受理尚未确认;冻结额度保留,请联系管理员核账,勿重复提交", false);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
TaskOutcomeJournal.Completion completion = null;
|
||||
try {
|
||||
// Journal a received result before touching the ledger. A failed settlement
|
||||
// must only retry locally, never generate again or release its reservation.
|
||||
if (upstream.get("_moyiInlineResult") instanceof Map<?,?> inline) {
|
||||
Map<String,Object> result = new LinkedHashMap<>();
|
||||
inline.forEach((key, value) -> result.put(String.valueOf(key), value));
|
||||
finishTask(taskId, "succeeded", 100, result, null, null);
|
||||
continue;
|
||||
completion = outcomes.observeTerminal(taskId, "succeeded", result, null, null);
|
||||
} else {
|
||||
OffsetDateTime submittedAt = now();
|
||||
outcomes.accepted(taskId, providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"),
|
||||
submittedAt, submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)));
|
||||
}
|
||||
OffsetDateTime submittedAt = now();
|
||||
jdbc.update("UPDATE model_tasks SET provider_task_id=?, provider_key_fingerprint=?, provider_deadline_at=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"), submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)), submittedAt, taskId);
|
||||
} catch (RuntimeException exception) {
|
||||
String code = exception instanceof ApiException error ? error.getCode() : "UPSTREAM_TASK_SUBMIT_FAILED";
|
||||
String message = exception instanceof ApiException error ? error.getMessage() : "上游异步任务创建失败";
|
||||
failTask(taskId, code, message);
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_RECEIPT_UNKNOWN", "已调用上游,但任务回执无法确认或保存;冻结额度保留,等待核账,勿重复提交", false);
|
||||
}
|
||||
// Deliberately outside the provider exception handler.
|
||||
completeObserved(completion);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,18 +322,18 @@ public class AsyncTaskService {
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status='running') < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
|
||||
AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status IN ('running','reconciling')) < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16)
|
||||
AND (
|
||||
? <= 0
|
||||
OR COALESCE(m.max_concurrency, ?) <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_model
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status IN ('running','reconciling'))
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
)
|
||||
AND (
|
||||
? <= 0
|
||||
OR (SELECT COUNT(*) FROM model_tasks running_key
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status IN ('running','reconciling'))
|
||||
< ?
|
||||
)
|
||||
ORDER BY t.created_at
|
||||
@@ -332,19 +348,26 @@ public class AsyncTaskService {
|
||||
if(!gatewayGuard.tryTaskUserSlot((UUID)row.get("api_key_id"),row.get("id").toString(),providerTimeoutMs+180000)) return null;
|
||||
try {jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));}
|
||||
catch(RuntimeException e) {gatewayGuard.releaseTaskUserSlot((UUID)row.get("user_id"),row.get("id").toString());throw e;}
|
||||
usage.updateTask(row.get("id").toString(), "running", null, null, null);
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
private void recoverInterruptedClaims() {
|
||||
List<String> taskIds = inTransaction(() -> jdbc.queryForList(
|
||||
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
|
||||
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND completion_status IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
|
||||
String.class));
|
||||
for (String taskId : taskIds) {
|
||||
failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交");
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游提交回执丢失或进程中断;冻结额度保留,等待核账,勿重复提交", true);
|
||||
}
|
||||
}
|
||||
|
||||
private void completeObserved(TaskOutcomeJournal.Completion completion) {
|
||||
if (completion == null) return;
|
||||
finishTask(completion.taskId(), completion.status(), "succeeded".equals(completion.status()) ? 100 : 0,
|
||||
completion.result(), completion.code(), completion.message());
|
||||
}
|
||||
|
||||
private void failTask(String taskId, String code, String message) {
|
||||
finishTask(taskId, "failed", 0, null, code, message);
|
||||
}
|
||||
@@ -354,8 +377,10 @@ public class AsyncTaskService {
|
||||
// crash after marking failed/succeeded leaves a hold that polling never
|
||||
// revisits. The conditional update also prevents duplicate settlement.
|
||||
inTransaction(() -> {
|
||||
Map<String,Object> task=jdbc.queryForMap("SELECT status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
|
||||
if(!List.of("queued","running").contains(task.get("status"))) return null;
|
||||
Map<String,Object> task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId);
|
||||
if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null;
|
||||
// A late failure must not override a durably observed success whose settlement is pending.
|
||||
if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null;
|
||||
AsyncPriceSchedule.Bill bill=null;
|
||||
String finalCode=code,finalMessage=message;
|
||||
if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) {
|
||||
@@ -363,7 +388,7 @@ public class AsyncTaskService {
|
||||
catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; }
|
||||
}
|
||||
int changed = jdbc.update(
|
||||
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
|
||||
"UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')",
|
||||
status, progress, result == null ? null : writeJson(result),
|
||||
finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId);
|
||||
if (changed == 0) return null;
|
||||
@@ -397,6 +422,7 @@ public class AsyncTaskService {
|
||||
task.put("progress", resultSet.getInt("progress"));
|
||||
task.put("createdAt", resultSet.getObject("created_at", OffsetDateTime.class));
|
||||
task.put("updatedAt", resultSet.getObject("updated_at", OffsetDateTime.class));
|
||||
task.put("requiresReconciliation", "reconciling".equals(resultSet.getString("status")));
|
||||
putIfPresent(task, "costAmount", resultSet, "cost_amount");
|
||||
String result = resultSet.getString("result");
|
||||
if (result != null) task.put("result", readMap(result));
|
||||
@@ -440,9 +466,12 @@ public class AsyncTaskService {
|
||||
|
||||
private String providerTaskId(Map<String,Object> response) {
|
||||
Object direct = response.get("id");
|
||||
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id");
|
||||
if (direct == null) throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回异步任务编号", 502);
|
||||
return String.valueOf(direct);
|
||||
if (direct == null) direct = response.get("task_id");
|
||||
if (direct == null && response.get("data") instanceof Map<?,?> data) direct = data.get("task_id") == null ? data.get("id") : data.get("task_id");
|
||||
if (!(direct instanceof String value) || value.isBlank() || value.length() > 160 || value.chars().anyMatch(Character::isISOControl)) {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回有效的异步任务编号", 502);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private void pollProviderTask(Map<String,Object> row) {
|
||||
@@ -456,7 +485,7 @@ public class AsyncTaskService {
|
||||
deadline = createdAt.plus(java.time.Duration.ofMillis(providerTimeoutMs));
|
||||
}
|
||||
if (!now().isBefore(deadline)) {
|
||||
failTask(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;平台冻结额度已释放。上游可能仍在执行,请先核实后再提交");
|
||||
outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;冻结额度保留,等待管理员核账,勿重复提交", false);
|
||||
return;
|
||||
}
|
||||
Map<String,Object> response;
|
||||
@@ -464,7 +493,7 @@ public class AsyncTaskService {
|
||||
response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("model_id")), String.valueOf(row.get("upstream_model")), String.valueOf(row.get("provider_task_id")), row.get("provider_key_fingerprint") == null ? null : String.valueOf(row.get("provider_key_fingerprint")));
|
||||
} catch (RuntimeException exception) {
|
||||
if (exception instanceof ApiException error && "UPSTREAM_KEY_NOT_AVAILABLE".equals(error.getCode())) {
|
||||
failTask(taskId, error.getCode(), error.getMessage());
|
||||
outcomes.requireReconciliation(taskId, error.getCode(), "提交任务使用的上游凭据已不可用,执行结果未知;冻结额度保留,等待核账", false);
|
||||
return;
|
||||
}
|
||||
// Even a permanent authentication/configuration error only proves
|
||||
@@ -487,9 +516,9 @@ public class AsyncTaskService {
|
||||
if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) {
|
||||
// Persistence failures must roll back and remain pollable, rather
|
||||
// than being mistaken for upstream failures and refunded.
|
||||
finishTask(taskId, status, progress, result,
|
||||
completeObserved(outcomes.observeTerminal(taskId, status, result,
|
||||
"failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null,
|
||||
"failed".equals(status) ? "上游异步任务失败" : null);
|
||||
"failed".equals(status) ? "上游异步任务失败" : null));
|
||||
} else {
|
||||
updatePollingState(taskId, progress, null, null);
|
||||
}
|
||||
@@ -497,7 +526,7 @@ public class AsyncTaskService {
|
||||
|
||||
private void updatePollingState(String taskId, Integer progress, String code, String message) {
|
||||
inTransaction(() -> {
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running')",
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL",
|
||||
progress, code == null ? null : writeJson(Map.of("code", code, "message", message, "retryable", true)), now(), taskId);
|
||||
if (changed > 0) usage.updateTask(taskId, "running", null, code, message);
|
||||
return null;
|
||||
@@ -515,10 +544,12 @@ public class AsyncTaskService {
|
||||
Object raw = response.get("status");
|
||||
if (raw == null && response.get("data") instanceof Map<?,?> data) raw = data.get("status");
|
||||
String value = String.valueOf(raw == null ? "running" : raw).toLowerCase(java.util.Locale.ROOT);
|
||||
if (value.contains("success") || value.equals("completed")) return "succeeded";
|
||||
if (value.contains("fail") || value.equals("error") || value.equals("banned")) return "failed";
|
||||
if (value.contains("cancel")) return "cancelled";
|
||||
return "running";
|
||||
return switch (value) {
|
||||
case "success", "succeeded", "completed" -> "succeeded";
|
||||
case "failed", "failure", "error", "banned" -> "failed";
|
||||
case "cancelled", "canceled" -> "cancelled";
|
||||
default -> "running";
|
||||
};
|
||||
}
|
||||
|
||||
private int progress(Map<String,Object> response, String status) {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
/** Durable provider observations only. No upstream calls, wallet mutation, retry or refund policy. */
|
||||
final class TaskOutcomeJournal {
|
||||
record Completion(String taskId, String status, Map<String,Object> result, String code, String message) { }
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper mapper;
|
||||
private final TransactionTemplate transactions;
|
||||
private final UsageAuditService usage;
|
||||
|
||||
TaskOutcomeJournal(JdbcTemplate jdbc, ObjectMapper mapper, TransactionTemplate transactions, UsageAuditService usage) {
|
||||
this.jdbc = jdbc;
|
||||
this.mapper = mapper;
|
||||
this.transactions = transactions;
|
||||
this.usage = usage;
|
||||
}
|
||||
|
||||
Completion observeTerminal(String taskId, String status, Map<String,Object> result, String code, String message) {
|
||||
if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal outcome required");
|
||||
return transaction(() -> {
|
||||
var row = jdbc.queryForMap("SELECT status,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE id=? FOR UPDATE", taskId);
|
||||
if (!List.of("queued", "running", "reconciling").contains(row.get("status"))) return null;
|
||||
if (row.get("completion_status") != null) return completion(taskId, row);
|
||||
jdbc.update("UPDATE model_tasks SET completion_status=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),updated_at=now() WHERE id=?",
|
||||
status, json(result), code == null ? null : json(Map.of("code", code, "message", message)), taskId);
|
||||
return new Completion(taskId, status, result, code, message);
|
||||
});
|
||||
}
|
||||
|
||||
List<Completion> pendingCompletions() {
|
||||
return jdbc.queryForList("SELECT id,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE status IN ('running','reconciling') AND completion_status IS NOT NULL ORDER BY updated_at LIMIT 50")
|
||||
.stream().map(row -> completion(String.valueOf(row.get("id")), row)).toList();
|
||||
}
|
||||
|
||||
void accepted(String taskId, String providerTaskId, Object keyFingerprint, OffsetDateTime submittedAt, OffsetDateTime deadline) {
|
||||
transaction(() -> {
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=? WHERE id=? AND status IN ('running','reconciling') AND provider_task_id IS NULL AND completion_status IS NULL",
|
||||
providerTaskId, keyFingerprint, deadline, submittedAt, taskId);
|
||||
if (changed > 0) usage.updateTask(taskId, "running", null, null, null);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
void requireReconciliation(String taskId, String code, String message, boolean interruptedClaim) {
|
||||
transaction(() -> {
|
||||
String claimGuard = interruptedClaim
|
||||
? " AND provider_task_id IS NULL AND updated_at < now()-interval '2 minutes'" : "";
|
||||
int changed = jdbc.update("UPDATE model_tasks SET status='reconciling',error=CAST(? AS jsonb),reconciliation_required_at=now(),updated_at=now() WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL" + claimGuard,
|
||||
json(Map.of("code", code, "message", message, "retryable", false, "requiresReconciliation", true)), taskId);
|
||||
if (changed > 0) {
|
||||
usage.updateTask(taskId, "reconciling", null, code, message);
|
||||
jdbc.update("UPDATE usage_events SET cost_status='pending',cost_amount=NULL WHERE provider_task_id=? AND event_kind='task'", taskId);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private Completion completion(String taskId, Map<String,Object> row) {
|
||||
Map<String,Object> error = map(row.get("error"));
|
||||
return new Completion(taskId, String.valueOf(row.get("completion_status")), map(row.get("result")),
|
||||
error == null ? null : (String) error.get("code"), error == null ? null : (String) error.get("message"));
|
||||
}
|
||||
|
||||
private Map<String,Object> map(Object value) {
|
||||
if (value == null) return null;
|
||||
try { return mapper.readValue(String.valueOf(value), new TypeReference<>() {}); }
|
||||
catch (JsonProcessingException exception) { throw new IllegalStateException("Invalid persisted provider observation"); }
|
||||
}
|
||||
|
||||
private String json(Object value) {
|
||||
if (value == null) return null;
|
||||
try { return mapper.writeValueAsString(value); }
|
||||
catch (JsonProcessingException exception) { throw new IllegalStateException("Provider observation cannot be encoded"); }
|
||||
}
|
||||
|
||||
private <T> T transaction(Supplier<T> work) {
|
||||
return transactions == null ? work.get() : transactions.execute(status -> work.get());
|
||||
}
|
||||
}
|
||||
@@ -157,15 +157,17 @@ public class UsageAuditService {
|
||||
WITH pending AS (
|
||||
SELECT model_id,
|
||||
COUNT(*) FILTER (WHERE status='running') AS running,
|
||||
COUNT(*) FILTER (WHERE status='queued') AS queued
|
||||
FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id
|
||||
COUNT(*) FILTER (WHERE status='queued') AS queued,
|
||||
COUNT(*) FILTER (WHERE status='reconciling') AS reconciling
|
||||
FROM model_tasks WHERE status IN ('queued','running','reconciling') GROUP BY model_id
|
||||
)
|
||||
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
|
||||
CASE WHEN m.max_concurrency = 0 THEN 0
|
||||
ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency,
|
||||
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
|
||||
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued
|
||||
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued,
|
||||
COALESCE(pending.reconciling, 0) AS reconciling
|
||||
FROM model_catalog m JOIN model_providers p ON p.id=m.provider_id
|
||||
JOIN model_definitions d ON d.id=m.logical_model_id
|
||||
LEFT JOIN pending ON pending.model_id=m.id
|
||||
@@ -182,6 +184,7 @@ public class UsageAuditService {
|
||||
group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0);
|
||||
group.put("running", rs.getLong("running"));
|
||||
group.put("queued", rs.getLong("queued"));
|
||||
group.put("reconciling", rs.getLong("reconciling"));
|
||||
return group;
|
||||
}, defaultModelLimit);
|
||||
List<String> synchronousIds = groups.stream().filter(group -> !"async".equals(group.get("mode"))).map(group -> group.get("id").toString()).toList();
|
||||
@@ -193,12 +196,13 @@ public class UsageAuditService {
|
||||
group.put("running", current == null ? null : current.running());
|
||||
group.put("queued", current == null ? null : current.queued());
|
||||
}
|
||||
// Keep disabled groups visible while their queued or running work still exists.
|
||||
// Keep disabled groups visible while any work or unresolved financial hold still exists.
|
||||
groups = groups.stream().filter(group -> Boolean.TRUE.equals(group.get("routeEnabled"))
|
||||
|| positive(group.get("running")) || positive(group.get("queued"))).toList();
|
||||
|| positive(group.get("running")) || positive(group.get("queued")) || positive(group.get("reconciling"))).toList();
|
||||
result.put("capacityGroups", groups);
|
||||
result.put("running", totalCapacity(groups, "running"));
|
||||
result.put("waiting", totalCapacity(groups, "queued"));
|
||||
result.put("reconciling", totalCapacity(groups, "reconciling"));
|
||||
result.put("capacityConfig", Map.of(
|
||||
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
|
||||
"defaultModelConcurrency", defaultModelLimit,
|
||||
|
||||
Reference in new issue
Block a user