fix async dispatch stats and truthful status
This commit is contained in:
1 parent
9750aceb03
commit
8e9f0d52fd
4 files changed
+110
-48
No files matched your search
@@ -348,19 +348,46 @@ public class CatalogService {
|
||||
auth.audit("catalog."+kind+"_deleted",admin.id(),new AuthService.RequestMeta(null,null),Map.of("id",id));
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public Map<String, Object> overview(String token, AuthService auth) {
|
||||
UserView user = auth.requireUser(token);
|
||||
Integer callsToday = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'call' AND created_at >= CURRENT_DATE", Integer.class, user.id());
|
||||
Integer callsMonth = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'call' AND created_at >= date_trunc('month', now())", Integer.class, user.id());
|
||||
Integer tasks = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'task'", Integer.class, user.id());
|
||||
Integer callsToday = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'call' AND created_at >= CURRENT_DATE", Integer.class, user.id());
|
||||
Integer callsMonth = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'call' AND created_at >= date_trunc('month', now())", Integer.class, user.id());
|
||||
Integer tasks = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'task'", Integer.class, user.id());
|
||||
Map<String, Object> taskCounts = new LinkedHashMap<>();
|
||||
jdbc.query("SELECT status, COUNT(*) AS count FROM model_tasks WHERE user_id = ? GROUP BY status", (rs, row) -> {
|
||||
taskCounts.put(rs.getString("status"), rs.getLong("count"));
|
||||
return null;
|
||||
}, user.id());
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("generatedAt", now());
|
||||
result.put("callsToday", callsToday == null ? 0 : callsToday);
|
||||
result.put("callsThisMonth", callsMonth == null ? 0 : callsMonth);
|
||||
result.put("asyncTasks", tasks == null ? 0 : tasks);
|
||||
result.put("tasks", Map.of("queued", 0, "running", 0, "succeeded", 0, "failed", 0));
|
||||
result.put("series", List.of());
|
||||
result.put("ledger", List.of());
|
||||
result.put("tasks", Map.of(
|
||||
"queued", taskCounts.getOrDefault("queued", 0L),
|
||||
"running", taskCounts.getOrDefault("running", 0L),
|
||||
"succeeded", taskCounts.getOrDefault("succeeded", 0L),
|
||||
"failed", taskCounts.getOrDefault("failed", 0L)
|
||||
));
|
||||
result.put("series", jdbc.query(
|
||||
"SELECT to_char(day, 'YYYY-MM-DD') AS day, COUNT(ue.id) AS count " +
|
||||
"FROM generate_series(CURRENT_DATE - INTERVAL '6 days', CURRENT_DATE, INTERVAL '1 day') day " +
|
||||
"LEFT JOIN usage_events ue ON ue.user_id = ? AND ue.deleted_at IS NULL AND ue.event_kind = 'call' " +
|
||||
"AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " +
|
||||
"GROUP BY day ORDER BY day",
|
||||
(rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count")), user.id()));
|
||||
Number averageLatency = jdbc.queryForObject(
|
||||
"SELECT AVG(latency_ms) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status='succeeded' AND latency_ms IS NOT NULL",
|
||||
Number.class, user.id());
|
||||
result.put("averageLatencyMs", averageLatency == null ? null : averageLatency.doubleValue());
|
||||
Number finished = jdbc.queryForObject(
|
||||
"SELECT COUNT(*) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status IN ('succeeded','failed','cancelled')",
|
||||
Number.class, user.id());
|
||||
Number successful = jdbc.queryForObject(
|
||||
"SELECT COUNT(*) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status='succeeded'",
|
||||
Number.class, user.id());
|
||||
result.put("availabilityPercent", finished == null || finished.longValue() == 0 ? null : successful.doubleValue() * 100.0 / finished.doubleValue());
|
||||
var balances=jdbc.queryForList("SELECT balance, reserved_balance FROM user_wallets WHERE user_id=?",user.id());
|
||||
var balance=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("balance");
|
||||
var reserved=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("reserved_balance");
|
||||
|
||||
@@ -18,6 +18,7 @@ import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Supplier;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -33,6 +34,7 @@ public class AsyncTaskService {
|
||||
private final CatalogService catalogService;
|
||||
private final int maxQueuedPerModel;
|
||||
private final int defaultModelConcurrency;
|
||||
private final int defaultKeyConcurrency;
|
||||
private final GatewayGuard gatewayGuard;
|
||||
private final UsageAuditService usage;
|
||||
private final AsyncProviderClient providerClient;
|
||||
@@ -41,7 +43,7 @@ public class AsyncTaskService {
|
||||
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
|
||||
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
|
||||
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, gatewayGuard, usage, null, null);
|
||||
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, gatewayGuard, usage, null, null);
|
||||
}
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
@@ -51,17 +53,19 @@ public class AsyncTaskService {
|
||||
CatalogService catalogService,
|
||||
@Value("${moyi.tasks.max-queued-per-model:100}") int maxQueuedPerModel,
|
||||
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
|
||||
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
|
||||
GatewayGuard gatewayGuard,
|
||||
UsageAuditService usage,
|
||||
AsyncProviderClient providerClient,
|
||||
PlatformTransactionManager transactionManager
|
||||
) {
|
||||
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
|
||||
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
|
||||
this.jdbc = jdbc;
|
||||
this.objectMapper = objectMapper;
|
||||
this.catalogService = catalogService;
|
||||
this.maxQueuedPerModel = maxQueuedPerModel;
|
||||
this.defaultModelConcurrency = defaultModelConcurrency;
|
||||
this.defaultKeyConcurrency = defaultKeyConcurrency;
|
||||
this.gatewayGuard = gatewayGuard;
|
||||
this.usage = usage;
|
||||
this.providerClient = providerClient;
|
||||
@@ -165,10 +169,10 @@ public class AsyncTaskService {
|
||||
private void dispatchQueuedTasks() {
|
||||
if (providerClient == null) return;
|
||||
// A process can die after claiming a task but before receiving an
|
||||
// upstream task id. Return those claims to the queue after a short
|
||||
// grace period so they neither consume a concurrency slot forever nor
|
||||
// leave the user's reservation stuck.
|
||||
jdbc.update("UPDATE model_tasks SET status='queued', updated_at=now() WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes'");
|
||||
// upstream task id. Fail those claims after a grace period. Retrying
|
||||
// them blindly could submit the same generation twice when the
|
||||
// upstream accepted the request just before the process died.
|
||||
recoverInterruptedClaims();
|
||||
for (int i = 0; i < 50; i++) {
|
||||
Map<String,Object> task = claimQueuedTask();
|
||||
if (task == null) return;
|
||||
@@ -183,38 +187,61 @@ public class AsyncTaskService {
|
||||
);
|
||||
jdbc.update("UPDATE model_tasks SET provider_task_id=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), now(), taskId);
|
||||
} catch (RuntimeException exception) {
|
||||
jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status='running'",
|
||||
writeJson(Map.of("message", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage())), now(), now(), taskId);
|
||||
releaseTask(taskId);
|
||||
usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_SUBMIT_FAILED", "上游异步任务创建失败");
|
||||
failTask(taskId, "UPSTREAM_TASK_SUBMIT_FAILED", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String,Object> claimQueuedTask() {
|
||||
String sql = """
|
||||
WITH candidate AS (
|
||||
SELECT t.id
|
||||
FROM model_tasks t
|
||||
JOIN model_catalog m ON m.id=t.model_id
|
||||
WHERE t.status='queued'
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
AND pg_try_advisory_xact_lock(hashtextextended('moyi:async:' || t.model_id, 0))
|
||||
AND (SELECT COUNT(*) FROM model_tasks running
|
||||
WHERE running.model_id=t.model_id AND running.status='running')
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
ORDER BY t.created_at
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE model_tasks t
|
||||
SET status='running', updated_at=now()
|
||||
FROM candidate c
|
||||
WHERE t.id=c.id AND t.status='queued'
|
||||
RETURNING t.id, t.provider_id, t.upstream_model, t.task_type, t.prompt
|
||||
""";
|
||||
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency);
|
||||
return rows.isEmpty() ? null : rows.getFirst();
|
||||
return inTransaction(() -> {
|
||||
// Keep the lock until the status update commits. The previous
|
||||
// pg_try_advisory_xact_lock was executed in autocommit mode, so
|
||||
// the lock ended before the UPDATE and did not coordinate two
|
||||
// workers reliably.
|
||||
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
|
||||
String sql = """
|
||||
SELECT t.id, t.api_key_id, m.provider_id, m.upstream_model, t.task_type, t.prompt
|
||||
FROM model_tasks t
|
||||
JOIN model_catalog m ON m.id=t.model_id
|
||||
JOIN model_providers p ON p.id=m.provider_id
|
||||
WHERE t.status='queued'
|
||||
AND t.provider_task_id IS NULL
|
||||
AND m.status='active' AND m.deleted_at IS NULL
|
||||
AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND (SELECT COUNT(*) FROM model_tasks running_model
|
||||
WHERE running_model.model_id=t.model_id AND running_model.status='running')
|
||||
< COALESCE(m.max_concurrency, ?)
|
||||
AND (SELECT COUNT(*) FROM model_tasks running_key
|
||||
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
|
||||
< ?
|
||||
ORDER BY t.created_at
|
||||
LIMIT 1
|
||||
FOR UPDATE OF t SKIP LOCKED
|
||||
""";
|
||||
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency, defaultKeyConcurrency);
|
||||
if (rows.isEmpty()) return null;
|
||||
Map<String,Object> row = rows.getFirst();
|
||||
jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
private void recoverInterruptedClaims() {
|
||||
List<String> taskIds = inTransaction(() -> jdbc.queryForList(
|
||||
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
|
||||
String.class));
|
||||
for (String taskId : taskIds) {
|
||||
failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交");
|
||||
}
|
||||
}
|
||||
|
||||
private void failTask(String taskId, String code, String message) {
|
||||
int changed = jdbc.update(
|
||||
"UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
|
||||
writeJson(Map.of("code", code, "message", message)), now(), now(), taskId);
|
||||
if (changed == 0) return;
|
||||
releaseTask(taskId);
|
||||
usage.updateTask(taskId, "failed", null, code, message);
|
||||
}
|
||||
|
||||
private Map<String, Object> requireKey(String authorization) {
|
||||
@@ -393,6 +420,10 @@ public class AsyncTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
private <T> T inTransaction(Supplier<T> work) {
|
||||
return transactions == null ? work.get() : transactions.execute(status -> work.get());
|
||||
}
|
||||
|
||||
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
|
||||
|
||||
private void putIfPresent(Map<String,Object> target, String name, ResultSet resultSet, String column) throws SQLException {
|
||||
|
||||
Reference in new issue
Block a user