feat: add audited async task reconciliation and frozen bindings
This commit is contained in:
1 parent
5be06a97c1
commit
c716ec7478
28 files changed
+1312
-76
No files matched your search
+64
@@ -0,0 +1,64 @@
|
||||
package com.moyi.controlplane.gateway;
|
||||
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@EnabledIfEnvironmentVariable(named="MOYI_GATEWAY_TEST_REDIS_PORT", matches="[0-9]{2,5}")
|
||||
class TaskLeaseRedisTest {
|
||||
private LettuceConnectionFactory factory;
|
||||
private StringRedisTemplate redis;
|
||||
private GatewayGuard guard;
|
||||
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID();
|
||||
private final String prefix = "moyi:test:" + UUID.randomUUID() + ":";
|
||||
private String bucket() { return prefix + "gateway:slots:user:" + userId; }
|
||||
|
||||
@BeforeEach void setup() {
|
||||
factory = new LettuceConnectionFactory("127.0.0.1", Integer.parseInt(System.getenv("MOYI_GATEWAY_TEST_REDIS_PORT")));
|
||||
factory.afterPropertiesSet(); factory.start();
|
||||
redis = new StringRedisTemplate(factory); redis.afterPropertiesSet();
|
||||
guard = new GatewayGuard(redis, 60, 300, 4, prefix);
|
||||
var policies = mock(EnterprisePolicies.class);
|
||||
when(policies.taskLimit(keyId)).thenReturn(new EnterprisePolicies.UserLimit(userId, 120, 1));
|
||||
guard.setEnterprisePolicies(policies);
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() {
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) TransactionSynchronizationManager.clearSynchronization();
|
||||
if (redis != null) redis.delete(bucket());
|
||||
if (factory != null) factory.destroy();
|
||||
}
|
||||
|
||||
@Test void renewalDoesNotConsumeAnotherSlotAndDifferentTaskIsBlocked() {
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
|
||||
assertEquals(1L, redis.opsForZSet().zCard(bucket()));
|
||||
assertFalse(guard.tryTaskUserSlot(keyId, "different-task", 180000));
|
||||
assertTrue(redis.getExpire(bucket(), java.util.concurrent.TimeUnit.MILLISECONDS) > 250000);
|
||||
}
|
||||
|
||||
@Test void rolledBackRenewalKeepsThePreexistingLease() {
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 180000));
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "original-task", 300000));
|
||||
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
assertNotNull(redis.opsForZSet().score(bucket(), "original-task"));
|
||||
}
|
||||
|
||||
@Test void rolledBackNewClaimRemovesOnlyItsOwnLease() {
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
assertTrue(guard.tryTaskUserSlot(keyId, "new-task", 180000));
|
||||
for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) synchronization.afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK);
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
assertNull(redis.opsForZSet().score(bucket(), "new-task"));
|
||||
}
|
||||
}
|
||||
+9
@@ -289,6 +289,15 @@ class AsyncProviderClientTest {
|
||||
} finally { server.stop(0); }
|
||||
}
|
||||
|
||||
@Test void pollingBindingValidationChecksTheExistingCredentialWithoutAnHttpCall() throws Exception {
|
||||
var client = clientFor("http://127.0.0.1:1");
|
||||
String fingerprint = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256").digest("test-key".getBytes(StandardCharsets.UTF_8)));
|
||||
org.junit.jupiter.api.Assertions.assertDoesNotThrow(() -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", fingerprint));
|
||||
assertEquals("UPSTREAM_KEY_NOT_AVAILABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "0".repeat(64))).getCode());
|
||||
assertEquals("TASK_NOT_POLLABLE", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "image", fingerprint)).getCode());
|
||||
assertEquals("INVALID_CREDENTIAL_FINGERPRINT", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "test-key")).getCode());
|
||||
}
|
||||
|
||||
@Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() {
|
||||
var client = clientFor("http://127.0.0.1:1");
|
||||
var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private")));
|
||||
|
||||
+31
-1
@@ -65,9 +65,11 @@ class AsyncTaskPostgresTest {
|
||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId);
|
||||
usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null);
|
||||
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
|
||||
// This fixture knows its original binding; a migration must not guess it for real legacy jobs.
|
||||
jdbc.update("UPDATE model_tasks SET provider_id_snapshot='provider-vast',upstream_model_snapshot='v3.1-20260211' WHERE id=?", taskId);
|
||||
when(catalog.keyOwnerId(keyId)).thenReturn(userId);
|
||||
when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true);
|
||||
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
|
||||
when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "upstreamModel", "v3.1-20260211", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44)));
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() {
|
||||
@@ -97,6 +99,34 @@ class AsyncTaskPostgresTest {
|
||||
return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal());
|
||||
}
|
||||
|
||||
@Test void queuedTaskRetainsAdmittedExecutionBindingWhenRouteIsEdited() {
|
||||
String created = String.valueOf(create("frozen-queue", Map.of("model", "model-mock-3d", "prompt", "chair")).get("id"));
|
||||
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=now() WHERE id=?", taskId);
|
||||
when(provider.submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap())).thenReturn(Map.of("id", "original-route-task"));
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("original-route-task", jdbc.queryForObject("SELECT provider_task_id FROM model_tasks WHERE id=?", String.class, created));
|
||||
verify(provider).submit(eq("provider-vast"), eq("v3.1-20260211"), eq("model-mock-3d"), eq("3d"), anyMap());
|
||||
}
|
||||
|
||||
@Test void acceptedTaskPollsFrozenProviderRatherThanEditedCatalog() {
|
||||
jdbc.update("UPDATE model_catalog SET provider_id='provider-bytedance',upstream_model='different-upstream' WHERE id='model-mock-3d'");
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
|
||||
when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null)).thenReturn(Map.of("status", "running", "progress", 55));
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("running", status());
|
||||
assertEquals(55, jdbc.queryForObject("SELECT progress FROM model_tasks WHERE id=?", Integer.class, taskId));
|
||||
verify(provider).poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null);
|
||||
}
|
||||
|
||||
@Test void acceptedLegacyTaskWithoutProofOfBindingIsNotPolledOnAGuessedRoute() {
|
||||
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL,updated_at=now()-interval '6 seconds' WHERE id=?", taskId);
|
||||
tasks.completeMockTasks();
|
||||
assertEquals("reconciling", status());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() {
|
||||
var input = Map.<String,Object>of("model", "model-mock-3d", "prompt", "one chair");
|
||||
var first = create("client-operation-1", input);
|
||||
|
||||
+295
@@ -0,0 +1,295 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.moyi.controlplane.auth.AuthService;
|
||||
import com.moyi.controlplane.auth.PasswordHasher;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.common.ApiExceptionHandler;
|
||||
import com.moyi.controlplane.gateway.GatewayGuard;
|
||||
import com.moyi.controlplane.usage.UsageAuditService;
|
||||
import jakarta.servlet.http.Cookie;
|
||||
import java.math.BigDecimal;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
|
||||
|
||||
@EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+")
|
||||
class TaskReconciliationPostgresTest {
|
||||
private final ObjectMapper json = new ObjectMapper().findAndRegisterModules().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
|
||||
private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID(), adminId = UUID.randomUUID(), secondAdminId = UUID.randomUUID();
|
||||
private final String taskId = "reconcile-" + UUID.randomUUID();
|
||||
private final String fingerprint = "c".repeat(64);
|
||||
private final AuthService.RequestMeta meta = new AuthService.RequestMeta("127.0.0.1", "isolated-reconciliation-test");
|
||||
private DriverManagerDataSource base;
|
||||
private JdbcTemplate jdbc;
|
||||
private String schema;
|
||||
private AuthService auth;
|
||||
private GatewayGuard guard;
|
||||
private AsyncProviderClient provider;
|
||||
private TaskReconciliationService service;
|
||||
private MockMvc http;
|
||||
|
||||
@BeforeEach void setup() {
|
||||
String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL");
|
||||
schema = "reconciliation_test_" + UUID.randomUUID().toString().replace("-", "");
|
||||
base = new DriverManagerDataSource(url, "moyi_role_test", "");
|
||||
Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate();
|
||||
var source = new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", "");
|
||||
jdbc = new JdbcTemplate(source);
|
||||
var manager = new DataSourceTransactionManager(source);
|
||||
for (var entry : Map.of(userId, "user", adminId, "admin", secondAdminId, "admin").entrySet()) {
|
||||
jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name,role) VALUES (?,?,'username','not-loginable','Test',?)", entry.getKey(), entry.getKey().toString(), entry.getValue());
|
||||
}
|
||||
session("admin-session", adminId); session("second-admin-session", secondAdminId); session("user-session", userId);
|
||||
jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-key',?)", keyId, userId, "b".repeat(64));
|
||||
jdbc.update("INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)", userId);
|
||||
jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot,pricing_snapshot,reconciliation_required_at) VALUES (?,?,?,'model-mock-3d','3d','reconciling','private prompt',1.44,'original-provider-task',?,'provider-vast','v3.1-20260211','{\"unit\":\"CNY / task\",\"price\":1.44}'::jsonb,now())", taskId, userId, keyId, fingerprint);
|
||||
jdbc.update("INSERT INTO gateway_budget_holds(request_id,user_id,key_id,amount) VALUES (?,?,?,1.44)", taskId, userId, keyId);
|
||||
var usage = new UsageAuditService(jdbc, json);
|
||||
usage.recordTask(userId, keyId, "model-mock-3d", "provider-vast", taskId, "reconciling", Map.of("prompt", "private prompt"), null, "UNKNOWN", "pending");
|
||||
auth = spy(new AuthService(jdbc, mock(PasswordHasher.class), json, 168));
|
||||
guard = mock(GatewayGuard.class);
|
||||
when(guard.tryTaskUserSlot(any(), anyString(), anyLong())).thenReturn(true);
|
||||
provider = mock(AsyncProviderClient.class);
|
||||
var settlements = new TaskSettlementService(jdbc, json, guard, usage, manager);
|
||||
service = new TaskReconciliationService(jdbc, json, auth, settlements, provider, guard, usage, manager, 120_000);
|
||||
http = MockMvcBuilders.standaloneSetup(new TaskReconciliationController(service))
|
||||
.setControllerAdvice(new ApiExceptionHandler()).setMessageConverters(new MappingJackson2HttpMessageConverter(json)).build();
|
||||
}
|
||||
|
||||
@AfterEach void cleanup() { if (base != null && schema != null) new JdbcTemplate(base).execute("DROP SCHEMA " + schema + " CASCADE"); }
|
||||
|
||||
private void session(String token, UUID user) {
|
||||
jdbc.update("INSERT INTO auth_sessions(id,token_hash,user_id,expires_at) VALUES (?,?,?,now()+interval '1 hour')", UUID.randomUUID(), AuthService.hashSecret(token), user);
|
||||
}
|
||||
private OffsetDateTime version() { return jdbc.queryForObject("SELECT updated_at FROM model_tasks WHERE id=?", OffsetDateTime.class, taskId); }
|
||||
private String state() { return jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, taskId); }
|
||||
private BigDecimal wallet(String column) { return jdbc.queryForObject("SELECT " + column + " FROM user_wallets WHERE user_id=?", BigDecimal.class, userId); }
|
||||
private int count(String table) { return jdbc.queryForObject("SELECT count(*) FROM " + table, Integer.class); }
|
||||
private TaskReconciliationCommand command(String action) {
|
||||
return new TaskReconciliationCommand(UUID.randomUUID(), action, version(), "provider-bill/test-001", "Verified against the provider task history and billing record",
|
||||
"resume_polling".equals(action) ? "original-provider-task" : null,
|
||||
"resume_polling".equals(action) ? fingerprint : null,
|
||||
"confirm_succeeded".equals(action) ? new BigDecimal("1.44") : null,
|
||||
"confirm_succeeded".equals(action) ? Map.of("url", "https://example.test/generated.glb") : null);
|
||||
}
|
||||
private Map<String,Object> reconcile(TaskReconciliationCommand command) { return service.reconcile("admin-session", taskId, command, meta); }
|
||||
private void unchanged() {
|
||||
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertEquals(0, count("wallet_entries")); assertEquals(0, count("task_reconciliations"));
|
||||
assertEquals(1, count("gateway_budget_holds"));
|
||||
}
|
||||
|
||||
@Test void realSessionsEnforceAdministratorAndCsrfBarrier() throws Exception {
|
||||
var command = command("confirm_failed");
|
||||
http.perform(get("/api/admin/task-reconciliation")).andExpect(status().isUnauthorized());
|
||||
http.perform(get("/api/admin/task-reconciliation").cookie(new Cookie("moyi_session", "user-session"))).andExpect(status().isForbidden());
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "user-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content(json.writeValueAsBytes(command))).andExpect(status().isForbidden());
|
||||
unchanged();
|
||||
verifyNoInteractions(provider, guard);
|
||||
}
|
||||
|
||||
@Test void failureConfirmationReleasesOnceAndReplayReturnsTheOperationReceipt() throws Exception {
|
||||
var command = command("confirm_failed");
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId).cookie(new Cookie("moyi_session", "admin-session")).header("X-Moyi-Source", "console").contentType("application/json").content(json.writeValueAsBytes(command)))
|
||||
.andExpect(status().isOk()).andExpect(jsonPath("$.task.status").value("failed")).andExpect(jsonPath("$.replayed").value(false));
|
||||
assertEquals(true, reconcile(command).get("replayed"));
|
||||
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals(0, count("wallet_entries")); assertEquals(0, count("gateway_budget_holds"));
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM auth_audit_log WHERE event_type='task.reconciled'", Integer.class));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void successUsesFrozenPriceAndDebitsExactlyOnce() {
|
||||
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":99}'::jsonb WHERE id='model-mock-3d'");
|
||||
var command = command("confirm_succeeded");
|
||||
reconcile(command); reconcile(command);
|
||||
assertEquals("succeeded", state());
|
||||
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||
assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
|
||||
assertEquals(new BigDecimal("1.44000000"), jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId));
|
||||
verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void quoteIsReadOnlyAndUsesTheFrozenVersionNotEditableCatalogPrices() throws Exception {
|
||||
jdbc.update("UPDATE model_catalog SET pricing='{\"unit\":\"CNY / task\",\"price\":999}'::jsonb WHERE id='model-mock-3d'");
|
||||
http.perform(post("/api/admin/task-reconciliation/" + taskId + "/quote").cookie(new Cookie("moyi_session", "admin-session")).contentType("application/json").content("{}"))
|
||||
.andExpect(status().isOk()).andExpect(jsonPath("$.expectedCost").value("1.44"));
|
||||
unchanged();
|
||||
assertEquals(0, count("auth_audit_log"));
|
||||
}
|
||||
|
||||
@Test void concurrentIdenticalOperationsShareOneReceiptAndOneDebit() throws Exception {
|
||||
var command = command("confirm_succeeded");
|
||||
var start = new CountDownLatch(1);
|
||||
try (var executor = Executors.newFixedThreadPool(8)) {
|
||||
var results = new ArrayList<java.util.concurrent.Future<Map<String,Object>>>();
|
||||
for (int i = 0; i < 8; i++) results.add(executor.submit(() -> { assertTrue(start.await(10, TimeUnit.SECONDS)); return reconcile(command); }));
|
||||
start.countDown();
|
||||
int originals = 0;
|
||||
for (var result : results) if (Boolean.FALSE.equals(result.get(30, TimeUnit.SECONDS).get("replayed"))) originals++;
|
||||
assertEquals(1, originals);
|
||||
}
|
||||
assertEquals(1, count("wallet_entries")); assertEquals(1, count("task_reconciliations"));
|
||||
assertEquals(new BigDecimal("8.56000000"), wallet("balance"));
|
||||
}
|
||||
|
||||
@Test void concurrentDifferentDecisionsHaveExactlyOneWinner() throws Exception {
|
||||
var success = command("confirm_succeeded"); var failure = command("confirm_failed");
|
||||
var start = new CountDownLatch(1);
|
||||
try (var executor = Executors.newFixedThreadPool(2)) {
|
||||
var results = new ArrayList<java.util.concurrent.Future<Integer>>();
|
||||
for (var command : List.of(success, failure)) results.add(executor.submit(() -> {
|
||||
assertTrue(start.await(10, TimeUnit.SECONDS));
|
||||
try { reconcile(command); return 200; } catch (ApiException exception) { return exception.getStatus(); }
|
||||
}));
|
||||
start.countDown();
|
||||
assertEquals(List.of(200, 409), results.stream().map(f -> { try { return f.get(30, TimeUnit.SECONDS); } catch (Exception exception) { throw new AssertionError(exception); } }).sorted().toList());
|
||||
}
|
||||
assertEquals(1, count("task_reconciliations")); assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals("succeeded".equals(state()) ? new BigDecimal("8.56000000") : new BigDecimal("10.00000000"), wallet("balance"));
|
||||
}
|
||||
|
||||
@Test void operationIdCannotBeReusedForDifferentActionOrAnotherAdministrator() {
|
||||
var command = command("confirm_failed"); reconcile(command);
|
||||
var changed = new TaskReconciliationCommand(command.requestId(), "confirm_succeeded", command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.44"), Map.of());
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(changed)).getStatus());
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> service.reconcile("second-admin-session", taskId, command, meta)).getStatus());
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
}
|
||||
|
||||
@Test void staleVersionAndWrongExpectedAmountLeaveFinancialStateUnchanged() {
|
||||
var command = command("confirm_succeeded");
|
||||
var wrong = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), null, null, new BigDecimal("1.45"), command.result());
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(wrong)).getStatus()); unchanged();
|
||||
jdbc.update("UPDATE model_tasks SET updated_at=updated_at+interval '1 second' WHERE id=?", taskId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command)).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void missingEvidenceCannotAuthorizeARefund() {
|
||||
var command = command("confirm_failed");
|
||||
var invalid = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), "", "short", null, null, null, null);
|
||||
assertEquals(422, assertThrows(ApiException.class, () -> reconcile(invalid)).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void missingTokenUsageCannotReleaseTheReservationUnderASuccessLabel() {
|
||||
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
|
||||
assertEquals("TASK_USAGE_REQUIRED", assertThrows(ApiException.class, () -> reconcile(command("confirm_succeeded"))).getCode()); unchanged();
|
||||
}
|
||||
|
||||
@Test void auditFailureRollsBackTaskWalletUsageAndReceiptTogether() {
|
||||
var command = command("confirm_succeeded");
|
||||
doThrow(new IllegalStateException("audit database unavailable")).when(auth).audit(eq("task.reconciled"), eq(adminId), any(), anyMap());
|
||||
assertThrows(IllegalStateException.class, () -> reconcile(command)); unchanged();
|
||||
assertNull(jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId));
|
||||
assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId));
|
||||
}
|
||||
|
||||
@Test void resumeOnlyQueriesTheOriginalBindingWithoutChangingTheWallet() {
|
||||
var command = command("resume_polling"); reconcile(command); reconcile(command);
|
||||
assertEquals("running", state());
|
||||
assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance"));
|
||||
assertEquals(new BigDecimal("10.00000000"), wallet("balance"));
|
||||
assertEquals(1, count("gateway_budget_holds")); assertEquals(0, count("wallet_entries"));
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
verify(provider, times(1)).requirePollableBinding("provider-vast", "model-mock-3d", "v3.1-20260211", "3d", fingerprint);
|
||||
verifyNoMoreInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void differentProviderTaskCannotBeAttachedToAnAcceptedJob() {
|
||||
var command = command("resume_polling");
|
||||
var other = new TaskReconciliationCommand(command.requestId(), command.action(), command.expectedUpdatedAt(), command.evidenceReference(), command.note(), "some-other-task", fingerprint, null, null);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(other)).getStatus()); unchanged(); verifyNoInteractions(provider);
|
||||
}
|
||||
|
||||
@Test void sameProvenUpstreamTaskCannotBeAssociatedWithTwoLocalTasks() {
|
||||
assertThrows(DataAccessException.class, () -> jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id,provider_key_fingerprint,provider_id_snapshot,upstream_model_snapshot) VALUES ('duplicate-local-task',?,?,'model-mock-3d','3d','running','',0,'original-provider-task',?,'provider-vast','v3.1-20260211')", userId, keyId, fingerprint));
|
||||
unchanged();
|
||||
}
|
||||
|
||||
@Test void validTokenUsageIsQuotedAndSettledOnceUsingTheFrozenSchedule() {
|
||||
jdbc.update("UPDATE model_tasks SET pricing_snapshot='{\"unit\":\"CNY / 1M tokens\",\"input\":1,\"output\":10,\"taskBilling\":\"openai-image-tokens\"}'::jsonb WHERE id=?", taskId);
|
||||
var result = Map.<String,Object>of("usage", Map.of("input_tokens", 100, "output_tokens", 100));
|
||||
var quote = service.quote("admin-session", taskId, result);
|
||||
assertEquals("0.00110000", quote.get("expectedCost"));
|
||||
var command = new TaskReconciliationCommand(UUID.randomUUID(), "confirm_succeeded", version(), "provider-token-bill", "Verified exact input and output token usage from provider record", null, null, new BigDecimal("0.0011"), result);
|
||||
reconcile(command); reconcile(command);
|
||||
assertEquals(new BigDecimal("9.99890000"), wallet("balance"));
|
||||
assertEquals(0, wallet("reserved_balance").signum());
|
||||
assertEquals(100L, jdbc.queryForObject("SELECT input_units FROM usage_events WHERE provider_task_id=?", Long.class, taskId));
|
||||
assertEquals(1, count("wallet_entries"));
|
||||
}
|
||||
|
||||
@Test void unavailableCredentialAndAmbiguousLegacyRouteCannotResumePolling() {
|
||||
doThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "safe message", 503)).when(provider).requirePollableBinding(anyString(), anyString(), anyString(), anyString(), anyString());
|
||||
assertEquals(503, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
|
||||
jdbc.update("UPDATE model_tasks SET provider_id_snapshot=NULL,upstream_model_snapshot=NULL WHERE id=?", taskId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("resume_polling"))).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void existingObservedOutcomeCannotBeOverruledManually() {
|
||||
jdbc.update("UPDATE model_tasks SET completion_status='succeeded' WHERE id=?", taskId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus()); unchanged();
|
||||
}
|
||||
|
||||
@Test void insufficientHoldIsAConflictNotABalancingPlug() {
|
||||
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
|
||||
assertEquals(409, assertThrows(ApiException.class, () -> reconcile(command("confirm_failed"))).getStatus());
|
||||
assertEquals("reconciling", state()); assertEquals(new BigDecimal("10.00000000"), wallet("balance")); assertEquals(0, count("task_reconciliations"));
|
||||
}
|
||||
|
||||
@Test void receiptsAreAppendOnlyAndNeverContainPromptOrGeneratedContent() {
|
||||
reconcile(command("confirm_succeeded"));
|
||||
String stored = jdbc.queryForObject("SELECT row_to_json(r)::text FROM task_reconciliations r", String.class);
|
||||
assertFalse(stored.contains("private prompt")); assertFalse(stored.contains("generated.glb"));
|
||||
assertThrows(DataAccessException.class, () -> jdbc.update("DELETE FROM task_reconciliations"));
|
||||
assertThrows(DataAccessException.class, () -> jdbc.update("UPDATE task_reconciliations SET note='rewritten history'"));
|
||||
assertEquals(1, count("task_reconciliations"));
|
||||
}
|
||||
|
||||
@Test void listAndHistoryAreBoundedAdministratorViewsWithoutRawPayloads() {
|
||||
var list = service.list("admin-session", 1);
|
||||
assertEquals(1L, list.get("total"));
|
||||
String encoded = assertDoesNotThrow(() -> json.writeValueAsString(list));
|
||||
assertFalse(encoded.contains("private prompt")); assertFalse(encoded.contains("input_payload"));
|
||||
reconcile(command("confirm_failed"));
|
||||
assertEquals(1, ((List<?>) service.detail("admin-session", taskId).get("history")).size());
|
||||
assertEquals(0L, service.list("admin-session", 1).get("total"));
|
||||
}
|
||||
|
||||
@Test void freeFailureHasAnExplicitZeroCostAndNoLedgerEntry() {
|
||||
jdbc.update("UPDATE model_tasks SET reserved_amount=0 WHERE id=?", taskId);
|
||||
jdbc.update("UPDATE user_wallets SET reserved_balance=0 WHERE user_id=?", userId);
|
||||
reconcile(command("confirm_failed"));
|
||||
assertEquals(0, jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?", BigDecimal.class, taskId).signum());
|
||||
assertEquals(0, count("wallet_entries"));
|
||||
}
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package com.moyi.controlplane.task;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class TaskReconciliationResultTest {
|
||||
private final ObjectMapper json = new ObjectMapper();
|
||||
|
||||
@Test void acceptsOnlyNormalizedAssetsAndIntegralUsage() {
|
||||
var result = TaskReconciliationResult.normalize(Map.of("url", "https://example.test/image.png", "usage", Map.of("input_tokens", "10", "output_tokens", 20,
|
||||
"input_tokens_details", Map.of("text_tokens", 2, "image_tokens", 8))), json);
|
||||
assertEquals(10L, ((Map<?,?>) result.get("usage")).get("input_tokens"));
|
||||
}
|
||||
|
||||
@Test void rejectsCredentialsRawPayloadsAndUnsafeAssetUrls() {
|
||||
for (Map<String,Object> value : List.<Map<String,Object>>of(
|
||||
Map.of("Authorization", "secret"), Map.of("base64", "large media"),
|
||||
Map.of("url", "javascript:alert(1)"), Map.of("url", "https://user:password@example.test/a"),
|
||||
Map.of("usage", Map.of("api_key", "secret")), Map.of("usage", Map.of("input_tokens", -1)),
|
||||
Map.of("usage", Map.of("output_tokens", 1.5)), Map.of("usage", Map.of("input_tokens", 100_000_001)))) {
|
||||
assertEquals(422, assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(value, json)).getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Test void rejectsOversizeObjectsAndCommandLoggingOmitsEvidenceAndPayload() {
|
||||
assertThrows(ApiException.class, () -> TaskReconciliationResult.normalize(Map.of("url", "https://example.test/" + "x".repeat(40_000)), json));
|
||||
var command = new TaskReconciliationCommand(java.util.UUID.randomUUID(), "confirm_succeeded", null, "private bill", "private note", null, null, null, Map.of("url", "private result"));
|
||||
assertFalse(command.toString().contains("private"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user