package com.moyi.controlplane.auth; import com.fasterxml.jackson.databind.ObjectMapper; import com.moyi.controlplane.common.ApiException; import com.moyi.controlplane.common.ApiExceptionHandler; import com.moyi.controlplane.security.SecretCipher; import java.util.ArrayList; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import javax.sql.DataSource; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.Configuration; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.transaction.IllegalTransactionStateException; import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.support.TransactionSynchronizationManager; import org.springframework.transaction.support.TransactionTemplate; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; @EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL",matches="jdbc:postgresql://127[.]0[.]0[.]1:[0-9]+/.+") class VerificationDispatchPostgresTest { @Configuration @EnableTransactionManagement(proxyTargetClass=true) static class Transactions {} private static final String ACTIVE="active@example.test", UNKNOWN="unknown@example.test", DISABLED="disabled@example.test"; private final ObjectMapper json=new ObjectMapper().findAndRegisterModules(); private final SecretCipher cipher=new SecretCipher("isolated-delivery-encryption"); private DriverManagerDataSource base; private String schema, invitation; private JdbcTemplate jdbc; private AnnotationConfigApplicationContext context; private VerificationDispatchService dispatch; private VerificationService verification; private MockMvc http; private final TestSender sender=new TestSender(); static class TestSender implements VerificationSender { final Map codes=new ConcurrentHashMap<>(); final AtomicInteger calls=new AtomicInteger(); volatile boolean available=true, fail=false; volatile CountDownLatch arrived, release; volatile VerificationPurpose lastPurpose; public boolean available(String channel) { return available; } public void send(String channel,String account,String code) { send(channel,account,code,VerificationPurpose.REGISTRATION); } public void send(String channel,String account,String code,VerificationPurpose purpose) { assertFalse(TransactionSynchronizationManager.isActualTransactionActive(),"Network delivery must not hold a DB transaction"); calls.incrementAndGet(); lastPurpose=purpose; if (arrived!=null) arrived.countDown(); if (release!=null) try { assertTrue(release.await(10,TimeUnit.SECONDS)); } catch (InterruptedException error) { Thread.currentThread().interrupt(); throw new IllegalStateException("Interrupted fixture"); } if (fail) throw new ApiException("VERIFICATION_SEND_FAILED","Synthetic provider detail must stay private",502); codes.put(account,code); } } @BeforeEach void setup() { String url=System.getenv("MOYI_AUTH_TEST_JDBC_URL"); schema="verification_dispatch_"+UUID.randomUUID().toString().replace("-",""); base=new DriverManagerDataSource(url,"moyi_role_test",""); var source=new DriverManagerDataSource(url+"?currentSchema="+schema,"moyi_role_test",""); Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate(); jdbc=new JdbcTemplate(source); UUID creator=UUID.randomUUID(); invitation="synthetic-invite-"+UUID.randomUUID(); jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name,status) VALUES (?,?,'email','not-a-login-hash','Synthetic','active'),(?,?,'email','not-a-login-hash','Synthetic','disabled')",creator,ACTIVE,UUID.randomUUID(),DISABLED); jdbc.update("INSERT INTO invitation_codes(id,code_hash,code_prefix,label,max_uses,created_by) VALUES (?,?,?,'Synthetic',100,?)",UUID.randomUUID(),AuthService.hashSecret(invitation),"synthetic",creator); context=new AnnotationConfigApplicationContext(); context.register(Transactions.class); context.registerBean(DataSource.class,()->source); context.registerBean(DataSourceTransactionManager.class,()->new DataSourceTransactionManager(source)); context.registerBean(VerificationDispatchService.class,()->new VerificationDispatchService(jdbc,sender,cipher,json,context.getBean(DataSourceTransactionManager.class))); context.registerBean(VerificationService.class,()->new VerificationService(jdbc,context.getBean(VerificationDispatchService.class),"isolated-pepper")); context.refresh(); dispatch=context.getBean(VerificationDispatchService.class); verification=context.getBean(VerificationService.class); http=MockMvcBuilders.standaloneSetup(new AuthController(mock(AuthService.class),verification,mock(VerifiedAccountService.class),168,true)) .setControllerAdvice(new ApiExceptionHandler()).build(); } @AfterEach void cleanup() { if (sender.release!=null) sender.release.countDown(); if (context!=null) context.close(); if (base!=null && schema!=null) new JdbcTemplate(base).execute("DROP SCHEMA "+schema+" CASCADE"); } private UUID delivery(String account,VerificationPurpose purpose) { return jdbc.queryForObject("SELECT delivery_id FROM "+purpose.table+" WHERE account=?",UUID.class,account); } private void releaseQuota(String account) { jdbc.update("UPDATE verification_send_events SET created_at=clock_timestamp()-interval '61 seconds' WHERE account=?",account); } private void assertTerminal(UUID id,String state) { var row=jdbc.queryForMap("SELECT status,payload_ciphertext,finished_at FROM verification_delivery_outbox WHERE id=?",id); assertEquals(state,row.get("status")); assertNull(row.get("payload_ciphertext")); assertNotNull(row.get("finished_at")); } @Test void resetRequestsHaveIdenticalReceiptsAndNoRequestSideDeliveryForAllAccountStates() throws Exception { String reference=null; for (String account : java.util.List.of(ACTIVE,UNKNOWN,DISABLED)) { var response=http.perform(post("/api/auth/password-reset/code").contentType("application/json").content(json.writeValueAsBytes(Map.of("account",account)))) .andExpect(status().isOk()).andExpect(jsonPath("$.sent").value(true)).andExpect(jsonPath("$.code").doesNotExist()) .andExpect(jsonPath("$.token").doesNotExist()).andExpect(cookie().doesNotExist("moyi_session")).andReturn().getResponse().getContentAsString(); if (reference==null) reference=response; else assertEquals(json.readTree(reference),json.readTree(response)); } assertEquals(0,sender.calls.get()); assertEquals(3,jdbc.queryForObject("SELECT count(*) FROM verification_send_events",Integer.class)); assertEquals(3,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox WHERE status='queued'",Integer.class)); assertEquals(3,jdbc.queryForObject("SELECT count(*) FROM password_reset_codes",Integer.class)); } @Test void knownAndUnknownRepeatedRequestsExposeTheSameLimit() throws Exception { for (String account : java.util.List.of(ACTIVE,UNKNOWN,DISABLED)) { verification.sendReset(account,"127.0.0.1"); http.perform(post("/api/auth/password-reset/code").contentType("application/json").content(json.writeValueAsBytes(Map.of("account",account)))) .andExpect(status().isTooManyRequests()).andExpect(jsonPath("$.error.code").value("VERIFICATION_RATE_LIMIT")); } assertEquals(3,jdbc.queryForObject("SELECT count(*) FROM verification_send_events",Integer.class)); } @Test void workerDeliversOnlyToActiveUsersAndPurgesAllFinishedPayloads() { for (String account : java.util.List.of(ACTIVE,UNKNOWN,DISABLED)) verification.sendReset(account,"127.0.0.1"); for (int i=0;i<3;i++) assertTrue(dispatch.dispatchOne()); assertEquals(1,sender.calls.get()); assertEquals(VerificationPurpose.PASSWORD_RESET,sender.lastPurpose); assertTrue(sender.codes.containsKey(ACTIVE)); assertFalse(sender.codes.containsKey(UNKNOWN)); assertFalse(sender.codes.containsKey(DISABLED)); assertTerminal(delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET),"sent"); assertTerminal(delivery(UNKNOWN,VerificationPurpose.PASSWORD_RESET),"discarded"); assertTerminal(delivery(DISABLED,VerificationPurpose.PASSWORD_RESET),"discarded"); } @Test void payloadContainsNoPlaintextAccountOrOtpAndIsBoundToItsGeneration() throws Exception { verification.sendReset(ACTIVE,"127.0.0.1"); UUID id=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); String stored=jdbc.queryForObject("SELECT payload_ciphertext FROM verification_delivery_outbox WHERE id=?",String.class,id); var decoded=json.readTree(cipher.decrypt(stored)); assertTrue(stored.startsWith("v1$")); assertFalse(stored.contains(ACTIVE)); assertFalse(stored.contains("account")); assertEquals(id.toString(),decoded.path("id").asText()); assertEquals("password-reset",decoded.path("purpose").asText()); assertTrue(decoded.path("code").asText().matches("[0-9]{6}")); assertNotEquals(decoded.path("code").asText(),jdbc.queryForObject("SELECT code_hash FROM password_reset_codes WHERE account=?",String.class,ACTIVE)); } @Test void providerFailureStaysOutOfTheReceiptAndIsNotRetried() { sender.fail=true; assertEquals(true,verification.sendReset(ACTIVE,"127.0.0.1").get("sent")); UUID id=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); assertTrue(dispatch.dispatchOne()); assertTerminal(id,"failed"); assertEquals(1,sender.calls.get()); assertFalse(dispatch.dispatchOne()); assertEquals(1,sender.calls.get()); assertTrue(jdbc.queryForObject("SELECT consumed FROM password_reset_codes WHERE delivery_id=?",Boolean.class,id)); assertEquals("DELIVERY_FAILED",jdbc.queryForObject("SELECT failure_code FROM verification_delivery_outbox WHERE id=?",String.class,id)); assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM verification_send_events",Integer.class)); } @Test void sendingDoesNotHoldTheGlobalQuotaLockOrADatabaseTransaction() throws Exception { verification.sendReset(ACTIVE,"127.0.0.1"); sender.arrived=new CountDownLatch(1); sender.release=new CountDownLatch(1); try (var executor=Executors.newFixedThreadPool(2)) { var worker=executor.submit(dispatch::dispatchOne); assertTrue(sender.arrived.await(3,TimeUnit.SECONDS)); var request=executor.submit(()->verification.sendReset(UNKNOWN,"127.0.0.2")); assertEquals(true,request.get(3,TimeUnit.SECONDS).get("sent")); assertEquals("sending",jdbc.queryForObject("SELECT status FROM verification_delivery_outbox WHERE id=?",String.class,delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET))); sender.release.countDown(); assertTrue(worker.get(3,TimeUnit.SECONDS)); } } @Test void supersededQueueEntriesCannotDeliverOldCodesOrInvalidateNewOnes() { verification.sendReset(ACTIVE,"127.0.0.1"); UUID old=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); releaseQuota(ACTIVE); verification.sendReset(ACTIVE,"127.0.0.1"); UUID current=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); assertNotEquals(old,current); assertTrue(dispatch.dispatchOne()); assertTerminal(old,"discarded"); assertEquals(0,sender.calls.get()); assertFalse(jdbc.queryForObject("SELECT consumed FROM password_reset_codes WHERE delivery_id=?",Boolean.class,current)); assertTrue(dispatch.dispatchOne()); assertTerminal(current,"sent"); assertEquals(1,sender.calls.get()); assertEquals(ACTIVE,verification.consumeReset(ACTIVE,sender.codes.get(ACTIVE))); } @Test void oldInFlightFailureDoesNotConsumeTheReplacementCode() throws Exception { verification.sendReset(ACTIVE,"127.0.0.1"); UUID old=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); sender.fail=true; sender.arrived=new CountDownLatch(1); sender.release=new CountDownLatch(1); try (var executor=Executors.newSingleThreadExecutor()) { var worker=executor.submit(dispatch::dispatchOne); assertTrue(sender.arrived.await(3,TimeUnit.SECONDS)); releaseQuota(ACTIVE); verification.sendReset(ACTIVE,"127.0.0.2"); UUID replacement=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); sender.release.countDown(); assertTrue(worker.get(3,TimeUnit.SECONDS)); assertTerminal(old,"failed"); assertFalse(jdbc.queryForObject("SELECT consumed FROM password_reset_codes WHERE delivery_id=?",Boolean.class,replacement)); sender.fail=false; sender.arrived=null; sender.release=null; assertTrue(dispatch.dispatchOne()); assertTerminal(replacement,"sent"); } } @Test void expiredAndAmbiguousSendingRowsAreErasedWithoutAutomaticReplay() { verification.sendReset(ACTIVE,"127.0.0.1"); UUID id=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); jdbc.update("UPDATE verification_delivery_outbox SET status='sending',started_at=clock_timestamp() WHERE id=?",id); assertFalse(dispatch.dispatchOne()); assertEquals(0,sender.calls.get()); jdbc.update("UPDATE verification_delivery_outbox SET expires_at=clock_timestamp()-interval '1 second' WHERE id=?",id); assertFalse(dispatch.dispatchOne()); assertTerminal(id,"discarded"); assertEquals(0,sender.calls.get()); jdbc.update("UPDATE verification_delivery_outbox SET finished_at=clock_timestamp()-interval '3 days' WHERE id=?",id); assertFalse(dispatch.dispatchOne()); assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox",Integer.class)); } @Test void disabledAfterEnqueueAndAlreadyConsumedCodesAreNotSent() { verification.sendReset(ACTIVE,"127.0.0.1"); UUID id=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); jdbc.update("UPDATE users SET status='disabled' WHERE account=?",ACTIVE); assertTrue(dispatch.dispatchOne()); assertTerminal(id,"discarded"); jdbc.update("UPDATE users SET status='active' WHERE account=?",ACTIVE); releaseQuota(ACTIVE); verification.sendReset(ACTIVE,"127.0.0.1"); id=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); jdbc.update("UPDATE password_reset_codes SET consumed=true WHERE delivery_id=?",id); assertTrue(dispatch.dispatchOne()); assertTerminal(id,"discarded"); assertEquals(0,sender.calls.get()); } @Test void missingEncryptionOrDisabledSenderIsAccountIndependentAndCreatesNoWork() { var missingKey=new VerificationDispatchService(jdbc,sender,new SecretCipher(""),json,context.getBean(DataSourceTransactionManager.class)); try { assertFalse(missingKey.available("email")); assertFalse(missingKey.available("phone")); } finally { missingKey.close(); } sender.available=false; for (String account : java.util.List.of(ACTIVE,UNKNOWN,DISABLED)) { assertEquals("VERIFICATION_UNAVAILABLE",assertThrows(ApiException.class,()->verification.sendReset(account,"127.0.0.1")).getCode()); } assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox",Integer.class)); assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM verification_send_events",Integer.class)); } @Test void queueFailureRollsBackTheCodeAndQuotaRecordTogether() { jdbc.execute("ALTER TABLE verification_delivery_outbox ADD CONSTRAINT injected_failure CHECK (purpose <> 'password-reset') NOT VALID"); assertThrows(DataIntegrityViolationException.class,()->verification.sendReset(ACTIVE,"127.0.0.1")); for (String table : java.util.List.of("verification_delivery_outbox","verification_send_events","password_reset_codes")) { assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM "+table,Integer.class)); } jdbc.execute("ALTER TABLE verification_delivery_outbox DROP CONSTRAINT injected_failure"); assertEquals(true,verification.sendReset(ACTIVE,"127.0.0.1").get("sent")); } @Test void concurrentRequestsAcrossConnectionsCannotDuplicateTheSameAccountAdmission() throws Exception { var start=new CountDownLatch(1); var results=new ArrayList(); try (var executor=Executors.newFixedThreadPool(2)) { var requests=new ArrayList>(); for (int i=0;i<2;i++) requests.add(executor.submit(()->{ assertTrue(start.await(3,TimeUnit.SECONDS)); try { verification.sendReset(UNKNOWN,"127.0.0.1"); return "accepted"; } catch (ApiException error) { return error.getCode(); } })); start.countDown(); for (var request : requests) results.add(request.get(5,TimeUnit.SECONDS)); } assertEquals(1,results.stream().filter("accepted"::equals).count()); assertEquals(1,results.stream().filter("VERIFICATION_RATE_LIMIT"::equals).count()); assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox",Integer.class)); } @Test void concurrentWorkerClaimsSendAGenerationAtMostOnce() throws Exception { verification.sendReset(ACTIVE,"127.0.0.1"); var start=new CountDownLatch(1); try (var executor=Executors.newFixedThreadPool(2)) { var first=executor.submit(()->{start.await(); return dispatch.dispatchOne();}); var second=executor.submit(()->{start.await(); return dispatch.dispatchOne();}); start.countDown(); assertNotEquals(first.get(5,TimeUnit.SECONDS),second.get(5,TimeUnit.SECONDS)); } assertEquals(1,sender.calls.get()); assertTerminal(delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET),"sent"); } @Test void accountDailySourceHourlyAndGlobalBudgetsIncludeUnknownAccounts() { for (int i=0;i<10;i++) { verification.sendReset(UNKNOWN,"127.0.0.1"); releaseQuota(UNKNOWN); } assertEquals("VERIFICATION_RATE_LIMIT",assertThrows(ApiException.class,()->verification.sendReset(UNKNOWN,"127.0.0.2")).getCode()); jdbc.update("DELETE FROM verification_send_events"); jdbc.execute("INSERT INTO verification_send_events(id,account,ip_address) SELECT md5('source-'||g)::uuid,'synthetic-'||g||'@example.test','127.0.0.9' FROM generate_series(1,30) g"); assertEquals("VERIFICATION_RATE_LIMIT",assertThrows(ApiException.class,()->verification.sendReset(ACTIVE,"127.0.0.9")).getCode()); assertEquals("VERIFICATION_RATE_LIMIT",assertThrows(ApiException.class,()->verification.sendReset("other@example.test","127.0.0.9")).getCode()); jdbc.update("DELETE FROM verification_send_events"); jdbc.execute("INSERT INTO verification_send_events(id,account,ip_address) SELECT md5('global-'||g)::uuid,'synthetic-'||g||'@example.test','192.0.2.1' FROM generate_series(1,1000) g"); assertEquals("VERIFICATION_RATE_LIMIT",assertThrows(ApiException.class,()->verification.sendReset(ACTIVE,"127.0.0.1")).getCode()); assertEquals("VERIFICATION_RATE_LIMIT",assertThrows(ApiException.class,()->verification.sendReset("another@example.test","127.0.0.2")).getCode()); } @Test void crossPurposeQuotaIsSharedWhileCodesRemainPurposeBound() { verification.send(UNKNOWN,invitation,"127.0.0.1"); assertEquals("VERIFICATION_RATE_LIMIT",assertThrows(ApiException.class,()->verification.sendReset(UNKNOWN,"127.0.0.1")).getCode()); assertTrue(dispatch.dispatchOne()); String code=sender.codes.get(UNKNOWN); assertEquals(VerificationPurpose.REGISTRATION,sender.lastPurpose); verification.consume(UNKNOWN,code); assertEquals("VERIFICATION_INVALID",assertThrows(ApiException.class,()->verification.consumeReset(UNKNOWN,code)).getCode()); } @Test void allInvalidResetCodeStatesHaveTheSamePublicError() { String expected=assertThrows(ApiException.class,()->verification.consumeReset(UNKNOWN,"000000")).getMessage(); verification.sendReset(ACTIVE,"127.0.0.1"); assertTrue(dispatch.dispatchOne()); String correct=sender.codes.get(ACTIVE), wrong="000000".equals(correct)?"111111":"000000"; assertEquals(expected,assertThrows(ApiException.class,()->verification.consumeReset(ACTIVE,wrong)).getMessage()); jdbc.update("UPDATE password_reset_codes SET attempts=5 WHERE account=?",ACTIVE); assertEquals(expected,assertThrows(ApiException.class,()->verification.consumeReset(ACTIVE,correct)).getMessage()); jdbc.update("UPDATE password_reset_codes SET attempts=0,expires_at=clock_timestamp()-interval '1 second' WHERE account=?",ACTIVE); assertEquals(expected,assertThrows(ApiException.class,()->verification.consumeReset(ACTIVE,correct)).getMessage()); } @Test void ciphertextTamperingAndEnvelopeSwappingNeverReachTheSender() { verification.sendReset(ACTIVE,"127.0.0.1"); UUID first=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); jdbc.update("UPDATE verification_delivery_outbox SET payload_ciphertext='v1$invalid$ciphertext' WHERE id=?",first); assertTrue(dispatch.dispatchOne()); assertTerminal(first,"failed"); assertEquals(0,sender.calls.get()); releaseQuota(ACTIVE); verification.sendReset(ACTIVE,"127.0.0.1"); UUID second=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); String valid=jdbc.queryForObject("SELECT payload_ciphertext FROM verification_delivery_outbox WHERE id=?",String.class,second); verification.sendReset(UNKNOWN,"127.0.0.2"); UUID third=delivery(UNKNOWN,VerificationPurpose.PASSWORD_RESET); jdbc.update("UPDATE verification_delivery_outbox SET payload_ciphertext=? WHERE id=?",valid,third); assertTrue(dispatch.dispatchOne()); assertTerminal(second,"sent"); assertTrue(dispatch.dispatchOne()); assertTerminal(third,"failed"); assertEquals(1,sender.calls.get()); } @Test void enqueuingRequiresTheCallingTransaction() { assertThrows(IllegalTransactionStateException.class,()->dispatch.enqueue(VerificationPurpose.REGISTRATION,UNKNOWN,"123456","a".repeat(64))); assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox",Integer.class)); } @Test void failedReceiptPersistenceNeverResendsTheMessage() { verification.sendReset(ACTIVE,"127.0.0.1"); UUID id=delivery(ACTIVE,VerificationPurpose.PASSWORD_RESET); jdbc.execute("ALTER TABLE verification_delivery_outbox ADD CONSTRAINT injected_receipt_failure CHECK (status <> 'sent') NOT VALID"); assertThrows(DataIntegrityViolationException.class,dispatch::dispatchOne); assertEquals(1,sender.calls.get()); assertEquals("sending",jdbc.queryForObject("SELECT status FROM verification_delivery_outbox WHERE id=?",String.class,id)); jdbc.execute("ALTER TABLE verification_delivery_outbox DROP CONSTRAINT injected_receipt_failure"); assertFalse(dispatch.dispatchOne()); assertEquals(1,sender.calls.get()); } @Test void scheduledDeliveryHasOnlyTwoWorkersAndNoUnboundedWaitingQueue() throws Exception { for (int i=0;i<6;i++) verification.send("queued-"+i+"@example.test",invitation,"127.0.0.1"); sender.arrived=new CountDownLatch(2); sender.release=new CountDownLatch(1); try { dispatch.dispatchPending(); assertTrue(sender.arrived.await(3,TimeUnit.SECONDS)); for (int i=0;i<10;i++) dispatch.dispatchPending(); assertEquals(2,sender.calls.get()); assertEquals(2,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox WHERE status='sending'",Integer.class)); assertEquals(4,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox WHERE status='queued'",Integer.class)); } finally { sender.release.countDown(); dispatch.close(); } } @Test void blockedQuotaLockTimesOutAndDoesNotLeavePartialAdmission() throws Exception { var locked=new CountDownLatch(1); var unlock=new CountDownLatch(1); try (var executor=Executors.newSingleThreadExecutor()) { var blocker=executor.submit(()->new TransactionTemplate(context.getBean(DataSourceTransactionManager.class)).executeWithoutResult(status->{ jdbc.queryForList("SELECT pg_advisory_xact_lock(734092615)"); locked.countDown(); try { assertTrue(unlock.await(8,TimeUnit.SECONDS)); } catch (InterruptedException error) { Thread.currentThread().interrupt(); throw new IllegalStateException(error); } })); assertTrue(locked.await(3,TimeUnit.SECONDS)); long start=System.nanoTime(); try { assertThrows(org.springframework.dao.DataAccessException.class,()->verification.sendReset(UNKNOWN,"127.0.0.1")); assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-start)<5000); } finally { unlock.countDown(); } blocker.get(3,TimeUnit.SECONDS); } assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM verification_send_events",Integer.class)); assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM verification_delivery_outbox",Integer.class)); assertEquals(true,verification.sendReset(UNKNOWN,"127.0.0.1").get("sent")); } }