diff --git a/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryLifecycleListener.java b/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryLifecycleListener.java index 2701f60..4dbb961 100644 --- a/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryLifecycleListener.java +++ b/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryLifecycleListener.java @@ -147,7 +147,12 @@ static void handleUnload(Entity entity) { unload(List.of(entity)); } - /** Saves every animal of one entities event in a single transaction instead of one per animal. */ + /** + * Saves every animal of one entities event, and the entity copies taken with them, in a single + * relaxed transaction ({@link HusbandryRepository#saveLoaded}): chunks load and unload all the + * time, and each synced commit holds the server thread for a disk sync. The once-a-minute + * husbandry tick puts these saves on disk ({@link HusbandryRepository#syncRelaxed}). + */ private static void load(Iterable entities) { Writes writes = new Writes(); try { @@ -241,7 +246,7 @@ private static void unload(Entity entity, Writes writes) { } } - /** Rows queued by one load or unload pass; animals are written before their snapshots, as before. */ + /** Rows queued by one load or unload pass; animals are written before their snapshots. */ private static final class Writes { private final List animals = new ArrayList<>(); private final Map snapshots = new HashMap<>(); @@ -256,6 +261,15 @@ void flush() { if (repository == null) { return; } + try { + repository.saveLoaded(animals, snapshots, System.currentTimeMillis(), true); + return; + } catch (SqliteDatabaseException together) { + // One bad row rolls back the whole pass. Save the animals and the snapshots apart, + // each fully synced as before, so only the rows that still fail are lost. + Bukkit.getLogger().warning("[Cooking] Husbandry save of a chunk pass failed as a whole, saving row by row: " + + together.getMessage()); + } try { repository.upsertAnimals(animals); } catch (SqliteDatabaseException batch) { diff --git a/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryRepository.java b/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryRepository.java index b1a37ba..4c092bb 100644 --- a/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryRepository.java +++ b/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryRepository.java @@ -1,6 +1,7 @@ package net.tfminecraft.cooking.husbandry; import java.io.File; +import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -13,6 +14,7 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Consumer; import net.tfminecraft.tlibs.database.SqliteDatabase; import net.tfminecraft.tlibs.database.SqliteDatabaseException; @@ -122,6 +124,13 @@ ON CONFLICT(animal_uuid) DO UPDATE SET data = excluded.data, saved_at = excluded private final SqliteDatabase database; private HusbandryOwnershipStore ownership; + /** + * A relaxed save has been committed and no complete checkpoint from {@link #syncRelaxed()} has + * followed it yet. Only that checkpoint clears this. A later synced commit usually puts the + * relaxed save on disk too, but not always: a commit that changes no page (the same rows + * written again) writes nothing and so syncs nothing. Read and written under the database's lock. + */ + private boolean unsynced; private HusbandryRepository(SqliteDatabase database) { this.database = database; @@ -260,29 +269,153 @@ public void upsertAnimal(HusbandryAnimal animal) { } public void upsertAnimals(Collection animals) { - if (animals == null || animals.isEmpty()) { + List valid = valid(animals); + if (valid.isEmpty()) { return; } + database.runTransaction(connection -> writeAnimals(connection, valid)); + } + + private static List valid(Collection animals) { List valid = new ArrayList<>(); + if (animals == null) { + return valid; + } for (HusbandryAnimal animal : animals) { if (animal == null || animal.uuid() == null) { continue; } valid.add(animal); } - if (valid.isEmpty()) { + return valid; + } + + private static void writeAnimals(Connection connection, List valid) { + try (PreparedStatement statement = connection.prepareStatement(UPSERT_ANIMAL)) { + for (HusbandryAnimal animal : valid) { + bindAnimal(statement, animal); + statement.executeUpdate(); + } + } catch (SQLException e) { + throw new SqliteDatabaseException("Failed to upsert animals", e); + } + } + + /** + * The routine state of loaded animals and the entity copies taken with it, in one transaction: + * the animals first, since a copy is only stored for an animal that has a row. + * + *

A {@code relaxed} save is committed without its own disk sync, which is what a save on the + * server thread costs. A crash or kill of the server process still loses nothing, because the + * commit already reached the operating system. {@link #syncRelaxed()} puts it on disk, and the + * once-a-minute husbandry tick calls that, so an operating-system crash or power loss can only + * roll back relaxed saves made since then. Every other write keeps {@code synchronous=FULL}. + */ + public void saveLoaded(Collection animals, Map snapshots, long savedAt, + boolean relaxed) { + List valid = valid(animals); + Map copies = snapshots == null ? Map.of() : snapshots; + if (valid.isEmpty() && copies.isEmpty()) { return; } - database.runTransaction(connection -> { - try (PreparedStatement statement = connection.prepareStatement(UPSERT_ANIMAL)) { - for (HusbandryAnimal animal : valid) { - bindAnimal(statement, animal); - statement.executeUpdate(); - } - } catch (SQLException e) { - throw new SqliteDatabaseException("Failed to upsert animals", e); + Consumer work = connection -> { + writeAnimals(connection, valid); + writeSnapshots(connection, copies, savedAt); + }; + // One lock for the level change, the transaction and the change back: no other thread's + // write can run between them and be committed without its sync. + synchronized (database) { + if (!relaxed) { + // Relaxed saves before this stay marked: this commit syncs the log only if it + // changes a page, and rows written again unchanged change none. + database.runTransaction(work); + return; } - }); + Throwable failure = null; + try { + // Inside the try: a switch that fails after it took effect is undone below as well. + synchronous("NORMAL"); + // Marked before the commit, not after: TLibs can still throw once the commit has + // gone through, and a spare checkpoint after a save that rolled back costs little. + unsynced = true; + database.runTransaction(work); + } catch (Throwable e) { + // Errors too: TLibs rolls back and rethrows them, and the level must still go back. + failure = e; + throw e; + } finally { + restoreSynced(failure); + } + } + } + + /** + * Back to synced commits after a relaxed save, whatever became of it. A connection that cannot + * be switched back is closed: ownership and estate changes rely on their sync, so husbandry + * then fails loudly until the plugin restarts instead of committing without one. + */ + private void restoreSynced(Throwable failure) { + try { + synchronous("FULL"); + } catch (Throwable restore) { + try { + closeDatabase(); + } catch (Throwable closing) { + restore.addSuppressed(closing); + } + if (failure == null) { + throw new SqliteDatabaseException("Could not restore synced husbandry commits", restore); + } + // The save's own failure is the one reported; this one travels with it. + failure.addSuppressed(restore); + } + } + + /** Sets how commits on this connection reach the disk. Only possible outside a transaction. */ + void synchronous(String level) { + database.execute("PRAGMA synchronous=" + level); + } + + /** The connection's {@code PRAGMA synchronous} level (2 = FULL, 1 = NORMAL). For tests. */ + int synchronousLevel() { + return queryOne("PRAGMA synchronous", result -> result.getInt(1)).orElseThrow(); + } + + void closeDatabase() { + database.close(); + } + + /** + * Puts relaxed saves on disk, if any are waiting. They stay marked until a checkpoint has + * copied the whole log, so one that could not is tried again. + * + * @return whether no relaxed save is waiting any more + */ + public boolean syncRelaxed() { + synchronized (database) { + if (unsynced && checkpointedAll()) { + unsynced = false; + } + return !unsynced; + } + } + + /** + * Runs a passive checkpoint and tells whether it copied the whole write-ahead log, which + * SQLite only does after syncing that log. A passive checkpoint does not throw when another + * connection is in its way: it reports busy, or fewer frames copied than the log holds. + */ + boolean checkpointedAll() { + // Columns: busy, frames in the log, frames checkpointed. + return queryOne("PRAGMA wal_checkpoint(PASSIVE)", + result -> result.getInt(1) == 0 && result.getInt(2) == result.getInt(3)).orElse(false); + } + + /** Whether a relaxed save is still waiting for its sync. For tests. */ + boolean relaxedPending() { + synchronized (database) { + return unsynced; + } } public Optional getAnimal(UUID uuid) { @@ -395,20 +528,25 @@ public void upsertSnapshots(Map snapshots, long savedAt) { if (snapshots == null || snapshots.isEmpty()) { return; } - database.runTransaction(connection -> { - try (PreparedStatement statement = connection.prepareStatement(UPSERT_SNAPSHOT)) { - for (Map.Entry snapshot : snapshots.entrySet()) { - if (snapshot.getKey() == null || snapshot.getValue() == null) { - continue; - } - String uuid = snapshot.getKey().toString(); - bindParams(statement, uuid, snapshot.getValue(), savedAt, uuid); - statement.executeUpdate(); + database.runTransaction(connection -> writeSnapshots(connection, snapshots, savedAt)); + } + + private static void writeSnapshots(Connection connection, Map snapshots, long savedAt) { + if (snapshots.isEmpty()) { + return; + } + try (PreparedStatement statement = connection.prepareStatement(UPSERT_SNAPSHOT)) { + for (Map.Entry snapshot : snapshots.entrySet()) { + if (snapshot.getKey() == null || snapshot.getValue() == null) { + continue; } - } catch (SQLException e) { - throw new SqliteDatabaseException("Failed to upsert snapshots", e); + String uuid = snapshot.getKey().toString(); + bindParams(statement, uuid, snapshot.getValue(), savedAt, uuid); + statement.executeUpdate(); } - }); + } catch (SQLException e) { + throw new SqliteDatabaseException("Failed to upsert snapshots", e); + } } public Optional getSnapshot(UUID uuid) { diff --git a/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryTickTask.java b/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryTickTask.java index 91c6fd9..3b7d0c3 100644 --- a/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryTickTask.java +++ b/src/main/java/net/tfminecraft/cooking/husbandry/HusbandryTickTask.java @@ -17,7 +17,11 @@ public final class HusbandryTickTask { private static final long PERIOD_TICKS = 20L * 60L; + /** Ticks in a row that left relaxed saves waiting for their sync before a warning is logged. */ + static final int WAITING_WARN_TICKS = 5; + private static int taskId = -1; + private static int waitingTicks; private HusbandryTickTask() {} @@ -25,6 +29,7 @@ public static void start() { if (taskId != -1 || Cooking.plugin == null) { return; } + waitingTicks = 0; taskId = Bukkit.getScheduler().runTaskTimer( Cooking.plugin, HusbandryTickTask::tick, PERIOD_TICKS, PERIOD_TICKS).getTaskId(); } @@ -35,6 +40,11 @@ public static void stop() { } Bukkit.getScheduler().cancelTask(taskId); taskId = -1; + // A reload starts the minute over: sync what waits now, not up to two minutes from the last tick. + HusbandryRepository repository = HusbandryEntities.repository(); + if (repository != null) { + syncRelaxed(repository); + } } private static void tick() { @@ -70,13 +80,47 @@ private static void tick() { } } if (dirty.isEmpty()) { + // Nothing loaded to save: the load and unload saves since the last tick still get synced. + syncRelaxed(repository); return; } try { - repository.upsertAnimals(dirty); + // One synced commit a minute for the loaded animals and their snapshots. + repository.saveLoaded(dirty, snapshots, now, false); + } catch (SqliteDatabaseException together) { + // A failing snapshot must not cost the animals their save: write the two apart. + Bukkit.getLogger().warning("[Cooking] Husbandry tick save failed as a whole, saving animals and snapshots apart: " + + together.getMessage()); + try { + repository.upsertAnimals(dirty); + } catch (SqliteDatabaseException ex) { + Bukkit.getLogger().severe("[Cooking] Failed to persist husbandry tick: " + ex.getMessage()); + } + HusbandrySnapshots.save(repository, snapshots); + } + // Whatever the save above managed or changed, the relaxed load and unload saves since the + // last tick are put on disk now. A save that rewrites unchanged rows would not do that. + syncRelaxed(repository); + } + + private static void syncRelaxed(HusbandryRepository repository) { + boolean synced; + try { + synced = repository.syncRelaxed(); } catch (SqliteDatabaseException ex) { - Bukkit.getLogger().severe("[Cooking] Failed to persist husbandry tick: " + ex.getMessage()); + Bukkit.getLogger().severe("[Cooking] Failed to sync husbandry saves: " + ex.getMessage()); + return; + } + // A checkpoint that another program holds back fails without an error, minute after minute. + if (synced) { + if (waitingTicks >= WAITING_WARN_TICKS) { + Bukkit.getLogger().info("[Cooking] Husbandry saves are synced to disk again."); + } + waitingTicks = 0; + } else if (++waitingTicks == WAITING_WARN_TICKS) { + Bukkit.getLogger().warning("[Cooking] Husbandry saves have waited " + WAITING_WARN_TICKS + + " minutes for their disk sync, probably because another program is reading husbandry.db." + + " A power loss would roll back the chunk load and unload saves since then."); } - HusbandrySnapshots.save(repository, snapshots); } } diff --git a/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRepositoryTest.java b/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRepositoryTest.java index 0f717f5..260934d 100644 --- a/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRepositoryTest.java +++ b/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRepositoryTest.java @@ -3,21 +3,35 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.io.File; +import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.UUID; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import net.tfminecraft.tlibs.database.SqliteDatabaseException; + class HusbandryRepositoryTest { @Test @@ -298,6 +312,306 @@ void snapshotsRoundTripAndGoWithTheirAnimal(@TempDir Path tempDir) { } } + @Test + void loadedSaveStoresAnimalsAndTheirSnapshotsTogetherWhetherRelaxedOrSynced(@TempDir Path tempDir) { + HusbandryRepository repository = HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile()); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + HusbandryAnimal horse = animal(UUID.randomUUID(), "HORSE", "Drake Maye", 200); + UUID noRow = UUID.randomUUID(); + // The horse's row is new in this very save: its snapshot must still be stored. + repository.saveLoaded(List.of(cow, horse), Map.of(horse.uuid(), new byte[] {7}, noRow, new byte[] {5}), 100L, true); + assertEquals(12, repository.getAnimal(cow.uuid()).orElseThrow().care()); + assertTrue(Arrays.equals(new byte[] {7}, repository.getSnapshot(horse.uuid()).orElseThrow().data())); + assertEquals(100L, repository.getSnapshot(horse.uuid()).orElseThrow().savedAt()); + assertTrue(repository.getSnapshot(noRow).isEmpty()); + assertEquals(2, repository.synchronousLevel(), "a relaxed save must hand the connection back fully synced"); + + cow.setCare(40); + repository.saveLoaded(List.of(cow), Map.of(horse.uuid(), new byte[] {8}), 200L, false); + assertEquals(40, repository.getAnimal(cow.uuid()).orElseThrow().care()); + assertTrue(Arrays.equals(new byte[] {8}, repository.getSnapshot(horse.uuid()).orElseThrow().data())); + assertEquals(2, repository.synchronousLevel()); + + // Nothing to write, null collections and rows without an id are all quiet no-ops. + repository.saveLoaded(null, null, 300L, true); + repository.saveLoaded(List.of(), Map.of(), 300L, false); + repository.saveLoaded(Arrays.asList((HusbandryAnimal) null), null, 300L, true); + repository.saveLoaded(null, Map.of(horse.uuid(), new byte[] {9}), 300L, true); + assertTrue(Arrays.equals(new byte[] {9}, repository.getSnapshot(horse.uuid()).orElseThrow().data())); + } finally { + repository.close(); + } + } + + /** One snapshot in a map that reports the connection's sync level when the save comes to write it. */ + private static Map reportingLevel(HusbandryRepository repository, List levels, UUID animal) { + return new HashMap(Map.of(animal, new byte[] {1})) { + @Override + public Set> entrySet() { + levels.add(repository.synchronousLevel()); + return super.entrySet(); + } + }; + } + + @Test + void aRelaxedSaveCommitsAtNormalAndHandsBackFullWhileASyncedSaveStaysAtFull(@TempDir Path tempDir) { + HusbandryRepository repository = HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile()); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + List levels = new ArrayList<>(); + repository.saveLoaded(List.of(cow), reportingLevel(repository, levels, cow.uuid()), 100L, true); + assertEquals(List.of(1), levels, "the transaction of a relaxed save runs at NORMAL"); + assertEquals(2, repository.synchronousLevel(), "and the connection is back at FULL once it returns"); + assertTrue(repository.relaxedPending()); + assertTrue(repository.getSnapshot(cow.uuid()).isPresent()); + + repository.saveLoaded(List.of(cow), reportingLevel(repository, levels, cow.uuid()), 200L, false); + assertEquals(List.of(1, 2), levels, "a synced save never leaves FULL"); + } finally { + repository.close(); + } + } + + @Test + void anErrorInsideARelaxedSaveStillRestoresSyncedCommits(@TempDir Path tempDir) { + HusbandryRepository repository = HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile()); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + repository.upsertAnimal(cow); + cow.setCare(77); + Map failing = new HashMap(Map.of(cow.uuid(), new byte[] {1})) { + @Override + public Set> entrySet() { + throw new AssertionError("out of memory, say"); + } + }; + AssertionError error = assertThrows(AssertionError.class, + () -> repository.saveLoaded(List.of(cow), failing, 100L, true)); + assertEquals("out of memory, say", error.getMessage()); + assertEquals(2, repository.synchronousLevel(), "an Error must not leave the connection at NORMAL"); + assertEquals(12, repository.getAnimal(cow.uuid()).orElseThrow().care(), "and the save rolled back"); + assertTrue(repository.relaxedPending(), "marked before the commit, whatever became of it"); + } finally { + repository.close(); + } + } + + @Test + void noOtherThreadsWriteRunsBetweenTheLevelChangesAndTheTransaction(@TempDir Path tempDir) throws Exception { + HusbandryRepository repository = spy(HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile())); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + HusbandryAnimal early = animal(UUID.randomUUID(), "PIG", "Ham", 8); + HusbandryAnimal late = animal(UUID.randomUUID(), "SHEEP", "Wool", 4); + // The two moments the database's own lock is free while the level is still NORMAL: + // after the level is lowered and before the transaction begins, and after the commit + // and before the level is raised again. A write that got in there would commit unsynced. + Thread beforeTransaction = new Thread(() -> repository.upsertAnimal(early)); + Thread afterCommit = new Thread(() -> repository.upsertAnimal(late)); + boolean[] waited = new boolean[2]; + doAnswer(call -> { + Object result = call.callRealMethod(); + beforeTransaction.start(); + beforeTransaction.join(500); + waited[0] = beforeTransaction.isAlive() && !repository.exists(early.uuid()); + return result; + }).when(repository).synchronous("NORMAL"); + doAnswer(call -> { + afterCommit.start(); + afterCommit.join(500); + waited[1] = afterCommit.isAlive() && beforeTransaction.isAlive() + && !repository.exists(early.uuid()) && !repository.exists(late.uuid()); + return call.callRealMethod(); + }).when(repository).synchronous("FULL"); + + repository.saveLoaded(List.of(cow), Map.of(), 100L, true); + + assertTrue(waited[0], "a write that arrives before the transaction begins must wait"); + assertTrue(waited[1], "and so must one that arrives after the commit, until the level is FULL again"); + beforeTransaction.join(5_000); + afterCommit.join(5_000); + assertFalse(beforeTransaction.isAlive()); + assertFalse(afterCommit.isAlive()); + assertTrue(repository.exists(early.uuid())); + assertTrue(repository.exists(late.uuid())); + assertEquals(2, repository.synchronousLevel()); + } finally { + repository.close(); + } + } + + @Test + void aFailedRelaxedSaveWritesNothingAndStillRestoresSyncedCommits(@TempDir Path tempDir) throws Exception { + File dbFile = tempDir.resolve("husbandry.db").toFile(); + HusbandryRepository repository = spy(HusbandryRepository.open(dbFile)); + try (Connection db = DriverManager.getConnection("jdbc:sqlite:" + dbFile)) { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + repository.upsertAnimal(cow); + db.createStatement().execute( + "CREATE TRIGGER reject_snapshots BEFORE INSERT ON snapshots BEGIN SELECT RAISE(FAIL,'no snapshots'); END"); + cow.setCare(77); + assertThrows(SqliteDatabaseException.class, + () -> repository.saveLoaded(List.of(cow), Map.of(cow.uuid(), new byte[] {1}), 100L, true)); + assertEquals(12, repository.getAnimal(cow.uuid()).orElseThrow().care(), "the animal rolled back with its snapshot"); + assertEquals(2, repository.synchronousLevel()); + // It is marked all the same, since a save can fail after its commit went through. + assertTrue(repository.relaxedPending()); + assertTrue(repository.syncRelaxed()); + verify(repository, times(1)).checkpointedAll(); + assertFalse(repository.relaxedPending()); + } finally { + repository.close(); + } + } + + @Test + void relaxedSavesStayMarkedUntilACheckpointHasCopiedThem(@TempDir Path tempDir) throws Exception { + HusbandryRepository repository = spy(HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile())); + Path log = tempDir.resolve("husbandry.db-wal"); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + assertTrue(repository.syncRelaxed(), "nothing is waiting"); + verify(repository, never()).checkpointedAll(); + + repository.saveLoaded(List.of(cow), Map.of(), 100L, true); + assertTrue(repository.syncRelaxed()); + assertFalse(repository.relaxedPending()); + repository.syncRelaxed(); + verify(repository, times(1)).checkpointedAll(); + + // A synced save afterwards is no proof. The relaxed save here changes the row, so the + // log now holds a commit that was never synced. The synced save after it writes the + // same row again, which changes no page: SQLite then writes nothing, so there is + // nothing for it to sync, and the relaxed commit is where it was. + cow.setCare(50); + repository.saveLoaded(List.of(cow), Map.of(), 100L, true); + byte[] before = Files.readAllBytes(log); + repository.saveLoaded(List.of(cow), Map.of(), 100L, false); + assertTrue(Arrays.equals(before, Files.readAllBytes(log)), "the same row again leaves the log untouched"); + assertTrue(repository.relaxedPending(), "only a checkpoint clears the mark"); + // Nor is a synced save that does change the row. + cow.setCare(90); + repository.saveLoaded(List.of(cow), Map.of(), 100L, false); + assertFalse(Arrays.equals(before, Files.readAllBytes(log)), "a changed row is written to the log"); + assertTrue(repository.relaxedPending()); + assertTrue(repository.syncRelaxed()); + assertFalse(repository.relaxedPending()); + verify(repository, times(2)).checkpointedAll(); + + // A checkpoint that fails leaves the saves marked, so the next minute tries again. + repository.saveLoaded(List.of(cow), Map.of(), 100L, true); + doThrow(new SqliteDatabaseException("disk")).doCallRealMethod().when(repository).checkpointedAll(); + assertThrows(SqliteDatabaseException.class, repository::syncRelaxed); + assertTrue(repository.relaxedPending()); + repository.syncRelaxed(); + assertFalse(repository.relaxedPending()); + verify(repository, times(4)).checkpointedAll(); + } finally { + repository.close(); + } + } + + @Test + void aCheckpointThatAnotherConnectionHoldsBackKeepsTheSavesMarked(@TempDir Path tempDir) throws Exception { + File dbFile = tempDir.resolve("husbandry.db").toFile(); + HusbandryRepository repository = HusbandryRepository.open(dbFile); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + repository.upsertAnimal(cow); + try (Connection reader = DriverManager.getConnection("jdbc:sqlite:" + dbFile)) { + // A reader that keeps looking at the database as it was before the relaxed save. + reader.setAutoCommit(false); + try (var rows = reader.createStatement().executeQuery("SELECT COUNT(*) FROM animals")) { + assertTrue(rows.next()); + } + cow.setCare(40); + repository.saveLoaded(List.of(cow), Map.of(), 100L, true); + // SQLite reports no error here: the checkpoint just stops at the reader's view. + assertFalse(repository.checkpointedAll()); + repository.saveLoaded(List.of(cow), Map.of(), 101L, true); + assertFalse(repository.syncRelaxed()); + assertTrue(repository.relaxedPending(), "nothing was synced, so the save still waits"); + reader.rollback(); + } + assertTrue(repository.syncRelaxed()); + assertFalse(repository.relaxedPending(), "with the reader gone the next try goes through"); + assertEquals(40, repository.getAnimal(cow.uuid()).orElseThrow().care()); + } finally { + repository.close(); + } + } + + @Test + void aLevelSwitchThatFailsAfterTakingEffectIsUndoneAsWell(@TempDir Path tempDir) { + HusbandryRepository repository = spy(HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile())); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + doAnswer(call -> { + call.callRealMethod(); + throw new SqliteDatabaseException("the statement would not close"); + }).when(repository).synchronous("NORMAL"); + SqliteDatabaseException failure = assertThrows(SqliteDatabaseException.class, + () -> repository.saveLoaded(List.of(cow), Map.of(), 100L, true)); + assertEquals("the statement would not close", failure.getMessage()); + assertEquals(2, repository.synchronousLevel(), "the connection must not stay at NORMAL"); + assertFalse(repository.exists(cow.uuid()), "no save ran"); + assertFalse(repository.relaxedPending()); + } finally { + repository.close(); + } + } + + @Test + void aConnectionThatCannotGoBackToSyncedCommitsIsClosed(@TempDir Path tempDir) { + HusbandryRepository repository = spy(HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile())); + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + doThrow(new SqliteDatabaseException("level stuck")).when(repository).synchronous("FULL"); + SqliteDatabaseException failure = assertThrows(SqliteDatabaseException.class, + () -> repository.saveLoaded(List.of(cow), Map.of(), 100L, true)); + assertEquals("Could not restore synced husbandry commits", failure.getMessage()); + assertEquals("level stuck", failure.getCause().getMessage()); + // Closed rather than left committing without syncs: every later use fails loudly. + assertThrows(SqliteDatabaseException.class, () -> repository.getAnimal(cow.uuid())); + assertThrows(SqliteDatabaseException.class, () -> repository.upsertAnimal(cow)); + } + + @Test + void aFailedSaveKeepsItsOwnFailureWhenTheLevelCannotBeRestoredEither(@TempDir Path tempDir) throws Exception { + File dbFile = tempDir.resolve("husbandry.db").toFile(); + HusbandryRepository repository = spy(HusbandryRepository.open(dbFile)); + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + repository.upsertAnimal(cow); + try (Connection db = DriverManager.getConnection("jdbc:sqlite:" + dbFile)) { + db.createStatement().execute( + "CREATE TRIGGER reject_snapshots BEFORE INSERT ON snapshots BEGIN SELECT RAISE(FAIL,'no snapshots'); END"); + } + doThrow(new SqliteDatabaseException("level stuck")).when(repository).synchronous("FULL"); + SqliteDatabaseException failure = assertThrows(SqliteDatabaseException.class, + () -> repository.saveLoaded(List.of(cow), Map.of(cow.uuid(), new byte[] {1}), 100L, true)); + assertEquals("Failed to upsert snapshots", failure.getMessage()); + assertEquals(1, failure.getSuppressed().length); + assertEquals("level stuck", failure.getSuppressed()[0].getMessage()); + assertThrows(SqliteDatabaseException.class, () -> repository.getAnimal(cow.uuid()), "closed here too"); + } + + @Test + void aCloseThatFailsWhileRestoringTravelsWithTheRestoreFailure(@TempDir Path tempDir) { + HusbandryRepository repository = spy(HusbandryRepository.open(tempDir.resolve("husbandry.db").toFile())); + try { + HusbandryAnimal cow = animal(UUID.randomUUID(), "COW", "Bess", 12); + doThrow(new SqliteDatabaseException("level stuck")).doCallRealMethod().when(repository).synchronous("FULL"); + doThrow(new SqliteDatabaseException("close stuck")).doCallRealMethod().when(repository).closeDatabase(); + SqliteDatabaseException failure = assertThrows(SqliteDatabaseException.class, + () -> repository.saveLoaded(List.of(cow), Map.of(), 100L, true)); + assertEquals("Could not restore synced husbandry commits", failure.getMessage()); + assertEquals("close stuck", failure.getCause().getSuppressed()[0].getMessage()); + } finally { + repository.close(); + } + } + private static HusbandryAnimal animal(UUID uuid, String type, String name, int care) { HusbandryAnimal animal = new HusbandryAnimal(uuid, type, name); animal.setCare(care); diff --git a/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRuntimeCoverageTest.java b/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRuntimeCoverageTest.java index 611026b..537a1bf 100644 --- a/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRuntimeCoverageTest.java +++ b/src/test/java/net/tfminecraft/cooking/husbandry/HusbandryRuntimeCoverageTest.java @@ -27,6 +27,7 @@ import org.mockbukkit.mockbukkit.ServerMock; import org.mockbukkit.mockbukkit.entity.*; import org.mockbukkit.mockbukkit.world.WorldMock; +import org.mockito.InOrder; import org.mockito.MockedStatic; import io.papermc.paper.entity.EntitySerializationFlag; @@ -65,7 +66,7 @@ class HusbandryRuntimeCoverageTest { assertTrue(f.repository.getSnapshot(animal.uuid()).isEmpty()); } - @Test void oneEntitiesEventSavesItsAnimalsAndSnapshotsInOneWriteEach() { + @Test void oneEntitiesEventSavesItsAnimalsAndSnapshotsInOneRelaxedWrite() { Cow first = f.spawn(Cow.class), second = f.spawn(Cow.class); HusbandryAnimal a = f.record(first), b = f.record(second); f.own(a, f.player); f.own(b, f.player); UnsafeValues unsafe = mock(UnsafeValues.class); @@ -76,22 +77,48 @@ class HusbandryRuntimeCoverageTest { bukkit.when(Bukkit::getUnsafe).thenReturn(unsafe); lifecycle.onEntitiesUnload(new EntitiesUnloadEvent(first.getChunk(), List.of(first, second, f.spawn(ItemDisplay.class)))); } - verify(f.repository).upsertAnimals(argThat(animals -> animals.size() == 2)); - verify(f.repository).upsertSnapshots(argThat(copies -> copies.size() == 2), anyLong()); + verify(f.repository).saveLoaded(argThat(animals -> animals.size() == 2), argThat(copies -> copies.size() == 2), anyLong(), eq(true)); + verify(f.repository, never()).upsertAnimals(any()); verify(f.repository, never()).upsertSnapshots(any(), anyLong()); assertArrayEquals(new byte[]{2}, f.repository.getSnapshot(b.uuid()).orElseThrow().data()); + assertNotNull(f.repository.getAnimal(a.uuid()).orElseThrow().unloadedAt()); + assertEquals(2, f.repository.synchronousLevel()); clearInvocations(f.repository); lifecycle.onEntitiesLoad(new EntitiesLoadEvent(first.getChunk(), List.of(first, second))); - verify(f.repository).upsertAnimals(argThat(animals -> animals.size() == 2)); + verify(f.repository).saveLoaded(argThat(animals -> animals.size() == 2), argThat(Map::isEmpty), anyLong(), eq(true)); + verify(f.repository, never()).upsertAnimals(any()); assertNull(f.repository.getAnimal(a.uuid()).orElseThrow().unloadedAt()); assertNull(f.repository.getAnimal(b.uuid()).orElseThrow().unloadedAt()); } + @Test void aFailedCombinedSaveStillWritesAnimalsAndSnapshotsApart() { + Cow first = f.spawn(Cow.class), second = f.spawn(Cow.class); + HusbandryAnimal a = f.record(first), b = f.record(second); f.own(a, f.player); f.own(b, f.player); + UnsafeValues unsafe = mock(UnsafeValues.class); + when(unsafe.serializeEntity(first, EntitySerializationFlag.FORCE)).thenReturn(new byte[]{1}); + when(unsafe.serializeEntity(second, EntitySerializationFlag.FORCE)).thenReturn(new byte[]{2}); + doThrow(new SqliteDatabaseException("together")).when(f.repository).saveLoaded(any(), any(), anyLong(), anyBoolean()); + clearInvocations(f.repository); + try (MockedStatic bukkit = mockStatic(Bukkit.class, CALLS_REAL_METHODS)) { + bukkit.when(Bukkit::getUnsafe).thenReturn(unsafe); + lifecycle.onEntitiesUnload(new EntitiesUnloadEvent(first.getChunk(), List.of(first, second))); + } + // The old two synced writes take over, and both go through. + verify(f.repository).upsertAnimals(argThat(animals -> animals.size() == 2)); + verify(f.repository).upsertSnapshots(argThat(copies -> copies.size() == 2), anyLong()); + verify(f.repository, never()).upsertAnimal(any()); + assertNotNull(f.repository.getAnimal(a.uuid()).orElseThrow().unloadedAt()); + assertNotNull(f.repository.getAnimal(b.uuid()).orElseThrow().unloadedAt()); + assertArrayEquals(new byte[]{1}, f.repository.getSnapshot(a.uuid()).orElseThrow().data()); + assertArrayEquals(new byte[]{2}, f.repository.getSnapshot(b.uuid()).orElseThrow().data()); + } + @Test void failedEventBatchFallsBackToOneWritePerAnimalAndSnapshot() { Cow first = f.spawn(Cow.class), second = f.spawn(Cow.class); HusbandryAnimal a = f.record(first), b = f.record(second); f.own(a, f.player); f.own(b, f.player); UnsafeValues unsafe = mock(UnsafeValues.class); when(unsafe.serializeEntity(first, EntitySerializationFlag.FORCE)).thenReturn(new byte[]{1}); when(unsafe.serializeEntity(second, EntitySerializationFlag.FORCE)).thenReturn(new byte[]{2}); + doThrow(new SqliteDatabaseException("together")).when(f.repository).saveLoaded(any(), any(), anyLong(), anyBoolean()); doThrow(new SqliteDatabaseException("batch")).when(f.repository).upsertAnimals(argThat(animals -> animals != null && animals.size() > 1)); doThrow(new SqliteDatabaseException("bad row")).when(f.repository).upsertAnimals( argThat(animals -> animals != null && animals.size() == 1 && animals.iterator().next().uuid().equals(a.uuid()))); @@ -286,18 +313,67 @@ class HusbandryRuntimeCoverageTest { ItemDisplay display = f.spawn(ItemDisplay.class); HusbandryAnimal nonLiving = new HusbandryAnimal(display.getUniqueId(), "ITEM_DISPLAY", "Display"); HusbandryEntities.putLoaded(nonLiving); UnsafeValues unsafe = mock(UnsafeValues.class); when(unsafe.serializeEntity(horse)).thenReturn(new byte[]{3}); when(unsafe.serializeEntity(refused)).thenThrow(new IllegalArgumentException("unavailable")); try (MockedStatic bukkit = mockStatic(Bukkit.class, CALLS_REAL_METHODS)) { - bukkit.when(Bukkit::getUnsafe).thenReturn(unsafe); HusbandryTickTask.start(); HusbandryTickTask.start(); f.server.getScheduler().performTicks(1200); + bukkit.when(Bukkit::getUnsafe).thenReturn(unsafe); clearInvocations(f.repository); HusbandryTickTask.start(); HusbandryTickTask.start(); f.server.getScheduler().performTicks(1200); assertEquals(1, f.repository.getAnimal(animal.uuid()).orElseThrow().care()); assertEquals(unchanged, notVisited.lastProcessedAt()); assertArrayEquals(new byte[]{3}, f.repository.getSnapshot(animal.uuid()).orElseThrow().data()); assertTrue(f.repository.getSnapshot(cannotCapture.uuid()).isEmpty()); assertTrue(HusbandryEntities.getLoaded(gone.uuid()).isEmpty()); assertTrue(HusbandryEntities.getLoaded(nonLiving.uuid()).isEmpty()); + // The minute's save is one synced commit, and the tick then puts the relaxed load and unload saves on disk. + verify(f.repository).saveLoaded(argThat(animals -> animals.size() == 3), argThat(copies -> copies.size() == 1), anyLong(), eq(false)); + verify(f.repository, never()).upsertAnimals(any()); verify(f.repository).syncRelaxed(); + InOrder minute = inOrder(f.repository); minute.verify(f.repository).saveLoaded(any(), any(), anyLong(), eq(false)); minute.verify(f.repository).syncRelaxed(); + // A save that fails as a whole falls back to the animals and the snapshots apart. + doThrow(new SqliteDatabaseException("snapshot row")).doCallRealMethod().when(f.repository).saveLoaded(any(), any(), anyLong(), anyBoolean()); + animal.setCare(55); f.server.getScheduler().performTicks(1200); + verify(f.repository).upsertAnimals(argThat(animals -> animals.size() == 3)); verify(f.repository).upsertSnapshots(argThat(copies -> copies.size() == 1), anyLong()); + // The fallback ends the same way, so a save that keeps failing cannot hold that sync back. + verify(f.repository, times(2)).syncRelaxed(); + assertTrue(f.repository.getAnimal(animal.uuid()).orElseThrow().care() >= 55); + doThrow(new SqliteDatabaseException("together")).when(f.repository).saveLoaded(any(), any(), anyLong(), anyBoolean()); doThrow(new SqliteDatabaseException("tick write failure")).when(f.repository).upsertAnimals(anyList()); doThrow(new SqliteDatabaseException("snapshot write failure")).when(f.repository).upsertSnapshots(anyMap(), anyLong()); assertDoesNotThrow(() -> f.server.getScheduler().performTicks(1200)); when(f.plugin.getHusbandryRepository()).thenReturn(null); f.server.getScheduler().performTicks(1200); when(f.plugin.getHusbandryRepository()).thenReturn(f.repository); - HusbandryEntities.clearLoaded(); f.server.getScheduler().performTicks(1200); HusbandryTickTask.stop(); HusbandryTickTask.stop(); + // With nothing loaded there is no save to ride on: the tick syncs what load and unload left, and survives a failure. + clearInvocations(f.repository); + HusbandryEntities.clearLoaded(); f.server.getScheduler().performTicks(1200); verify(f.repository).syncRelaxed(); + doThrow(new SqliteDatabaseException("checkpoint")).when(f.repository).syncRelaxed(); assertDoesNotThrow(() -> f.server.getScheduler().performTicks(1200)); + HusbandryTickTask.stop(); HusbandryTickTask.stop(); Cooking.plugin = null; HusbandryTickTask.start(); Cooking.plugin = f.plugin; } } + @Test void theTickWarnsOnceWhenSavesKeepWaitingForTheirSyncAndSaysWhenTheyAreBack() { + List lines = new ArrayList<>(); + java.util.logging.Handler handler = new java.util.logging.Handler() { + @Override public void publish(java.util.logging.LogRecord record) { lines.add(record.getLevel() + " " + record.getMessage()); } + @Override public void flush() {} + @Override public void close() {} + }; + Bukkit.getLogger().addHandler(handler); + try { + // Nothing is loaded, so each tick only tries the sync; another program holds the checkpoint back. + doReturn(false).when(f.repository).syncRelaxed(); + HusbandryTickTask.start(); + f.server.getScheduler().performTicks(1200L * (HusbandryTickTask.WAITING_WARN_TICKS - 1)); + assertTrue(lines.stream().noneMatch(line -> line.contains("waited"))); + f.server.getScheduler().performTicks(1200L * 3); + assertEquals(1, lines.stream().filter(line -> line.startsWith("WARNING") && line.contains("waited 5 minutes")).count()); + doReturn(true).when(f.repository).syncRelaxed(); + f.server.getScheduler().performTicks(1200L * 2); + assertEquals(1, lines.stream().filter(line -> line.startsWith("INFO") && line.contains("synced to disk again")).count()); + // A short wait that clears by itself says nothing. + doReturn(false).when(f.repository).syncRelaxed(); f.server.getScheduler().performTicks(1200L * 2); + doReturn(true).when(f.repository).syncRelaxed(); f.server.getScheduler().performTicks(1200L); + assertEquals(1, lines.stream().filter(line -> line.contains("waited")).count()); + assertEquals(1, lines.stream().filter(line -> line.contains("synced to disk again")).count()); + // Stopping syncs once more, and copes with a closed repository. + clearInvocations(f.repository); HusbandryTickTask.stop(); verify(f.repository).syncRelaxed(); + HusbandryTickTask.start(); when(f.plugin.getHusbandryRepository()).thenReturn(null); clearInvocations(f.repository); + HusbandryTickTask.stop(); verify(f.repository, never()).syncRelaxed(); + } finally { + Bukkit.getLogger().removeHandler(handler); when(f.plugin.getHusbandryRepository()).thenReturn(f.repository); HusbandryTickTask.stop(); + } + } + @Test void locationMemoryFloorsCoordinatesAndIgnoresUnavailableEntities() { Cow cow = f.spawn(Cow.class); HusbandryAnimal animal = f.record(cow); cow.teleport(new Location(f.world, -1.2, 63.9, 2.7)); HusbandryLocation.remember(animal, cow); assertEquals("farm", animal.world()); assertEquals(-2, animal.x()); assertEquals(63, animal.y()); assertEquals(2, animal.z());