Improve rollback diagnostics and abort failed database initialization

This commit is contained in:
Intelli committed 2026-09-23 20:12:06 -06:00
1 parent bbe5917ec5
commit 4e5ddd2334
6 files changed
+89 -24

No files matched your search

@@ -524,7 +524,7 @@ public class ConfigHandler extends Queue {
Class.forName(ConfigHandler.databaseType.isDuckDB() ? "org.duckdb.DuckDBDriver" : "org.sqlite.JDBC");
} catch (Exception e) {
ErrorReporter.report(e);
throw new IllegalStateException("Failed to initialize " + ConfigHandler.databaseType.getDisplayName(), e);
}
} else {
HikariConfig config = new HikariConfig();
@@ -25,7 +25,6 @@ import net.coreprotect.consumer.Queue;
import net.coreprotect.consumer.process.Process;
import net.coreprotect.database.rollback.Rollback;
import net.coreprotect.database.rollback.RollbackComplete;
import net.coreprotect.language.Phrase;
import net.coreprotect.listener.player.InventoryChangeListener;
import net.coreprotect.model.BlockGroup;
import net.coreprotect.thread.Scheduler;
@@ -43,14 +42,14 @@ public class ContainerRollback extends Rollback {
final List<Object[]> lookupList = Lookup.performLookupRaw(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, location, radius, null, startTime, endTime, -1, -1, restrictWorld, lookup);
if (lookupList == null) {
sendAborted(user);
sendAborted(user, "Unable to load container rollback records from the database.");
return false;
}
if (rollbackType == 1) {
Collections.reverse(lookupList);
}
if (Consumer.isPersistenceHalted()) {
sendAborted(user);
sendAborted(user, "Database persistence halted before container rollback updates could be queued.");
return false;
}
@@ -61,7 +60,7 @@ public class ContainerRollback extends Rollback {
Queue.queueRollbackUpdate(userString, location, lookupList, Process.CONTAINER_ROLLBACK_UPDATE, rollbackType); // Perform update transaction in consumer
if (Consumer.isPersistenceHalted()) {
sendAborted(user);
sendAborted(user, "Database persistence halted while queuing container rollback updates.");
return false;
}
@@ -74,6 +73,7 @@ public class ContainerRollback extends Rollback {
try {
int[] rollbackHashData = ConfigHandler.rollbackHash.get(finalUserString);
if (Consumer.isPersistenceHalted()) {
Chat.console("Container rollback/restore for " + finalUserString + " aborted: database persistence halted before the container could be modified.");
ConfigHandler.rollbackHash.put(finalUserString, new int[] { rollbackHashData[0], rollbackHashData[1], rollbackHashData[2], 2, rollbackHashData[4] });
return;
}
@@ -189,7 +189,7 @@ public class ContainerRollback extends Rollback {
rollbackHashData = ConfigHandler.rollbackHash.get(finalUserString);
next = rollbackHashData[3];
if (sleepTime > 300000) {
Chat.console(Phrase.build(Phrase.ROLLBACK_ABORTED));
Chat.console("Container rollback/restore for " + finalUserString + " aborted: timed out after 300 seconds waiting for the container task.");
break;
}
}
@@ -12,6 +12,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -38,6 +39,7 @@ import net.coreprotect.model.entity.EntitySpawnData;
import net.coreprotect.model.entity.EntitySpawnRecord;
import net.coreprotect.paper.PaperAdapter;
import net.coreprotect.thread.Scheduler;
import net.coreprotect.utility.Chat;
import net.coreprotect.utility.EntitySpawnTracking;
import net.coreprotect.utility.EntityUtils;
import net.coreprotect.utility.ErrorReporter;
@@ -243,8 +245,8 @@ public final class EntitySpawnRollbackHandler {
Rollback.warnSkippedEntityRows("Skipping tracked entity spawn rows with missing kill data", skippedCompositeRows);
}
Set<Integer> claimedRows = preview == 0 ? claimTrackingRows(requiredRows) : ConcurrentHashMap.newKeySet();
Context context = new Context(userString, preview, rollbackType, inventoryRollback, claimedRows, radiusOrigin, radius);
Context context = new Context(userString, preview, rollbackType, inventoryRollback, null, radiusOrigin, radius);
Set<Integer> claimedRows = preview == 0 ? claimTrackingRows(requiredRows, context) : Collections.emptySet();
if (claimedRows == null) {
context.cancel();
return context;
@@ -602,6 +604,7 @@ public final class EntitySpawnRollbackHandler {
if (work.operation == WorkOperation.SPAWN_ROLLBACK) {
byte[] serializedState = EntityStatement.serializeData(EntitySpawnTracking.serializeState(entity));
if (serializedState == null) {
context.reportFailure("Entity state could not be serialized", work);
context.cancel();
completion.complete(false);
return;
@@ -698,6 +701,8 @@ public final class EntitySpawnRollbackHandler {
spawnFuture.whenComplete((entity, throwable) -> {
try {
if (throwable != null) {
context.reportFailure(throwable instanceof TimeoutException
? "Entity restoration timed out after 30 seconds" : "Entity restoration failed", work);
context.cancel();
ErrorReporter.report(throwable);
completion.complete(false);
@@ -705,6 +710,7 @@ public final class EntitySpawnRollbackHandler {
return;
}
if (entity == null) {
context.reportFailure("Entity restoration did not return an entity", work);
completion.complete(false);
context.endMutation();
return;
@@ -757,7 +763,7 @@ public final class EntitySpawnRollbackHandler {
CompletableFuture<Boolean> completion = new CompletableFuture<>();
boolean retained = false;
try {
if (!applyPreparedTransactions(work, entity) || (context.isCancelled() && !work.transactionContentsApplied)) {
if (!applyPreparedTransactions(context, work, entity) || (context.isCancelled() && !work.transactionContentsApplied)) {
context.cancel();
completion.complete(false);
return completion;
@@ -767,6 +773,7 @@ public final class EntitySpawnRollbackHandler {
if (work.operation == WorkOperation.COMPOSITE_ROLLBACK) {
byte[] serializedState = EntityStatement.serializeData(EntitySpawnTracking.serializeState(entity));
if (serializedState == null) {
context.reportFailure("Entity state could not be serialized", work);
context.cancel();
completion.complete(false);
return completion;
@@ -884,6 +891,9 @@ public final class EntitySpawnRollbackHandler {
private static void startTransaction(Context context, Work work, Entity entity, CompletableFuture<Boolean> completion) {
try {
if (context.isCancelled() || !entity.isValid() || !(entity instanceof InventoryHolder)) {
if (!context.isCancelled()) {
context.reportFailure("Entity is no longer valid or has no inventory", work);
}
completion.complete(false);
return;
}
@@ -902,6 +912,9 @@ public final class EntitySpawnRollbackHandler {
private static void startCreatedTransaction(Context context, Work work, CompletableFuture<Boolean> completion) {
try {
if (context.isCancelled() || !work.initializeCreatedTransactionContents()) {
if (!context.isCancelled()) {
context.reportFailure("Recorded entity inventory state is missing or invalid", work);
}
completion.complete(false);
return;
}
@@ -914,17 +927,19 @@ public final class EntitySpawnRollbackHandler {
}
}
private static boolean applyPreparedTransactions(Work work, Entity entity) {
private static boolean applyPreparedTransactions(Context context, Work work, Entity entity) {
if (work.transactions.isEmpty()) {
return true;
}
if (!(entity instanceof InventoryHolder) || work.originalTransactionContents == null || work.transactionContents == null) {
context.reportFailure("Restored entity has no inventory or prepared inventory state is missing", work);
return false;
}
Inventory inventory = ((InventoryHolder) entity).getInventory();
ItemStack[] currentContents = inventory.getStorageContents();
if (currentContents.length != work.originalTransactionContents.length || inventory.getMaxStackSize() != work.inventoryMaxStackSize || !ItemUtils.compareContainers(work.originalTransactionContents, currentContents)) {
context.reportFailure("Restored entity inventory does not match its recorded contents, size, or stack limit", work);
return false;
}
inventory.setStorageContents(ItemUtils.getContainerState(work.transactionContents));
@@ -937,6 +952,9 @@ public final class EntitySpawnRollbackHandler {
return;
}
if (context.isCancelled() || !entity.isValid()) {
if (!context.isCancelled()) {
context.reportFailure("Entity became invalid during inventory replay", work);
}
completion.complete(false);
return;
}
@@ -952,6 +970,9 @@ public final class EntitySpawnRollbackHandler {
return;
}
if (context.isCancelled() || !entity.isValid() || !ItemUtils.compareContainers(work.originalTransactionContents, inventory.getStorageContents())) {
if (!context.isCancelled()) {
context.reportFailure("Entity became invalid or its inventory changed during replay", work);
}
completion.complete(false);
return;
}
@@ -1009,6 +1030,7 @@ public final class EntitySpawnRollbackHandler {
while (work.transactionIndex < work.transactions.size()) {
Object[] row = work.transactions.get(work.transactionIndex);
if (!applyTransaction(context.rollbackType, work.transactionContents, work.inventoryMaxStackSize, row)) {
context.reportFailure("Invalid entity inventory transaction (rowid: " + row[0] + ", material: " + row[6] + ", amount: " + row[11] + ")", work);
return TransactionStep.FAILED;
}
work.transactionIndex++;
@@ -1130,6 +1152,7 @@ public final class EntitySpawnRollbackHandler {
}
Runnable start = () -> startTransaction(context, work, entity, completion);
Runnable retired = () -> {
context.reportFailure("Entity inventory task was retired or could not be scheduled", work);
context.cancel();
completion.complete(false);
};
@@ -1158,6 +1181,7 @@ public final class EntitySpawnRollbackHandler {
}
Location location = work.getLocation();
if (location == null || location.getWorld() == null) {
context.reportFailure("Entity inventory replay location is unavailable", work);
completion.complete(false);
return;
}
@@ -1174,6 +1198,7 @@ public final class EntitySpawnRollbackHandler {
private static void scheduleTransactionContinuation(Context context, Work work, Entity entity, Inventory inventory, CompletableFuture<Boolean> completion) {
Runnable continuation = () -> applyTransactionStep(context, work, entity, inventory, completion);
Runnable retired = () -> {
context.reportFailure("Entity inventory task was retired or could not be scheduled", work);
context.cancel();
completion.complete(false);
};
@@ -1198,6 +1223,7 @@ public final class EntitySpawnRollbackHandler {
private static void scheduleCreatedTransactionContinuation(Context context, Work work, CompletableFuture<Boolean> completion) {
Location location = work.getLocation();
if (location == null || location.getWorld() == null) {
context.reportFailure("Entity inventory replay location is unavailable", work);
completion.complete(false);
return;
}
@@ -1249,13 +1275,15 @@ public final class EntitySpawnRollbackHandler {
});
}
private static Set<Integer> claimTrackingRows(Set<Integer> requiredRows) {
Set<Integer> claimedRows = ConcurrentHashMap.newKeySet();
private static Set<Integer> claimTrackingRows(Set<Integer> requiredRows, Context context) {
Set<Integer> claimedRows = context.claimedRows;
for (Integer trackingRowId : requiredRows) {
if (!activeTrackingRows.add(trackingRowId)) {
for (Integer claimedRowId : claimedRows) {
releaseTrackingRow(claimedRowId);
}
claimedRows.clear();
context.reportFailure("Entity tracking row " + trackingRowId + " is already claimed by another rollback or restore", null);
return null;
}
claimedRows.add(trackingRowId);
@@ -1381,6 +1409,7 @@ public final class EntitySpawnRollbackHandler {
private final AtomicInteger items = new AtomicInteger();
private final AtomicInteger entities = new AtomicInteger();
private final AtomicBoolean cancelled = new AtomicBoolean();
private final AtomicBoolean failureReported = new AtomicBoolean();
private int activeMutations;
private boolean transactionWorkActive;
private boolean transactionWorkAdvancing;
@@ -1455,7 +1484,30 @@ public final class EntitySpawnRollbackHandler {
boolean isCancelled() {
int[] rollbackData = ConfigHandler.rollbackHash.get(userString);
return cancelled.get() || (preview == 0 && Consumer.isPersistenceHalted()) || (rollbackData != null && rollbackData[3] == 2);
if (!cancelled.get() && preview == 0 && Consumer.isPersistenceHalted()) {
reportFailure("Database persistence is halted", null);
return true;
}
return cancelled.get() || (rollbackData != null && rollbackData[3] == 2);
}
void reportFailure(String reason, Work value) {
if (!failureReported.compareAndSet(false, true)) {
return;
}
String details = "user: " + userString + ", operation: " + (rollbackType == 0 ? "rollback" : "restore");
if (value != null) {
details += ", work: " + value.operation + ", block rowid: " + value.blockRowId;
if (value.record != null) {
details += ", tracking rowid: " + value.record.getRowId() + ", entity: " + value.record.getUuid();
}
Location location = value.getLocation();
if (location != null) {
details += ", location: " + (location.getWorld() == null ? "unknown" : location.getWorld().getName())
+ " " + location.getBlockX() + " " + location.getBlockY() + " " + location.getBlockZ();
}
}
Chat.console("Rollback or restore failed: " + reason + " (" + details + ").");
}
void cancel() {
@@ -106,7 +106,7 @@ public class Rollback extends RollbackUtil {
}
if (lookupList == null) {
sendAborted(user);
sendAborted(user, "Unable to load rollback records from the database.");
return null;
}
@@ -188,7 +188,7 @@ public class Rollback extends RollbackUtil {
}
itemList = Lookup.performLookupRaw(statement, user, checkUuids, checkUsers, itemRestrictList, itemExcludeList, excludeUserList, itemActionList, EntityActionFilter.DEFAULT, entityContext, location, radius, null, startTime, endTime, -1, -1, restrictWorld, lookup, null);
if (itemList == null) {
sendAborted(user);
sendAborted(user, "Unable to load container rollback records from the database.");
return null;
}
@@ -323,6 +323,7 @@ public class Rollback extends RollbackUtil {
// Perform update transaction(s) in consumer
if (preview == 0) {
if (Consumer.isPersistenceHalted()) {
entitySpawnContext.reportFailure("Database persistence halted before rollback updates could be queued.", null);
entitySpawnContext.cancel();
sendAborted(user);
return null;
@@ -468,6 +469,11 @@ public class Rollback extends RollbackUtil {
}
}
protected static void sendAborted(CommandSender user, String reason) {
Chat.console("Rollback/restore for " + (user == null ? "#server" : user.getName()) + " aborted: " + reason);
sendAborted(user);
}
private static List<Object[]> routeEntityContainerInventoryRows(List<Object[]> rows, Map<Integer, EntitySpawnRecord> records, Location commandLocation) {
World fallbackWorld = commandLocation == null ? null : commandLocation.getWorld();
List<World> loadedWorlds = Bukkit.getWorlds();
@@ -542,7 +548,7 @@ public class Rollback extends RollbackUtil {
}
CompletableFuture<Boolean> batchFuture = scheduleFoliaChunkBatchTask(batchState, rollbackType, preview, userString, user, inventoryRollback, verbose, actionList, blockDataCache, entitySpawnContext);
if (!awaitChunkTasks(Collections.singletonList(batchFuture), preview) || !awaitChunkTasks(entitySpawnContext.drainPending(), preview)) {
if (!awaitChunkTasks(Collections.singletonList(batchFuture), preview, entitySpawnContext) || !awaitChunkTasks(entitySpawnContext.drainPending(), preview, entitySpawnContext)) {
Chat.console(Phrase.build(Phrase.ROLLBACK_ABORTED));
entitySpawnContext.cancel();
break;
@@ -650,7 +656,7 @@ public class Rollback extends RollbackUtil {
CompletableFuture<Boolean> completion = new CompletableFuture<>();
scheduleChunkBatchTask(batchState, worldMap, dataList, itemDataList, rollbackType, preview, userString, user, inventoryRollback, verbose, actionList, blockDataCache, entitySpawnContext, completion, 0);
if (!awaitRollbackCompletion(completion, batchState, preview) || !awaitChunkTasks(entitySpawnContext.drainPending(), preview)) {
if (!awaitRollbackCompletion(completion, batchState, preview, entitySpawnContext) || !awaitChunkTasks(entitySpawnContext.drainPending(), preview, entitySpawnContext)) {
Chat.console(Phrase.build(Phrase.ROLLBACK_ABORTED));
entitySpawnContext.cancel();
}
@@ -709,7 +715,7 @@ public class Rollback extends RollbackUtil {
}, delay);
}
private static boolean awaitRollbackCompletion(CompletableFuture<Boolean> completion, RollbackBatchState batchState, int preview) throws InterruptedException {
private static boolean awaitRollbackCompletion(CompletableFuture<Boolean> completion, RollbackBatchState batchState, int preview, EntitySpawnRollbackHandler.Context context) throws InterruptedException {
int delay = preview == 1 ? 1 : 5;
int lastChunkCount = -1;
long stalledTime = 0;
@@ -723,6 +729,7 @@ public class Rollback extends RollbackUtil {
stalledTime += delay;
if (stalledTime > 300000) {
context.reportFailure("Rollback chunk processing made no progress for 300 seconds (chunk " + chunkCount + " of " + batchState.totalChunks() + ").", null);
completion.complete(false);
return false;
}
@@ -733,6 +740,8 @@ public class Rollback extends RollbackUtil {
return Boolean.TRUE.equals(completion.getNow(Boolean.FALSE));
}
catch (Exception e) {
context.reportFailure("A scheduled rollback chunk task completed exceptionally.", null);
ErrorReporter.report(e);
return false;
}
}
@@ -879,6 +888,7 @@ public class Rollback extends RollbackUtil {
private static boolean processChunkWorld(int chunkX, int chunkZ, long chunkKey, int worldId, HashMap<Long, ArrayList<Object[]>> blockList, HashMap<Long, ArrayList<Object[]>> itemList, int rollbackType, int preview, String userString, CommandSender user, World world, boolean inventoryRollback, RollbackBlockDataCache blockDataCache, EntitySpawnRollbackHandler.Context entitySpawnContext) {
if (preview == 0 && Consumer.isPersistenceHalted()) {
entitySpawnContext.reportFailure("Database persistence halted during rollback chunk processing.", null);
return false;
}
ArrayList<Object[]> blockData = blockList != null ? blockList.getOrDefault(chunkKey, new ArrayList<>()) : new ArrayList<>();
@@ -968,7 +978,7 @@ public class Rollback extends RollbackUtil {
}
}
private static boolean awaitChunkTasks(List<CompletableFuture<Boolean>> futures, int preview) throws InterruptedException {
private static boolean awaitChunkTasks(List<CompletableFuture<Boolean>> futures, int preview, EntitySpawnRollbackHandler.Context context) throws InterruptedException {
if (futures.isEmpty()) {
return true;
}
@@ -990,6 +1000,7 @@ public class Rollback extends RollbackUtil {
int delay = preview == 1 ? 1 : 5;
sleepTime += delay;
if (sleepTime > 300000) {
context.reportFailure("Timed out after 300 seconds waiting for scheduled rollback tasks.", null);
return false;
}
Thread.sleep(delay);
@@ -1001,6 +1012,8 @@ public class Rollback extends RollbackUtil {
result = future.getNow(Boolean.FALSE);
}
catch (Exception e) {
context.reportFailure("A scheduled rollback task completed exceptionally.", null);
ErrorReporter.report(e);
return false;
}
@@ -68,7 +68,7 @@ public class EntityStatement {
return serializeDataStrict(data, kind, databaseType);
}
catch (Exception e) {
ErrorReporter.report(e, ConfigHandler.EDITION_BRANCH.contains("-dev"));
ErrorReporter.report(e);
return null;
}
}
@@ -142,8 +142,8 @@ public class EntityStatement {
resultSet.close();
}
catch (Exception e) { // only print exception on development branch
ErrorReporter.report(e, ConfigHandler.EDITION_BRANCH.contains("-dev"));
catch (Exception e) {
ErrorReporter.report(e);
}
return result;
@@ -195,7 +195,7 @@ public class EntityStatement {
return deserializeDataStrict(data, kind);
}
catch (Exception e) {
ErrorReporter.report(e, ConfigHandler.EDITION_BRANCH.contains("-dev"));
ErrorReporter.report(e);
return result;
}
}
@@ -110,7 +110,7 @@ public class EntityUtil {
return completion;
}
if (!legacyTransition) {
completion.completeOnTimeout(null, ENTITY_RESTORE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
completion.orTimeout(ENTITY_RESTORE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
Location restoreLocation = EntitySpawnTracking.isPlacedEntityType(type) ? EntitySpawnTracking.getKillRestoreLocation(blockLocation.getWorld(), list) : null;