From 56ba90b58bc76e03e75c2f2607f3d0b40b8bc7c8 Mon Sep 17 00:00:00 2001 From: Intelli Date: Wed, 23 Sep 2026 13:59:32 -0600 Subject: [PATCH] Align ClickHouse lookup pagination with DuckDB --- .../command/lookup/StandardLookupThread.java | 17 ++--- .../net/coreprotect/config/ConfigHandler.java | 2 +- .../java/net/coreprotect/database/Lookup.java | 6 +- .../net/coreprotect/database/LookupRaw.java | 65 ++++++++++++++----- 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/main/java/net/coreprotect/command/lookup/StandardLookupThread.java b/src/main/java/net/coreprotect/command/lookup/StandardLookupThread.java index 9eaf1e85..0df5a9d8 100644 --- a/src/main/java/net/coreprotect/command/lookup/StandardLookupThread.java +++ b/src/main/java/net/coreprotect/command/lookup/StandardLookupThread.java @@ -161,8 +161,8 @@ public class StandardLookupThread implements Runnable { ConfigHandler.lookupRadius.put(player.getName(), radius); ConfigHandler.lookupOutputMode.put(player.getName(), outputMode == LookupOutputMode.COUNT ? LookupOutputMode.DETAIL : outputMode); ConfigHandler.lookupRollbackState.put(player.getName(), rollbackState); - if (typeLookup != 5 || outputMode != LookupOutputMode.DETAIL || !ConfigHandler.databaseType.isDuckDB()) { - ConfigHandler.lookupDuckDBCursor.remove(player.getName()); + if (typeLookup != 5 || outputMode != LookupOutputMode.DETAIL || !ConfigHandler.databaseType.isColumnar()) { + ConfigHandler.lookupCursor.remove(player.getName()); } if (connection != null) { @@ -290,20 +290,17 @@ public class StandardLookupThread implements Runnable { rowData[4] = rows; ConfigHandler.lookupRows.put(player.getName(), rowData); } - if (lookupPage == null && outputMode == LookupOutputMode.DETAIL && ConfigHandler.databaseType.isDuckDB() && pageStart < rows) { - LookupCursor cursor = ConfigHandler.lookupDuckDBCursor.get(player.getName()); - if (cursor == null || cursor.getNextPage() != page || cursor.getPageSize() != displayResults) { - cursor = null; - } - lookupPage = Lookup.performDuckDBLookupPage(statement, player, uuidList, userList, blockList, excludedBlocks, excludedUsers, actions, entityActionFilter, messageFilters, entityContext, finalLocation, radius, rowData, timeStart, timeEnd, (int) pageStart, displayResults, rows, cursor, restrict_world, true, entityContainerId, rollbackState); + if (lookupPage == null && outputMode == LookupOutputMode.DETAIL && ConfigHandler.databaseType.isColumnar() && pageStart < rows) { + LookupCursor cursor = ConfigHandler.lookupCursor.get(player.getName()); + lookupPage = Lookup.performLookupPage(statement, player, uuidList, userList, blockList, excludedBlocks, excludedUsers, actions, entityActionFilter, messageFilters, entityContext, finalLocation, radius, rowData, timeStart, timeEnd, (int) pageStart, displayResults, rows, cursor, restrict_world, true, entityContainerId, rollbackState); } if (lookupPage != null) { LookupCursor nextCursor = lookupPage.getNextCursor(); if (nextCursor == null) { - ConfigHandler.lookupDuckDBCursor.remove(player.getName()); + ConfigHandler.lookupCursor.remove(player.getName()); } else { - ConfigHandler.lookupDuckDBCursor.put(player.getName(), nextCursor); + ConfigHandler.lookupCursor.put(player.getName(), nextCursor); } } if (outputMode == LookupOutputMode.COUNT) { diff --git a/src/main/java/net/coreprotect/config/ConfigHandler.java b/src/main/java/net/coreprotect/config/ConfigHandler.java index b0a3d3bf..091fb95e 100644 --- a/src/main/java/net/coreprotect/config/ConfigHandler.java +++ b/src/main/java/net/coreprotect/config/ConfigHandler.java @@ -272,7 +272,7 @@ public class ConfigHandler extends Queue { public static Map lookupRadius = syncMap(); public static Map lookupTime = syncMap(); public static Map lookupRows = syncMap(); - public static Map lookupDuckDBCursor = syncMap(); + public static Map lookupCursor = syncMap(); public static Map uuidCache = syncMap(); public static Map uuidCacheReversed = syncMap(); public static Map playerIdCache = syncMap(); diff --git a/src/main/java/net/coreprotect/database/Lookup.java b/src/main/java/net/coreprotect/database/Lookup.java index 5b84dec6..54601bf7 100755 --- a/src/main/java/net/coreprotect/database/Lookup.java +++ b/src/main/java/net/coreprotect/database/Lookup.java @@ -301,11 +301,11 @@ public class Lookup extends Queue { } public static LookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitCount, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) { - return performDuckDBLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState); + return performLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState); } - public static LookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) { - LookupRaw.RawLookupPage page = LookupRaw.performDuckDBLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, knownTotalRows, cursor, restrictWorld, lookup, entityContainerId, rollbackState); + public static LookupPage performLookupPage(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) { + LookupRaw.RawLookupPage page = LookupRaw.performLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, knownTotalRows, cursor, restrictWorld, lookup, entityContainerId, rollbackState); List rows = LookupConverter.convertRawLookup(statement, page.getRows()); return new LookupPage(page.getTotalRows(), rows, page.getNextCursor()); } diff --git a/src/main/java/net/coreprotect/database/LookupRaw.java b/src/main/java/net/coreprotect/database/LookupRaw.java index d2ffc91b..be2b0d0f 100644 --- a/src/main/java/net/coreprotect/database/LookupRaw.java +++ b/src/main/java/net/coreprotect/database/LookupRaw.java @@ -75,12 +75,23 @@ public class LookupRaw extends Queue { } static RawLookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitCount, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) { - return performDuckDBLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState); + return performLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState); } - static RawLookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) { - if (!ConfigHandler.databaseType.isDuckDB()) { - throw new IllegalStateException("DuckDB lookup pages require DuckDB"); + static RawLookupPage performLookupPage(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) { + if (!ConfigHandler.databaseType.isColumnar()) { + throw new IllegalStateException("Lookup pages require a columnar database"); + } + if (ConfigHandler.databaseType.isClickHouse() && knownTotalRows < 0L) { + throw new IllegalArgumentException("ClickHouse lookup pages require a known row count"); + } + if (limitCount <= 0 || limitOffset < 0) { + throw new IllegalArgumentException("Lookup pages require a positive size and non-negative offset"); + } + int currentPage = (limitOffset / limitCount) + 1; + if (cursor != null && (knownTotalRows < 0L || limitOffset == 0 || limitOffset % limitCount != 0 + || cursor.getNextPage() != currentPage || cursor.getPageSize() != limitCount)) { + cursor = null; } boolean paused = false; @@ -102,10 +113,12 @@ public class LookupRaw extends Queue { boolean first = true; while (results.next()) { if (first) { - totalRows = results.getLong("total_count"); - if (knownTotalRows < 0L && rowData != null) { - for (int source = 0; source < rowData.length && source <= InventorySources.ENTITY_INTERACTION; source++) { - rowData[source] = results.getLong("count_" + source); + if (ConfigHandler.databaseType.isDuckDB()) { + totalRows = results.getLong("total_count"); + if (knownTotalRows < 0L && rowData != null) { + for (int source = 0; source < rowData.length && source <= InventorySources.ENTITY_INTERACTION; source++) { + rowData[source] = results.getLong("count_" + source); + } } } orderByTime = results.getBoolean("order_by_time"); @@ -129,9 +142,11 @@ public class LookupRaw extends Queue { return new RawLookupPage(totalRows, Collections.emptyList(), null); } List rows = performLookupRaw(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, -1, -1, restrictWorld, lookup, entityContainerId, rollbackState, pageRows, false); - int currentPage = limitCount > 0 ? (limitOffset / limitCount) + 1 : 1; + if (rows == null || rows.isEmpty()) { + return new RawLookupPage(totalRows, Collections.emptyList(), null); + } LookupCursor nextCursor = foundPageRow ? new LookupCursor(currentPage + 1, limitCount, cursorTime, cursorSource, cursorRowId, orderByTime) : null; - return new RawLookupPage(totalRows, rows == null ? Collections.emptyList() : rows, nextCursor); + return new RawLookupPage(totalRows, rows, nextCursor); } catch (Exception exception) { ErrorReporter.report(exception); @@ -451,7 +466,7 @@ public class LookupRaw extends Queue { } private static ResultSet rawLookupResultSet(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, boolean restrictWorld, boolean lookup, boolean count, Integer entityContainerId, boolean summary, boolean countGroups, boolean includeGroupCount, LookupRollbackState rollbackState, Map> pageRows, boolean selectPageRows) { - return rawLookupResultSet(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, restrictWorld, lookup, count, entityContainerId, summary, countGroups, includeGroupCount, rollbackState, pageRows, selectPageRows, 0, -1L, null); + return rawLookupResultSet(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, restrictWorld, lookup, count, entityContainerId, summary, countGroups, includeGroupCount, rollbackState, pageRows, selectPageRows, limitOffset, -1L, null); } private static ResultSet rawLookupResultSet(Statement statement, CommandSender user, List checkUuids, List checkUsers, List restrictList, Map excludeList, List excludeUserList, List actionList, EntityActionFilter entityActionFilter, List messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, boolean restrictWorld, boolean lookup, boolean count, Integer entityContainerId, boolean summary, boolean countGroups, boolean includeGroupCount, LookupRollbackState rollbackState, Map> pageRows, boolean selectPageRows, int pageOffset, long knownTotalRows, LookupCursor cursor) { @@ -1190,7 +1205,7 @@ public class LookupRaw extends Queue { if (selectPageRows) { if (ConfigHandler.databaseType.isClickHouse()) { - query = buildClickHousePageQuery(query, queryOrder, limitOffset, limitCount); + query = buildClickHousePageQuery(query, pageOffset, limitCount, cursor, queryOrder.contains("time DESC")); } else { query = buildDuckDBPageQuery(query, entityLocationCte, pageOffset, limitCount, knownTotalRows, cursor, queryOrder.contains("time DESC")); @@ -1403,16 +1418,34 @@ public class LookupRaw extends Queue { return query.toString(); } - private static String buildClickHousePageQuery(String sourceQuery, String queryOrder, int offset, int limit) { + private static String buildClickHousePageQuery(String sourceQuery, int offset, int limit, LookupCursor cursor, boolean orderByTime) { if (limit <= 0) { throw new IllegalArgumentException("ClickHouse lookup page size must be positive"); } if (offset < 0) { throw new IllegalArgumentException("ClickHouse lookup page offset must not be negative"); } - String candidateOrder = queryOrder.replace("rowid", "id"); - return "SELECT tbl,id FROM (" + sourceQuery + ") AS coreprotectLookupCandidates" - + candidateOrder + " LIMIT " + limit + " OFFSET " + offset; + if (cursor != null && cursor.isOrderByTime() != orderByTime) { + cursor = null; + } + StringBuilder query = new StringBuilder("SELECT tbl,id,time AS sort_time,") + .append(orderByTime).append(" AS order_by_time FROM (").append(sourceQuery) + .append(") AS coreprotectLookupCandidates"); + if (cursor != null) { + if (orderByTime) { + query.append(" WHERE (time,tbl,id)<(").append(cursor.getTime()).append(',') + .append(cursor.getSource()).append(',').append(cursor.getRowId()).append(')'); + } + else { + query.append(" WHERE id<").append(cursor.getRowId()); + } + } + query.append(orderByTime ? " ORDER BY time DESC,tbl DESC,id DESC" : " ORDER BY id DESC") + .append(" LIMIT ").append(limit); + if (cursor == null && offset > 0) { + query.append(" OFFSET ").append(offset); + } + return query.toString(); } private static String buildRollbackPredicate(LookupRollbackState rollbackState, boolean inventoryRollback) {