Align ClickHouse lookup pagination with DuckDB

src/main/java/net/coreprotect/command/lookup/StandardLookupThread.java

@@ -161,8 +161,8 @@ public class StandardLookupThread implements Runnable {
             ConfigHandler.lookupRadius.put(player.getName(), radius);
             ConfigHandler.lookupOutputMode.put(player.getName(), outputMode == LookupOutputMode.COUNT ? LookupOutputMode.DETAIL : outputMode);
             ConfigHandler.lookupRollbackState.put(player.getName(), rollbackState);
-            if (typeLookup != 5 || outputMode != LookupOutputMode.DETAIL || !ConfigHandler.databaseType.isDuckDB()) {
-                ConfigHandler.lookupDuckDBCursor.remove(player.getName());
+            if (typeLookup != 5 || outputMode != LookupOutputMode.DETAIL || !ConfigHandler.databaseType.isColumnar()) {
+                ConfigHandler.lookupCursor.remove(player.getName());
             }
 
             if (connection != null) {
@@ -290,20 +290,17 @@ public class StandardLookupThread implements Runnable {
                         rowData[4] = rows;
                         ConfigHandler.lookupRows.put(player.getName(), rowData);
                     }
-                    if (lookupPage == null && outputMode == LookupOutputMode.DETAIL && ConfigHandler.databaseType.isDuckDB() && pageStart < rows) {
-                        LookupCursor cursor = ConfigHandler.lookupDuckDBCursor.get(player.getName());
-                        if (cursor == null || cursor.getNextPage() != page || cursor.getPageSize() != displayResults) {
-                            cursor = null;
-                        }
-                        lookupPage = Lookup.performDuckDBLookupPage(statement, player, uuidList, userList, blockList, excludedBlocks, excludedUsers, actions, entityActionFilter, messageFilters, entityContext, finalLocation, radius, rowData, timeStart, timeEnd, (int) pageStart, displayResults, rows, cursor, restrict_world, true, entityContainerId, rollbackState);
+                    if (lookupPage == null && outputMode == LookupOutputMode.DETAIL && ConfigHandler.databaseType.isColumnar() && pageStart < rows) {
+                        LookupCursor cursor = ConfigHandler.lookupCursor.get(player.getName());
+                        lookupPage = Lookup.performLookupPage(statement, player, uuidList, userList, blockList, excludedBlocks, excludedUsers, actions, entityActionFilter, messageFilters, entityContext, finalLocation, radius, rowData, timeStart, timeEnd, (int) pageStart, displayResults, rows, cursor, restrict_world, true, entityContainerId, rollbackState);
                     }
                     if (lookupPage != null) {
                         LookupCursor nextCursor = lookupPage.getNextCursor();
                         if (nextCursor == null) {
-                            ConfigHandler.lookupDuckDBCursor.remove(player.getName());
+                            ConfigHandler.lookupCursor.remove(player.getName());
                         }
                         else {
-                            ConfigHandler.lookupDuckDBCursor.put(player.getName(), nextCursor);
+                            ConfigHandler.lookupCursor.put(player.getName(), nextCursor);
                         }
                     }
                     if (outputMode == LookupOutputMode.COUNT) {

src/main/java/net/coreprotect/config/ConfigHandler.java

@@ -272,7 +272,7 @@ public class ConfigHandler extends Queue {
     public static Map<String, Integer[]> lookupRadius = syncMap();
     public static Map<String, String> lookupTime = syncMap();
     public static Map<String, Long[]> lookupRows = syncMap();
-    public static Map<String, LookupCursor> lookupDuckDBCursor = syncMap();
+    public static Map<String, LookupCursor> lookupCursor = syncMap();
     public static Map<String, String> uuidCache = syncMap();
     public static Map<String, String> uuidCacheReversed = syncMap();
     public static Map<String, Integer> playerIdCache = syncMap();

src/main/java/net/coreprotect/database/Lookup.java

@@ -301,11 +301,11 @@ public class Lookup extends Queue {
     }
 
     public static LookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitCount, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) {
-        return performDuckDBLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState);
+        return performLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState);
     }
 
-    public static LookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) {
-        LookupRaw.RawLookupPage page = LookupRaw.performDuckDBLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, knownTotalRows, cursor, restrictWorld, lookup, entityContainerId, rollbackState);
+    public static LookupPage performLookupPage(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) {
+        LookupRaw.RawLookupPage page = LookupRaw.performLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, knownTotalRows, cursor, restrictWorld, lookup, entityContainerId, rollbackState);
         List<String[]> rows = LookupConverter.convertRawLookup(statement, page.getRows());
         return new LookupPage(page.getTotalRows(), rows, page.getNextCursor());
     }

src/main/java/net/coreprotect/database/LookupRaw.java

@@ -75,12 +75,23 @@ public class LookupRaw extends Queue {
     }
 
     static RawLookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitCount, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) {
-        return performDuckDBLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState);
+        return performLookupPage(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, 0, limitCount, -1L, null, restrictWorld, lookup, entityContainerId, rollbackState);
     }
 
-    static RawLookupPage performDuckDBLookupPage(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) {
-        if (!ConfigHandler.databaseType.isDuckDB()) {
-            throw new IllegalStateException("DuckDB lookup pages require DuckDB");
+    static RawLookupPage performLookupPage(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, long knownTotalRows, LookupCursor cursor, boolean restrictWorld, boolean lookup, Integer entityContainerId, LookupRollbackState rollbackState) {
+        if (!ConfigHandler.databaseType.isColumnar()) {
+            throw new IllegalStateException("Lookup pages require a columnar database");
+        }
+        if (ConfigHandler.databaseType.isClickHouse() && knownTotalRows < 0L) {
+            throw new IllegalArgumentException("ClickHouse lookup pages require a known row count");
+        }
+        if (limitCount <= 0 || limitOffset < 0) {
+            throw new IllegalArgumentException("Lookup pages require a positive size and non-negative offset");
+        }
+        int currentPage = (limitOffset / limitCount) + 1;
+        if (cursor != null && (knownTotalRows < 0L || limitOffset == 0 || limitOffset % limitCount != 0
+                || cursor.getNextPage() != currentPage || cursor.getPageSize() != limitCount)) {
+            cursor = null;
         }
 
         boolean paused = false;
@@ -102,12 +113,14 @@ public class LookupRaw extends Queue {
                 boolean first = true;
                 while (results.next()) {
                     if (first) {
+                        if (ConfigHandler.databaseType.isDuckDB()) {
                             totalRows = results.getLong("total_count");
                             if (knownTotalRows < 0L && rowData != null) {
                                 for (int source = 0; source < rowData.length && source <= InventorySources.ENTITY_INTERACTION; source++) {
                                     rowData[source] = results.getLong("count_" + source);
                                 }
                             }
+                        }
                         orderByTime = results.getBoolean("order_by_time");
                         first = false;
                     }
@@ -129,9 +142,11 @@ public class LookupRaw extends Queue {
                 return new RawLookupPage(totalRows, Collections.emptyList(), null);
             }
             List<Object[]> rows = performLookupRaw(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, -1, -1, restrictWorld, lookup, entityContainerId, rollbackState, pageRows, false);
-            int currentPage = limitCount > 0 ? (limitOffset / limitCount) + 1 : 1;
+            if (rows == null || rows.isEmpty()) {
+                return new RawLookupPage(totalRows, Collections.emptyList(), null);
+            }
             LookupCursor nextCursor = foundPageRow ? new LookupCursor(currentPage + 1, limitCount, cursorTime, cursorSource, cursorRowId, orderByTime) : null;
-            return new RawLookupPage(totalRows, rows == null ? Collections.emptyList() : rows, nextCursor);
+            return new RawLookupPage(totalRows, rows, nextCursor);
         }
         catch (Exception exception) {
             ErrorReporter.report(exception);
@@ -451,7 +466,7 @@ public class LookupRaw extends Queue {
     }
 
     private static ResultSet rawLookupResultSet(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, boolean restrictWorld, boolean lookup, boolean count, Integer entityContainerId, boolean summary, boolean countGroups, boolean includeGroupCount, LookupRollbackState rollbackState, Map<Integer, List<Long>> pageRows, boolean selectPageRows) {
-        return rawLookupResultSet(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, restrictWorld, lookup, count, entityContainerId, summary, countGroups, includeGroupCount, rollbackState, pageRows, selectPageRows, 0, -1L, null);
+        return rawLookupResultSet(statement, user, checkUuids, checkUsers, restrictList, excludeList, excludeUserList, actionList, entityActionFilter, messageFilters, entityContext, location, radius, rowData, startTime, endTime, limitOffset, limitCount, restrictWorld, lookup, count, entityContainerId, summary, countGroups, includeGroupCount, rollbackState, pageRows, selectPageRows, limitOffset, -1L, null);
     }
 
     private static ResultSet rawLookupResultSet(Statement statement, CommandSender user, List<String> checkUuids, List<String> checkUsers, List<Object> restrictList, Map<Object, Boolean> excludeList, List<String> excludeUserList, List<Integer> actionList, EntityActionFilter entityActionFilter, List<String> messageFilters, EntityLookupContext entityContext, Location location, Integer[] radius, Long[] rowData, long startTime, long endTime, int limitOffset, int limitCount, boolean restrictWorld, boolean lookup, boolean count, Integer entityContainerId, boolean summary, boolean countGroups, boolean includeGroupCount, LookupRollbackState rollbackState, Map<Integer, List<Long>> pageRows, boolean selectPageRows, int pageOffset, long knownTotalRows, LookupCursor cursor) {
@@ -1190,7 +1205,7 @@ public class LookupRaw extends Queue {
 
             if (selectPageRows) {
                 if (ConfigHandler.databaseType.isClickHouse()) {
-                    query = buildClickHousePageQuery(query, queryOrder, limitOffset, limitCount);
+                    query = buildClickHousePageQuery(query, pageOffset, limitCount, cursor, queryOrder.contains("time DESC"));
                 }
                 else {
                     query = buildDuckDBPageQuery(query, entityLocationCte, pageOffset, limitCount, knownTotalRows, cursor, queryOrder.contains("time DESC"));
@@ -1403,16 +1418,34 @@ public class LookupRaw extends Queue {
         return query.toString();
     }
 
-    private static String buildClickHousePageQuery(String sourceQuery, String queryOrder, int offset, int limit) {
+    private static String buildClickHousePageQuery(String sourceQuery, int offset, int limit, LookupCursor cursor, boolean orderByTime) {
         if (limit <= 0) {
             throw new IllegalArgumentException("ClickHouse lookup page size must be positive");
         }
         if (offset < 0) {
             throw new IllegalArgumentException("ClickHouse lookup page offset must not be negative");
         }
-        String candidateOrder = queryOrder.replace("rowid", "id");
-        return "SELECT tbl,id FROM (" + sourceQuery + ") AS coreprotectLookupCandidates"
-                + candidateOrder + " LIMIT " + limit + " OFFSET " + offset;
+        if (cursor != null && cursor.isOrderByTime() != orderByTime) {
+            cursor = null;
+        }
+        StringBuilder query = new StringBuilder("SELECT tbl,id,time AS sort_time,")
+                .append(orderByTime).append(" AS order_by_time FROM (").append(sourceQuery)
+                .append(") AS coreprotectLookupCandidates");
+        if (cursor != null) {
+            if (orderByTime) {
+                query.append(" WHERE (time,tbl,id)<(").append(cursor.getTime()).append(',')
+                        .append(cursor.getSource()).append(',').append(cursor.getRowId()).append(')');
+            }
+            else {
+                query.append(" WHERE id<").append(cursor.getRowId());
+            }
+        }
+        query.append(orderByTime ? " ORDER BY time DESC,tbl DESC,id DESC" : " ORDER BY id DESC")
+                .append(" LIMIT ").append(limit);
+        if (cursor == null && offset > 0) {
+            query.append(" OFFSET ").append(offset);
+        }
+        return query.toString();
     }
 
     private static String buildRollbackPredicate(LookupRollbackState rollbackState, boolean inventoryRollback) {