From 119ad1a85fdd34db0a6cb3fad410c4caaaefec27 Mon Sep 17 00:00:00 2001 From: mikemirzayanov Date: Fri, 28 Feb 2025 01:37:55 +0300 Subject: [PATCH 1/6] Better thread-safety --- .../com/codeforces/inmemo/TableUpdater.java | 243 ++++++++++++------ 1 file changed, 161 insertions(+), 82 deletions(-) diff --git a/src/main/java/com/codeforces/inmemo/TableUpdater.java b/src/main/java/com/codeforces/inmemo/TableUpdater.java index 4488fb4..9b18c00 100644 --- a/src/main/java/com/codeforces/inmemo/TableUpdater.java +++ b/src/main/java/com/codeforces/inmemo/TableUpdater.java @@ -11,8 +11,10 @@ import java.io.IOException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -21,8 +23,12 @@ */ class TableUpdater { private static final Logger logger = Logger.getLogger(TableUpdater.class); - private static final List tableUpdaterThreads = Collections.synchronizedList(new ArrayList<>()); - private static final AtomicInteger tableUpdaterThreadCount = new AtomicInteger(0); + + private static final List tableUpdaterThreads + = Collections.synchronizedList(new ArrayList<>()); + + private static final AtomicInteger tableUpdaterThreadCount + = new AtomicInteger(0); /** * Delay after meaningless update try. @@ -36,7 +42,8 @@ class TableUpdater { private static DataSource dataSource; private static final Map dataSourceByClazzName = new ConcurrentHashMap<>(); - private static final Collection> instances = new ArrayList<>(); + private static final Collection> instances + = Collections.synchronizedList(new ArrayList<>()); private final Table table; private final Thread thread; @@ -46,19 +53,21 @@ class TableUpdater { private final Jacuzzi jacuzzi; private final TypeOracle typeOracle; - private Object lastIndicatorValue; + private final AtomicReference lastIndicatorValue = new AtomicReference<>(); private final long startTimeMillis; - private final Map lastEntityIdsUpdateCount = new HashMap<>(); + private final Map lastEntityIdsUpdateCount = new ConcurrentHashMap<>(); TableUpdater(Table table, Object initialIndicatorValue) { if (dataSource == null) { - logger.error("It should be called static Inmemo#setDataSource() before any instance of TableUpdater."); - throw new InmemoException("It should be called static Inmemo#setDataSource() before any instance of TableUpdater."); + logger.error("It should be called static Inmemo#setDataSource()" + + " before any instance of TableUpdater."); + throw new InmemoException("It should be called static Inmemo#setDataSource()" + + " before any instance of TableUpdater."); } this.table = table; - this.lastIndicatorValue = initialIndicatorValue; + this.lastIndicatorValue.set(initialIndicatorValue); DataSource clazzDataSource = dataSourceByClazzName.get(table.getClazz().getName()); jacuzzi = Jacuzzi.getJacuzzi(clazzDataSource == null ? dataSource : clazzDataSource); @@ -103,9 +112,16 @@ private int getMaxUpdateSameIndicatorTimes() { } void insertOrUpdateById(Long id) { - RowRoll rows = jacuzzi.findRowRoll(String.format("SELECT * FROM %s WHERE %s = %s", - typeOracle.getTableName(), typeOracle.getIdColumn(), id.toString() - )); + if (id == null) { + return; + } + + RowRoll rows = jacuzzi.findRowRoll("SELECT * FROM " + + typeOracle.getTableName() + + " WHERE " + + typeOracle.getIdColumn() + + " = " + + id); if (rows == null || rows.isEmpty()) { return; @@ -118,7 +134,8 @@ void insertOrUpdateById(Long id) { table.insertOrUpdate(entity, row); table.insertOrUpdate(row); } else { - throw new InmemoException("Expected at most one item of " + table.getClazz() + " with id = " + id + '.'); + throw new InmemoException("Expected at most one item of " + + table.getClazz() + " with id = " + id + '.'); } } @@ -135,21 +152,37 @@ List findAndUpdateByEmergencyQueryFields(Object[] fields) { String formattedFields = typeOracle.getQueryFindSql(fieldNames); - RowRoll rows = jacuzzi.findRowRoll(String.format("SELECT * FROM %s WHERE %s ORDER BY %s", - typeOracle.getTableName(), formattedFields, typeOracle.getIdColumn()), fieldValues); + RowRoll rows = jacuzzi.findRowRoll("SELECT * FROM " + + typeOracle.getTableName() + + " WHERE " + + formattedFields + + " ORDER BY " + + typeOracle.getIdColumn(), fieldValues); if (rows == null || rows.isEmpty()) { return Collections.emptyList(); } - logger.warn("Emergency case: found " + rows.size() + " items of class " + table.getClazz().getName() + " [fields=" + formattedFields + "]."); + logger.warn("Emergency case: found " + + rows.size() + + " items of class " + + table.getClazz().getName() + + " [fields=" + + formattedFields + + "]."); List result = new ArrayList<>(rows.size()); for (int i = 0; i < rows.size(); i++) { Row row = rows.getRow(i); T entity = typeOracle.convertFromRow(row); - logger.warn("Emergency found: " + table.getClazz().getName() + " id=" + entity.getId() + " [fields=" + formattedFields + "]."); + logger.warn("Emergency found: " + + table.getClazz().getName() + + " id=" + + entity.getId() + + " [fields=" + + formattedFields + + "]."); result.add(entity); @@ -162,14 +195,16 @@ List findAndUpdateByEmergencyQueryFields(Object[] fields) { private static void validateFieldsArray(Object[] fields) { if (fields.length % 2 != 0) { - throw new IllegalArgumentException("EmergencyQueryFields array should have even length. Found: " + throw new IllegalArgumentException("EmergencyQueryFields array should have" + + " even length. Found: " + Arrays.toString(fields) + '.'); } for (int index = 0; index < fields.length; index += 2) { Object field = fields[index]; if (!(field instanceof String) || ((String) field).isEmpty()) { - throw new IllegalArgumentException("EmergencyQueryFields array must contain non-empty strings on even positions. Found: " + throw new IllegalArgumentException("EmergencyQueryFields array must contain" + + " non-empty strings on even positions. Found: " + Arrays.toString(fields) + '.'); } } @@ -181,9 +216,9 @@ void update() { } } - private void update(Random timeSleepRandom) { + private void randomizedUpdate() { List updatedIds = internalUpdate(); - sleepBetweenRescans(timeSleepRandom, updatedIds.size()); + sleepBetweenRescans(updatedIds.size()); } private List internalUpdate() { @@ -191,19 +226,27 @@ private List internalUpdate() { try { long startTimeMillis = System.currentTimeMillis(); - RowRoll rows = getRecentlyChangedRows(lastIndicatorValue); + Object prevLastIndicatorValue = lastIndicatorValue.get(); + RowRoll rows = getRecentlyChangedRows(prevLastIndicatorValue); long afterGetRecentlyChangedRowsMillis = System.currentTimeMillis(); long getRecentlyChangedMillis = afterGetRecentlyChangedRowsMillis - startTimeMillis; - if (rows.size() >= 100 || getRecentlyChangedMillis >= TimeUnit.SECONDS.toMillis(1)) { - logger.error("Table '" + table.getClazz().getSimpleName() + "': getRecentlyChangedRows returns " + rows.size() - + " rows [lastIndicatorValue=" + lastIndicatorValue - + ", thread=" + threadName + if (rows.size() >= 100 + || getRecentlyChangedMillis >= TimeUnit.SECONDS.toMillis(1)) { + logger.error("Table '" + + table.getClazz().getSimpleName() + + "': getRecentlyChangedRows returns " + + rows.size() + + " rows [prevLastIndicatorValue=" + + prevLastIndicatorValue + + ", lastIndicatorValue=" + + lastIndicatorValue.get() + + ", thread=" + + threadName + ", time=" + getRecentlyChangedMillis + " ms]."); } - Object previousIndicatorLastValue = lastIndicatorValue; boolean hasInsertOrUpdateByRow = table.hasInsertOrUpdateByRow(); List updatedIds = new ArrayList<>(); @@ -213,8 +256,9 @@ private List internalUpdate() { for (int i = 0; i < rows.size(); i++) { long id = (long) rows.getValue(i, idColumn); - if (Objects.equals(rows.getValue(i, indicatorFieldColumn), previousIndicatorLastValue) - && lastEntityIdsUpdateCount.containsKey(id) && lastEntityIdsUpdateCount.get(id) >= getMaxUpdateSameIndicatorTimes()) { + if (Objects.equals(rows.getValue(i, indicatorFieldColumn), prevLastIndicatorValue) + && lastEntityIdsUpdateCount.containsKey(id) + && lastEntityIdsUpdateCount.get(id) >= getMaxUpdateSameIndicatorTimes()) { continue; } @@ -231,22 +275,41 @@ private List internalUpdate() { } if (i > 0 && i % 100000 == 0) { - logger.warn("Inserted " + i + " rows in a batch [table=" + table.getClazz().getSimpleName() + "]."); + logger.warn("Inserted " + + i + + " rows in a batch [table=" + + table.getClazz().getSimpleName() + + "]."); } if (table.hasSize()) { int tableSize = table.size(); if (tableSize > 0 && tableSize % 100000 == 0) { - logger.warn("Table " + table.getClazz().getSimpleName() + " contains now " + tableSize + " rows."); + logger.warn("Table " + + table.getClazz().getSimpleName() + + " contains now " + + tableSize + + " rows."); } } - lastIndicatorValue = row.get(table.getIndicatorField()); + lastIndicatorValue.set(row.get(table.getIndicatorField())); } if (updatedIds.size() >= 10) { - logger.info(String.format("Thread '%s' has found %s rows to update in %d ms [lastIndicatorValue=" + lastIndicatorValue + "].", threadName, - rows.size(), getRecentlyChangedMillis)); + logger.info("Thread '" + + threadName + + "' has found " + + rows.size() + + "(" + + updatedIds.size() + + ") rows to update in " + + getRecentlyChangedMillis + + " ms [prevLastIndicatorValue=" + + prevLastIndicatorValue + + ", lastIndicatorValue=" + + lastIndicatorValue.get() + + "]."); if (updatedIds.size() <= 100) { StringBuilder ids = new StringBuilder(); @@ -259,8 +322,17 @@ private List internalUpdate() { logger.info("Updated entries have id=" + ids + '.'); } - logger.info(String.format("Thread '%s' has updated %d items in %d ms [lastIndicatorValue=" + lastIndicatorValue + "].", - threadName, updatedIds.size(), System.currentTimeMillis() - startTimeMillis)); + logger.info("Thread '" + + threadName + + "' has updated " + + updatedIds.size() + + " items in " + + (System.currentTimeMillis() - startTimeMillis) + + " ms [prevLastIndicatorValue=" + + prevLastIndicatorValue + + ", lastIndicatorValue=" + + lastIndicatorValue.get() + + "]."); } if (updatedIds.isEmpty() && !table.isPreloaded()) { @@ -285,13 +357,14 @@ private List internalUpdate() { table.setPreloaded(true); } - if (!Objects.equals(previousIndicatorLastValue, lastIndicatorValue)) { + Object newLastIndicatorValue = lastIndicatorValue.get(); + if (!Objects.equals(prevLastIndicatorValue, newLastIndicatorValue)) { lastEntityIdsUpdateCount.clear(); } List trulyUpdatedIds = new ArrayList<>(updatedIds.size()); for (int i = 0; i < rows.size(); i++) { - if (Objects.equals(rows.getValue(i, indicatorFieldColumn), lastIndicatorValue)) { + if (Objects.equals(rows.getValue(i, indicatorFieldColumn), newLastIndicatorValue)) { long id = (long) rows.getValue(i, idColumn); Integer updateCount = lastEntityIdsUpdateCount.get(id); if (updateCount == null) { @@ -318,26 +391,23 @@ private static int getIdColumn(RowRoll rowRoll) { return column; } - private void sleepBetweenRescans(Random timeSleepRandom, int updatedCount) { + private void sleepBetweenRescans(int updatedCount) { long rescanTimeMillis = getRescanTimeMillis(); if (updatedCount == 0) { - sleep((4 * rescanTimeMillis / 5) + timeSleepRandom.nextInt((int) (rescanTimeMillis / 5))); - } else if ((updatedCount << 1) > MAX_ROWS_IN_SINGLE_SQL_STATEMENT) { - logger.info(String.format( - "Thread '%s' will not sleep because it updated near maximum row count.", threadName - )); + sleep((4 * rescanTimeMillis / 5) + + ThreadLocalRandom.current().nextInt((int) (rescanTimeMillis / 5))); + } else if (updatedCount * 2 > MAX_ROWS_IN_SINGLE_SQL_STATEMENT) { + logger.info("Thread '" + + threadName + + "' will not sleep because it updated near maximum row count."); } else { - sleep(timeSleepRandom.nextInt((int) (rescanTimeMillis / 5))); + sleep(ThreadLocalRandom.current().nextInt((int) (rescanTimeMillis / 5))); } } private long getRescanTimeMillis() { - if (Inmemo.isDebug()) { - return RESCAN_TIME_MILLIS; // * 20; - } else { - return RESCAN_TIME_MILLIS; - } + return RESCAN_TIME_MILLIS; } private void sleep(long timeMillis) { @@ -353,7 +423,7 @@ private RowRoll getRecentlyChangedRows(Object indicatorLastValue) { long startTimeMillis = System.currentTimeMillis(); RowRoll rows = null; - if (lastIndicatorValue == null && table.isUseJournal()) { + if (lastIndicatorValue.get() == null && table.isUseJournal()) { RowRoll journalRows = table.readJournal(); if (journalRows != null && !journalRows.isEmpty()) { rows = journalRows; @@ -370,36 +440,40 @@ private RowRoll getRecentlyChangedRows(Object indicatorLastValue) { String forceIndexClause = table.getDatabaseIndex() == null ? "" : ("FORCE INDEX (" + table.getDatabaseIndex() + ')'); if (indicatorLastValue == null) { - rows = jacuzzi.findRowRoll( - String.format( - "SELECT * FROM %s %s ORDER BY %s, %s LIMIT %d", - typeOracle.getTableName(), - forceIndexClause, - table.getIndicatorField(), - typeOracle.getIdColumn(), - MAX_ROWS_IN_SINGLE_SQL_STATEMENT - ) - ); + rows = jacuzzi.findRowRoll("SELECT * FROM " + + typeOracle.getTableName() + + ' ' + + forceIndexClause + + " ORDER BY " + + table.getIndicatorField() + + ", " + + typeOracle.getIdColumn() + + " LIMIT " + + MAX_ROWS_IN_SINGLE_SQL_STATEMENT); } else { - rows = jacuzzi.findRowRoll( - String.format( - "SELECT * FROM %s %s WHERE %s >= ? ORDER BY %s, %s LIMIT %d", - typeOracle.getTableName(), - forceIndexClause, - table.getIndicatorField(), - table.getIndicatorField(), - typeOracle.getIdColumn(), - MAX_ROWS_IN_SINGLE_SQL_STATEMENT - ), indicatorLastValue + rows = jacuzzi.findRowRoll("SELECT * FROM " + + typeOracle.getTableName() + + ' ' + + forceIndexClause + + " WHERE " + + table.getIndicatorField() + + " >= ? ORDER BY " + + table.getIndicatorField() + + ", " + + typeOracle.getIdColumn() + + " LIMIT " + + MAX_ROWS_IN_SINGLE_SQL_STATEMENT, + indicatorLastValue ); } long queryTimeMillis = System.currentTimeMillis() - startTimeMillis; if (queryTimeMillis * 10 > getRescanTimeMillis()) { - logger.warn(String.format( - "Rescanning query for entity `%s` took too long time %d ms.", - table.getClazz().getName(), queryTimeMillis - )); + logger.warn("Rescanning query for entity `" + + table.getClazz().getName() + + "` took too long time " + + queryTimeMillis + + " ms."); } return rows; } @@ -421,20 +495,23 @@ private TableUpdaterRunnable(TableUpdater tableUpdater) { @Override public void run() { tableUpdaterThreadCount.incrementAndGet(); - @SuppressWarnings("UnsecureRandomNumberGeneration") Random sleepRandom = new Random(); - while (tableUpdater.running) { try { - tableUpdater.update(sleepRandom); + tableUpdater.randomizedUpdate(); } catch (Exception e) { - logger.error("Unexpected " + e.getClass().getName() + " exception in TableUpdaterRunnable of " + logger.error("Unexpected " + + e.getClass().getName() + + " exception in TableUpdaterRunnable of " + tableUpdater.threadName + ": " + e, e); } } tableUpdaterThreadCount.decrementAndGet(); - logger.warn("Inmemo update thread for " + tableUpdater.table.getClazz().getName() + " finished"); + + logger.warn("Inmemo update thread for " + + tableUpdater.table.getClazz().getName() + + " finished."); } } @@ -442,15 +519,17 @@ private static final class TableUpdaterThreadsPrinterRunnable implements Runnabl @SuppressWarnings("BusyWait") @Override public void run() { - while (true) { + while (!Thread.currentThread().isInterrupted()) { try { Thread.sleep(TimeUnit.SECONDS.toMillis(300)); } catch (InterruptedException e) { - logger.warn("TableUpdaterThreadsPrinterRunnable stopped because of InterruptedException."); + logger.warn("TableUpdaterThreadsPrinterRunnable stopped because of" + + " InterruptedException."); break; } if (tableUpdaterThreadCount.get() == 0) { - logger.warn("TableUpdaterThreadsPrinterRunnable stopped because of `tableUpdaterThreadCount.get() == 0`."); + logger.warn("TableUpdaterThreadsPrinterRunnable stopped because of" + + " `tableUpdaterThreadCount.get() == 0`."); break; } logger.info("tableUpdaterThreads.size()=" + tableUpdaterThreads.size() + "."); From c8043228c5b8f010315a3463154f81fbcc9bc535 Mon Sep 17 00:00:00 2001 From: mikemirzayanov Date: Fri, 28 Feb 2025 01:45:55 +0300 Subject: [PATCH 2/6] Syncs --- .../java/com/codeforces/inmemo/TableUpdater.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/codeforces/inmemo/TableUpdater.java b/src/main/java/com/codeforces/inmemo/TableUpdater.java index 9b18c00..611c60a 100644 --- a/src/main/java/com/codeforces/inmemo/TableUpdater.java +++ b/src/main/java/com/codeforces/inmemo/TableUpdater.java @@ -93,8 +93,10 @@ static void setDataSource(DataSource dataSource) { @SuppressWarnings("UnusedDeclaration") static void stop() { - for (TableUpdater instance : instances) { - instance.running = false; + synchronized (instances) { + for (TableUpdater instance : instances) { + instance.running = false; + } } } @@ -527,14 +529,18 @@ public void run() { + " InterruptedException."); break; } + if (tableUpdaterThreadCount.get() == 0) { logger.warn("TableUpdaterThreadsPrinterRunnable stopped because of" + " `tableUpdaterThreadCount.get() == 0`."); break; } - logger.info("tableUpdaterThreads.size()=" + tableUpdaterThreads.size() + "."); - for (Thread tableUpdaterRunnable : tableUpdaterThreads) { - printStackTrace(tableUpdaterRunnable); + + synchronized (tableUpdaterThreads) { + logger.info("tableUpdaterThreads.size()=" + tableUpdaterThreads.size() + "."); + for (Thread tableUpdaterRunnable : tableUpdaterThreads) { + printStackTrace(tableUpdaterRunnable); + } } } } From 1d258857d87f1463168c8b9026e0e04991c8917d Mon Sep 17 00:00:00 2001 From: mikemirzayanov Date: Fri, 28 Feb 2025 01:48:58 +0300 Subject: [PATCH 3/6] MAX_UPDATE_SAME_INDICATOR_TIMES = 10 --- src/main/java/com/codeforces/inmemo/TableUpdater.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/codeforces/inmemo/TableUpdater.java b/src/main/java/com/codeforces/inmemo/TableUpdater.java index 611c60a..332f8ed 100644 --- a/src/main/java/com/codeforces/inmemo/TableUpdater.java +++ b/src/main/java/com/codeforces/inmemo/TableUpdater.java @@ -36,7 +36,7 @@ class TableUpdater { private static final long RESCAN_TIME_MILLIS = 750; private static final int MAX_ROWS_IN_SINGLE_SQL_STATEMENT = 2_000_000; - private static final int MAX_UPDATE_SAME_INDICATOR_TIMES = 5; + private static final int MAX_UPDATE_SAME_INDICATOR_TIMES = 10; private final Lock updateLock = new ReentrantLock(); From 4f3df3175ca2535e59ad35821720051608598234 Mon Sep 17 00:00:00 2001 From: mikemirzayanov Date: Fri, 28 Feb 2025 01:51:38 +0300 Subject: [PATCH 4/6] More use getMaxUpdateSameIndicatorTimes() --- src/main/java/com/codeforces/inmemo/TableUpdater.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/codeforces/inmemo/TableUpdater.java b/src/main/java/com/codeforces/inmemo/TableUpdater.java index 332f8ed..0f54539 100644 --- a/src/main/java/com/codeforces/inmemo/TableUpdater.java +++ b/src/main/java/com/codeforces/inmemo/TableUpdater.java @@ -372,10 +372,11 @@ private List internalUpdate() { if (updateCount == null) { trulyUpdatedIds.add(id); updateCount = 1; - } else { - updateCount += 1; + lastEntityIdsUpdateCount.put(id, updateCount); + } else if (updateCount < getMaxUpdateSameIndicatorTimes()) { + updateCount++; + lastEntityIdsUpdateCount.put(id, updateCount); } - lastEntityIdsUpdateCount.put(id, updateCount); } } From 0ad315bb86cfc814bd8ecc797ef9f384ef5342d3 Mon Sep 17 00:00:00 2001 From: mikemirzayanov Date: Sun, 5 Jul 2026 13:10:11 +0300 Subject: [PATCH 5/6] Add inmemo bucket stats logging --- .../java/com/codeforces/inmemo/Index.java | 56 +++++++++++++++++++ .../java/com/codeforces/inmemo/Table.java | 29 ++++++++++ .../com/codeforces/inmemo/TableUpdater.java | 1 + 3 files changed, 86 insertions(+) diff --git a/src/main/java/com/codeforces/inmemo/Index.java b/src/main/java/com/codeforces/inmemo/Index.java index 8219212..9a279c0 100644 --- a/src/main/java/com/codeforces/inmemo/Index.java +++ b/src/main/java/com/codeforces/inmemo/Index.java @@ -267,6 +267,30 @@ long findCount(Object value, Matcher predicate) { return internalFindCount((V) value, predicate); } + @Nullable + BucketStats getBucketStats() { + if (unique) { + return null; + } + + assert map != null; + + long bucketCount = 0; + long totalBucketSize = 0; + int maxBucketSize = 0; + + for (TLongObjectMap bucket : map.values()) { + int bucketSize = bucket.size(); + bucketCount++; + totalBucketSize += bucketSize; + if (bucketSize > maxBucketSize) { + maxBucketSize = bucketSize; + } + } + + return new BucketStats(bucketCount, totalBucketSize, maxBucketSize); + } + /** * Helper interface to emergency query database if object is not found in memory. */ @@ -277,4 +301,36 @@ public interface EmergencyDatabaseHelper { */ Object[] getEmergencyQueryFields(@Nullable V indexValue); } + + static final class BucketStats { + private final long bucketCount; + private final long totalBucketSize; + private final int maxBucketSize; + + private BucketStats(long bucketCount, long totalBucketSize, int maxBucketSize) { + this.bucketCount = bucketCount; + this.totalBucketSize = totalBucketSize; + this.maxBucketSize = maxBucketSize; + } + + long getBucketCount() { + return bucketCount; + } + + long getTotalBucketSize() { + return totalBucketSize; + } + + int getMaxBucketSize() { + return maxBucketSize; + } + + double getAverageBucketSize() { + if (bucketCount == 0) { + return 0.0; + } + + return (double) totalBucketSize / bucketCount; + } + } } diff --git a/src/main/java/com/codeforces/inmemo/Table.java b/src/main/java/com/codeforces/inmemo/Table.java index a65e0c9..922a302 100644 --- a/src/main/java/com/codeforces/inmemo/Table.java +++ b/src/main/java/com/codeforces/inmemo/Table.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -294,6 +295,34 @@ List findAndUpdateByEmergencyQueryFields(Object[] fields) { return tableUpdater.findAndUpdateByEmergencyQueryFields(fields); } + void logBucketStats() { + lock.lock(); + try { + for (Index index : indices.values()) { + Index.BucketStats bucketStats = index.getBucketStats(); + if (bucketStats == null) { + continue; + } + + logger.info("Inmemo bucket stats [table=" + + ReflectionUtil.getTableClassName(clazz) + + ", index=" + + index.getName() + + ", buckets=" + + bucketStats.getBucketCount() + + ", totalBucketSize=" + + bucketStats.getTotalBucketSize() + + ", avgBucketSize=" + + String.format(Locale.US, "%.2f", bucketStats.getAverageBucketSize()) + + ", maxBucketSize=" + + bucketStats.getMaxBucketSize() + + "]."); + } + } finally { + lock.unlock(); + } + } + void deleteJournal() throws IOException { File journalFile = new File(journalsDir, getInmemoFilename()); if (journalFile.isFile()) { diff --git a/src/main/java/com/codeforces/inmemo/TableUpdater.java b/src/main/java/com/codeforces/inmemo/TableUpdater.java index 0f54539..41b70d6 100644 --- a/src/main/java/com/codeforces/inmemo/TableUpdater.java +++ b/src/main/java/com/codeforces/inmemo/TableUpdater.java @@ -356,6 +356,7 @@ private List internalUpdate() { logger.log(totalTimeMillis < TimeUnit.SECONDS.toMillis(1) ? Level.INFO : Level.WARN, "Inmemo preloaded " + ReflectionUtil.getTableClassName(table.getClazz()) + " in " + totalTimeMillis + " ms."); } + table.logBucketStats(); table.setPreloaded(true); } From e849a61b04af5c63869d0687cc9e9db5e3dcd5a3 Mon Sep 17 00:00:00 2001 From: mikemirzayanov Date: Sun, 5 Jul 2026 13:28:13 +0300 Subject: [PATCH 6/6] Add per-class journal opt-out --- pom.xml | 2 +- .../java/com/codeforces/inmemo/Inmemo.java | 47 +++++ .../java/com/codeforces/inmemo/Table.java | 24 +++ .../inmemo/UnsetJournalSupportTest.java | 178 ++++++++++++++++++ .../inmemo/model/JournalDisabledUser.java | 36 ++++ .../inmemo/model/JournalEnabledUser.java | 36 ++++ .../inmemo/model/JournalLateUnsetUser.java | 36 ++++ 7 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/codeforces/inmemo/UnsetJournalSupportTest.java create mode 100644 src/test/java/com/codeforces/inmemo/model/JournalDisabledUser.java create mode 100644 src/test/java/com/codeforces/inmemo/model/JournalEnabledUser.java create mode 100644 src/test/java/com/codeforces/inmemo/model/JournalLateUnsetUser.java diff --git a/pom.xml b/pom.xml index ce0ba51..9cea58c 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.codeforces.inmemo inmemo - 2.0-SNAPSHOT + 2.0.2-SNAPSHOT jar inmemo diff --git a/src/main/java/com/codeforces/inmemo/Inmemo.java b/src/main/java/com/codeforces/inmemo/Inmemo.java index fe169cc..39ec00d 100644 --- a/src/main/java/com/codeforces/inmemo/Inmemo.java +++ b/src/main/java/com/codeforces/inmemo/Inmemo.java @@ -35,6 +35,8 @@ public final class Inmemo { private static final ConcurrentMap beanCopiers = new ConcurrentHashMap<>(); private static volatile boolean debug; private static final Set> noSizeSupportClasses = new HashSet<>(); + private static final Set noJournalSupportTableClassNames + = Collections.newSetFromMap(new ConcurrentHashMap<>()); private Inmemo() { // No operations. @@ -92,10 +94,55 @@ public static void unsetSizeSupport(@Nonnull Class clazz) { noSizeSupportClasses.add(clazz); } + /** + * Disables journal support for a table class. The first call for a class + * must be made before createTable; repeated calls are always no-op. + * + * @param clazz Table item class. + */ + public static void unsetJournalSupport(@Nonnull Class clazz) { + String tableClassName = ReflectionUtil.getTableClassName(clazz); + tablesLock.lock(); + try { + if (noJournalSupportTableClassNames.contains(tableClassName)) { + return; + } + if (tables.containsKey(tableClassName)) { + throw new IllegalStateException("Inmemo.unsetJournalSupport(clazz) must be called" + + " before Inmemo.createTable [clazz=" + tableClassName + "]."); + } + noJournalSupportTableClassNames.add(tableClassName); + } finally { + tablesLock.unlock(); + } + } + static Set> getNoSizeSupportClasses() { return noSizeSupportClasses; } + static boolean isJournalSupportUnset(@Nonnull Class clazz) { + return noJournalSupportTableClassNames.contains(ReflectionUtil.getTableClassName(clazz)); + } + + static void putTableForUnsetJournalSupportTestOnly(Class clazz) { + tablesLock.lock(); + try { + tables.put(ReflectionUtil.getTableClassName(clazz), new Table<>(clazz, "id", null)); + } finally { + tablesLock.unlock(); + } + } + + static void removeTableForUnsetJournalSupportTestOnly(Class clazz) { + tablesLock.lock(); + try { + tables.remove(ReflectionUtil.getTableClassName(clazz)); + } finally { + tablesLock.unlock(); + } + } + /** * Creates new table, if there is already table for compatible class then doing nothing. * diff --git a/src/main/java/com/codeforces/inmemo/Table.java b/src/main/java/com/codeforces/inmemo/Table.java index 922a302..63280f2 100644 --- a/src/main/java/com/codeforces/inmemo/Table.java +++ b/src/main/java/com/codeforces/inmemo/Table.java @@ -72,6 +72,14 @@ public static void setJournalsDir(File journalsDir) { Table.journalsDir = journalsDir; } + static File getJournalsDirForTesting() { + return journalsDir; + } + + static void setJournalsDirForTesting(File journalsDir) { + Table.journalsDir = journalsDir; + } + Table(Class clazz, String indicatorField, Inmemo.Filter rowFilter) { this.clazz = clazz; if (indicatorField.contains("@")) { @@ -85,6 +93,11 @@ public static void setJournalsDir(File journalsDir) { clazzSpec = ReflectionUtil.getTableClassSpec(clazz); ids = Inmemo.getNoSizeSupportClasses().contains(clazz) ? null : new TLongHashSet(); this.rowFilter = rowFilter; + if (Inmemo.isJournalSupportUnset(clazz)) { + journal = null; + useJournal = false; + deleteStaleJournalFileQuietly(); + } } void createUpdater(Object initialIndicatorValue) { @@ -334,6 +347,17 @@ void deleteJournal() throws IOException { } } + private void deleteStaleJournalFileQuietly() { + try { + deleteJournal(); + logger.info("Journal support is disabled for table '" + clazz.getSimpleName() + + "', stale journal file (if any) has been deleted."); + } catch (IOException | RuntimeException e) { + logger.error("Journal support is disabled for table '" + clazz.getSimpleName() + + "', but failed to delete stale journal file.", e); + } + } + private String getInmemoFilename() { return clazz.getSimpleName() + ".inmemo"; } diff --git a/src/test/java/com/codeforces/inmemo/UnsetJournalSupportTest.java b/src/test/java/com/codeforces/inmemo/UnsetJournalSupportTest.java new file mode 100644 index 0000000..60e56e2 --- /dev/null +++ b/src/test/java/com/codeforces/inmemo/UnsetJournalSupportTest.java @@ -0,0 +1,178 @@ +package com.codeforces.inmemo; + +import com.codeforces.inmemo.model.JournalDisabledUser; +import com.codeforces.inmemo.model.JournalEnabledUser; +import com.codeforces.inmemo.model.JournalLateUnsetUser; +import org.jacuzzi.core.Row; +import org.jacuzzi.core.RowRoll; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.nio.file.Files; + +public class UnsetJournalSupportTest { + private String oldUseJournalProperty; + private File oldJournalsDir; + private File tempJournalsDir; + + @Before + public void setUp() throws Exception { + oldUseJournalProperty = System.getProperty("Inmemo.UseJournal"); + System.setProperty("Inmemo.UseJournal", "true"); + oldJournalsDir = Table.getJournalsDirForTesting(); + tempJournalsDir = Files.createTempDirectory("inmemo-journals-").toFile(); + Table.setJournalsDir(tempJournalsDir); + } + + @After + public void tearDown() { + if (oldUseJournalProperty == null) { + System.clearProperty("Inmemo.UseJournal"); + } else { + System.setProperty("Inmemo.UseJournal", oldUseJournalProperty); + } + + if (oldJournalsDir != null) { + Table.setJournalsDirForTesting(oldJournalsDir); + } + + deleteRecursively(tempJournalsDir); + } + + @Test + public void testUnsetDisablesJournal() throws Exception { + Inmemo.unsetJournalSupport(JournalDisabledUser.class); + Table table = new Table<>(JournalDisabledUser.class, "id", null); + + Assert.assertFalse(table.isUseJournal()); + table.insertOrUpdate(user(1L, "disabled", "disabled@example.com"), + row(1L, "disabled", "disabled@example.com")); + table.writeJournal(); + + Assert.assertFalse(journalFile(JournalDisabledUser.class).exists()); + Assert.assertNull(table.readJournal()); + } + + @Test + public void testStaleJournalFileDeletedOnConstruction() throws Exception { + Inmemo.unsetJournalSupport(JournalDisabledUser.class); + File journalFile = journalFile(JournalDisabledUser.class); + Files.write(journalFile.toPath(), new byte[]{1, 2, 3}); + + Assert.assertTrue(journalFile.isFile()); + new Table<>(JournalDisabledUser.class, "id", null); + + Assert.assertFalse(journalFile.exists()); + } + + @Test + public void testDefaultTableStillJournals() throws Exception { + Table table = new Table<>(JournalEnabledUser.class, "id", null); + + Assert.assertTrue(table.isUseJournal()); + table.insertOrUpdate(enabledUser(2L, "enabled", "enabled@example.com"), + row(2L, "enabled", "enabled@example.com")); + table.writeJournal(); + + Assert.assertTrue(journalFile(JournalEnabledUser.class).isFile()); + RowRoll rows = table.readJournal(); + Assert.assertNotNull(rows); + Assert.assertEquals(1, rows.size()); + + Row savedRow = rows.getRow(0); + Assert.assertEquals(2L, ((Number) savedRow.get("id")).longValue()); + Assert.assertEquals("enabled", savedRow.get("handle")); + Assert.assertEquals("enabled@example.com", savedRow.get("email")); + } + + @Test + public void testUnsetAfterCreateTableThrows() { + try { + Inmemo.putTableForUnsetJournalSupportTestOnly(JournalLateUnsetUser.class); + Inmemo.unsetJournalSupport(JournalLateUnsetUser.class); + Assert.fail("Late first unsetJournalSupport call must fail."); + } catch (IllegalStateException expected) { + // Expected. + } finally { + Inmemo.removeTableForUnsetJournalSupportTestOnly(JournalLateUnsetUser.class); + } + } + + @Test + public void testGlobalOffAndUnsetCompose() throws Exception { + System.clearProperty("Inmemo.UseJournal"); + Inmemo.unsetJournalSupport(JournalDisabledUser.class); + File staleJournalFile = journalFile(JournalDisabledUser.class); + Files.write(staleJournalFile.toPath(), new byte[]{1, 2, 3}); + + Table disabledTable = new Table<>(JournalDisabledUser.class, "id", null); + Table enabledTable = new Table<>(JournalEnabledUser.class, "id", null); + + Assert.assertFalse(disabledTable.isUseJournal()); + Assert.assertFalse(staleJournalFile.exists()); + Assert.assertFalse(enabledTable.isUseJournal()); + } + + @Test + public void testUnsetIsIdempotent() { + Inmemo.unsetJournalSupport(JournalDisabledUser.class); + Inmemo.unsetJournalSupport(JournalDisabledUser.class); + + try { + Inmemo.putTableForUnsetJournalSupportTestOnly(JournalDisabledUser.class); + Inmemo.unsetJournalSupport(JournalDisabledUser.class); + } finally { + Inmemo.removeTableForUnsetJournalSupportTestOnly(JournalDisabledUser.class); + } + } + + private static File journalFile(Class clazz) { + return new File(Table.getJournalsDirForTesting(), clazz.getSimpleName() + ".inmemo"); + } + + private static JournalDisabledUser user(long id, String handle, String email) { + JournalDisabledUser user = new JournalDisabledUser(); + user.setId(id); + user.setHandle(handle); + user.setEmail(email); + return user; + } + + private static JournalEnabledUser enabledUser(long id, String handle, String email) { + JournalEnabledUser user = new JournalEnabledUser(); + user.setId(id); + user.setHandle(handle); + user.setEmail(email); + return user; + } + + private static Row row(long id, String handle, String email) { + Row row = new Row(); + row.put("id", id); + row.put("handle", handle); + row.put("email", email); + return row; + } + + private static void deleteRecursively(File file) { + if (file == null || !file.exists()) { + return; + } + + if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files != null) { + for (File child : files) { + deleteRecursively(child); + } + } + } + + if (!file.delete() && file.exists()) { + throw new AssertionError("Can't delete " + file.getAbsolutePath()); + } + } +} diff --git a/src/test/java/com/codeforces/inmemo/model/JournalDisabledUser.java b/src/test/java/com/codeforces/inmemo/model/JournalDisabledUser.java new file mode 100644 index 0000000..2ea8698 --- /dev/null +++ b/src/test/java/com/codeforces/inmemo/model/JournalDisabledUser.java @@ -0,0 +1,36 @@ +package com.codeforces.inmemo.model; + +import com.codeforces.inmemo.HasId; +import org.jacuzzi.mapping.Id; + +public class JournalDisabledUser implements HasId { + @Id + private long id; + private String handle; + private String email; + + @Override + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getHandle() { + return handle; + } + + public void setHandle(String handle) { + this.handle = handle; + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } +} diff --git a/src/test/java/com/codeforces/inmemo/model/JournalEnabledUser.java b/src/test/java/com/codeforces/inmemo/model/JournalEnabledUser.java new file mode 100644 index 0000000..87a3bd4 --- /dev/null +++ b/src/test/java/com/codeforces/inmemo/model/JournalEnabledUser.java @@ -0,0 +1,36 @@ +package com.codeforces.inmemo.model; + +import com.codeforces.inmemo.HasId; +import org.jacuzzi.mapping.Id; + +public class JournalEnabledUser implements HasId { + @Id + private long id; + private String handle; + private String email; + + @Override + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getHandle() { + return handle; + } + + public void setHandle(String handle) { + this.handle = handle; + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } +} diff --git a/src/test/java/com/codeforces/inmemo/model/JournalLateUnsetUser.java b/src/test/java/com/codeforces/inmemo/model/JournalLateUnsetUser.java new file mode 100644 index 0000000..cd81193 --- /dev/null +++ b/src/test/java/com/codeforces/inmemo/model/JournalLateUnsetUser.java @@ -0,0 +1,36 @@ +package com.codeforces.inmemo.model; + +import com.codeforces.inmemo.HasId; +import org.jacuzzi.mapping.Id; + +public class JournalLateUnsetUser implements HasId { + @Id + private long id; + private String handle; + private String email; + + @Override + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getHandle() { + return handle; + } + + public void setHandle(String handle) { + this.handle = handle; + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } +}