diff --git a/pom.xml b/pom.xml
index f8ce3b8..9cea58c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
4.0.0
com.codeforces.inmemo
inmemo
- 2.0.1-SNAPSHOT
+ 2.0.2-SNAPSHOT
jar
inmemo
diff --git a/src/main/java/com/codeforces/inmemo/Index.java b/src/main/java/com/codeforces/inmemo/Index.java
index 8219212..9a279c0 100644
--- a/src/main/java/com/codeforces/inmemo/Index.java
+++ b/src/main/java/com/codeforces/inmemo/Index.java
@@ -267,6 +267,30 @@ long findCount(Object value, Matcher predicate) {
return internalFindCount((V) value, predicate);
}
+ @Nullable
+ BucketStats getBucketStats() {
+ if (unique) {
+ return null;
+ }
+
+ assert map != null;
+
+ long bucketCount = 0;
+ long totalBucketSize = 0;
+ int maxBucketSize = 0;
+
+ for (TLongObjectMap bucket : map.values()) {
+ int bucketSize = bucket.size();
+ bucketCount++;
+ totalBucketSize += bucketSize;
+ if (bucketSize > maxBucketSize) {
+ maxBucketSize = bucketSize;
+ }
+ }
+
+ return new BucketStats(bucketCount, totalBucketSize, maxBucketSize);
+ }
+
/**
* Helper interface to emergency query database if object is not found in memory.
*/
@@ -277,4 +301,36 @@ public interface EmergencyDatabaseHelper {
*/
Object[] getEmergencyQueryFields(@Nullable V indexValue);
}
+
+ static final class BucketStats {
+ private final long bucketCount;
+ private final long totalBucketSize;
+ private final int maxBucketSize;
+
+ private BucketStats(long bucketCount, long totalBucketSize, int maxBucketSize) {
+ this.bucketCount = bucketCount;
+ this.totalBucketSize = totalBucketSize;
+ this.maxBucketSize = maxBucketSize;
+ }
+
+ long getBucketCount() {
+ return bucketCount;
+ }
+
+ long getTotalBucketSize() {
+ return totalBucketSize;
+ }
+
+ int getMaxBucketSize() {
+ return maxBucketSize;
+ }
+
+ double getAverageBucketSize() {
+ if (bucketCount == 0) {
+ return 0.0;
+ }
+
+ return (double) totalBucketSize / bucketCount;
+ }
+ }
}
diff --git a/src/main/java/com/codeforces/inmemo/Inmemo.java b/src/main/java/com/codeforces/inmemo/Inmemo.java
index fe169cc..39ec00d 100644
--- a/src/main/java/com/codeforces/inmemo/Inmemo.java
+++ b/src/main/java/com/codeforces/inmemo/Inmemo.java
@@ -35,6 +35,8 @@ public final class Inmemo {
private static final ConcurrentMap beanCopiers = new ConcurrentHashMap<>();
private static volatile boolean debug;
private static final Set> noSizeSupportClasses = new HashSet<>();
+ private static final Set noJournalSupportTableClassNames
+ = Collections.newSetFromMap(new ConcurrentHashMap<>());
private Inmemo() {
// No operations.
@@ -92,10 +94,55 @@ public static void unsetSizeSupport(@Nonnull Class> clazz) {
noSizeSupportClasses.add(clazz);
}
+ /**
+ * Disables journal support for a table class. The first call for a class
+ * must be made before createTable; repeated calls are always no-op.
+ *
+ * @param clazz Table item class.
+ */
+ public static void unsetJournalSupport(@Nonnull Class> clazz) {
+ String tableClassName = ReflectionUtil.getTableClassName(clazz);
+ tablesLock.lock();
+ try {
+ if (noJournalSupportTableClassNames.contains(tableClassName)) {
+ return;
+ }
+ if (tables.containsKey(tableClassName)) {
+ throw new IllegalStateException("Inmemo.unsetJournalSupport(clazz) must be called"
+ + " before Inmemo.createTable [clazz=" + tableClassName + "].");
+ }
+ noJournalSupportTableClassNames.add(tableClassName);
+ } finally {
+ tablesLock.unlock();
+ }
+ }
+
static Set> getNoSizeSupportClasses() {
return noSizeSupportClasses;
}
+ static boolean isJournalSupportUnset(@Nonnull Class> clazz) {
+ return noJournalSupportTableClassNames.contains(ReflectionUtil.getTableClassName(clazz));
+ }
+
+ static void putTableForUnsetJournalSupportTestOnly(Class clazz) {
+ tablesLock.lock();
+ try {
+ tables.put(ReflectionUtil.getTableClassName(clazz), new Table<>(clazz, "id", null));
+ } finally {
+ tablesLock.unlock();
+ }
+ }
+
+ static void removeTableForUnsetJournalSupportTestOnly(Class clazz) {
+ tablesLock.lock();
+ try {
+ tables.remove(ReflectionUtil.getTableClassName(clazz));
+ } finally {
+ tablesLock.unlock();
+ }
+ }
+
/**
* Creates new table, if there is already table for compatible class then doing nothing.
*
diff --git a/src/main/java/com/codeforces/inmemo/Table.java b/src/main/java/com/codeforces/inmemo/Table.java
index a65e0c9..63280f2 100644
--- a/src/main/java/com/codeforces/inmemo/Table.java
+++ b/src/main/java/com/codeforces/inmemo/Table.java
@@ -16,6 +16,7 @@
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -71,6 +72,14 @@ public static void setJournalsDir(File journalsDir) {
Table.journalsDir = journalsDir;
}
+ static File getJournalsDirForTesting() {
+ return journalsDir;
+ }
+
+ static void setJournalsDirForTesting(File journalsDir) {
+ Table.journalsDir = journalsDir;
+ }
+
Table(Class clazz, String indicatorField, Inmemo.Filter rowFilter) {
this.clazz = clazz;
if (indicatorField.contains("@")) {
@@ -84,6 +93,11 @@ public static void setJournalsDir(File journalsDir) {
clazzSpec = ReflectionUtil.getTableClassSpec(clazz);
ids = Inmemo.getNoSizeSupportClasses().contains(clazz) ? null : new TLongHashSet();
this.rowFilter = rowFilter;
+ if (Inmemo.isJournalSupportUnset(clazz)) {
+ journal = null;
+ useJournal = false;
+ deleteStaleJournalFileQuietly();
+ }
}
void createUpdater(Object initialIndicatorValue) {
@@ -294,6 +308,34 @@ List findAndUpdateByEmergencyQueryFields(Object[] fields) {
return tableUpdater.findAndUpdateByEmergencyQueryFields(fields);
}
+ void logBucketStats() {
+ lock.lock();
+ try {
+ for (Index index : indices.values()) {
+ Index.BucketStats bucketStats = index.getBucketStats();
+ if (bucketStats == null) {
+ continue;
+ }
+
+ logger.info("Inmemo bucket stats [table="
+ + ReflectionUtil.getTableClassName(clazz)
+ + ", index="
+ + index.getName()
+ + ", buckets="
+ + bucketStats.getBucketCount()
+ + ", totalBucketSize="
+ + bucketStats.getTotalBucketSize()
+ + ", avgBucketSize="
+ + String.format(Locale.US, "%.2f", bucketStats.getAverageBucketSize())
+ + ", maxBucketSize="
+ + bucketStats.getMaxBucketSize()
+ + "].");
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
void deleteJournal() throws IOException {
File journalFile = new File(journalsDir, getInmemoFilename());
if (journalFile.isFile()) {
@@ -305,6 +347,17 @@ void deleteJournal() throws IOException {
}
}
+ private void deleteStaleJournalFileQuietly() {
+ try {
+ deleteJournal();
+ logger.info("Journal support is disabled for table '" + clazz.getSimpleName()
+ + "', stale journal file (if any) has been deleted.");
+ } catch (IOException | RuntimeException e) {
+ logger.error("Journal support is disabled for table '" + clazz.getSimpleName()
+ + "', but failed to delete stale journal file.", e);
+ }
+ }
+
private String getInmemoFilename() {
return clazz.getSimpleName() + ".inmemo";
}
diff --git a/src/main/java/com/codeforces/inmemo/TableUpdater.java b/src/main/java/com/codeforces/inmemo/TableUpdater.java
index 4488fb4..41b70d6 100644
--- a/src/main/java/com/codeforces/inmemo/TableUpdater.java
+++ b/src/main/java/com/codeforces/inmemo/TableUpdater.java
@@ -11,8 +11,10 @@
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -21,8 +23,12 @@
*/
class TableUpdater {
private static final Logger logger = Logger.getLogger(TableUpdater.class);
- private static final List tableUpdaterThreads = Collections.synchronizedList(new ArrayList<>());
- private static final AtomicInteger tableUpdaterThreadCount = new AtomicInteger(0);
+
+ private static final List tableUpdaterThreads
+ = Collections.synchronizedList(new ArrayList<>());
+
+ private static final AtomicInteger tableUpdaterThreadCount
+ = new AtomicInteger(0);
/**
* Delay after meaningless update try.
@@ -30,13 +36,14 @@ class TableUpdater {
private static final long RESCAN_TIME_MILLIS = 750;
private static final int MAX_ROWS_IN_SINGLE_SQL_STATEMENT = 2_000_000;
- private static final int MAX_UPDATE_SAME_INDICATOR_TIMES = 5;
+ private static final int MAX_UPDATE_SAME_INDICATOR_TIMES = 10;
private final Lock updateLock = new ReentrantLock();
private static DataSource dataSource;
private static final Map dataSourceByClazzName = new ConcurrentHashMap<>();
- private static final Collection> instances = new ArrayList<>();
+ private static final Collection> instances
+ = Collections.synchronizedList(new ArrayList<>());
private final Table> table;
private final Thread thread;
@@ -46,19 +53,21 @@ class TableUpdater {
private final Jacuzzi jacuzzi;
private final TypeOracle typeOracle;
- private Object lastIndicatorValue;
+ private final AtomicReference