diff --git a/plugins/README.md b/plugins/README.md index f14e070c01a..ece19006b58 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -148,6 +148,49 @@ NOTE: large db may GC overhead limit exceeded. - `--db`: db name. - `-h | --help`: provide the help info +## DB Backfill-Bloom + +DB backfill bloom rebuilds SectionBloom data for historical blocks so that `eth_getLogs` can filter by address and topics. This is useful when `isJsonRpcFilterEnabled` was disabled during historical block processing and was enabled later. + +### Prerequisites and behavior + +- Stop the node and any other process using the database before running the command. +- The database directory must contain the `properties` and `transactionRetStore` databases. `transactionRetStore` must contain at least one non-zero block. +- Ensure `storage.transHistory.switch` was enabled while the historical blocks were processed. +- The start and end block numbers are inclusive. +- The command creates or updates the `section-bloom` database in the specified database directory. +- The operation is idempotent. If it is interrupted, safely rerun the same block range. Existing SectionBloom bits are preserved and set again. Do not run multiple backfill processes concurrently. + +### Available parameters + +- `-d | --database-directory`: Parent directory containing the source databases and the destination `section-bloom` database. Default: `output-directory/database`. +- `-s | --start-block`: Inclusive start block. Optional; defaults to the earliest non-zero block in `transactionRetStore`. A lower value is automatically raised to the earliest available block. +- `-e | --end-block`: Inclusive end block. Optional; defaults to the latest solidified block in `properties`. A higher value is automatically reduced to the latest solidified block. +- `-c | --max-concurrency`: Maximum number of processing threads, from 1 to 128. Default: 8. Use 4–8 for SATA SSD, 8–16 for NVMe SSD, or 1–2 for HDD. The actual concurrency does not exceed the number of sections being processed. +- `-h | --help`: Display the help message. + +### Examples + +```shell script +# Full command +java -jar Toolkit.jar db backfill-bloom [-d ] [-s ] [-e ] [-c ] [-h] + +# Backfill the complete available range in the default database directory +java -jar Toolkit.jar db backfill-bloom + +# Backfill blocks 1,000,000 through 2,000,000, inclusive +java -jar Toolkit.jar db backfill-bloom -s 1000000 -e 2000000 + +# Use a custom database directory and eight processing threads +java -jar Toolkit.jar db backfill-bloom -d /path/to/database -c 8 +``` + +### Progress and performance + +The terminal progress bar displays completed blocks, elapsed time, and estimated remaining time. `toolkit.log` records progress every 10,000 scanned blocks and includes the percentage, elapsed time, average rate, and estimated remaining time. The final summary reports scanned and successful blocks, blocks containing logs, errors, Bloom writes, duration, rates, and the concurrency used. + +Performance depends on the number of logs, storage engine, disk, CPU, and database compaction. Increase `--max-concurrency` gradually while monitoring disk latency and CPU usage. + ## Keystore Keystore provides commands for managing account keystore files (Web3 Secret Storage format). diff --git a/plugins/build.gradle b/plugins/build.gradle index 09a13a19b1b..1530c6bcf06 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -70,6 +70,7 @@ dependencies { implementation 'io.github.tronprotocol:leveldb:1.18.2' } implementation project(":protocol") + implementation project(":chainbase") } check.dependsOn 'lint' @@ -149,7 +150,7 @@ def binaryRelease(taskName, jarName, mainClass) { // (and so partial / parallel builds cannot run binaryRelease before the // dependency jars exist). dependsOn (project(':protocol').jar, project(':platform').jar, - project(':crypto').jar, project(':common').jar) // explicit_dependency + project(':crypto').jar, project(':common').jar, project(':chainbase').jar) // explicit_dependency from { configurations.runtimeClasspath.collect { // https://docs.gradle.org/current/userguide/upgrading_version_6.html#changes_6.3 it.isDirectory() ? it : zipTree(it) diff --git a/plugins/src/main/java/common/org/tron/plugins/Db.java b/plugins/src/main/java/common/org/tron/plugins/Db.java index 84654dca934..0918d939dc5 100644 --- a/plugins/src/main/java/common/org/tron/plugins/Db.java +++ b/plugins/src/main/java/common/org/tron/plugins/Db.java @@ -12,6 +12,7 @@ DbConvert.class, DbLite.class, DbCopy.class, + DbBackfillBloom.class, DbRoot.class }, commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n" diff --git a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java new file mode 100644 index 00000000000..8f6efcb7149 --- /dev/null +++ b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java @@ -0,0 +1,557 @@ +package org.tron.plugins; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import me.tongfei.progressbar.ProgressBar; +import org.apache.commons.collections4.CollectionUtils; +import org.rocksdb.RocksDBException; +import org.tron.common.bloom.Bloom; +import org.tron.common.es.ExecutorServiceManager; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.ByteUtil; +import org.tron.core.capsule.TransactionRetCapsule; +import org.tron.core.exception.BadItemException; +import org.tron.core.exception.EventBloomException; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; +import picocli.CommandLine; + +@Slf4j(topic = "backfill-bloom") +@CommandLine.Command(name = "backfill-bloom", + description = { + "Backfill SectionBloom for historical blocks to enable eth_getLogs filtering.", + "The same block range can be safely rerun after interruption." + }, + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "1:Internal error: exception occurred, please check toolkit.log"}) +public class DbBackfillBloom implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"--database-directory", "-d"}, + defaultValue = "output-directory/database", + description = "Database directory path. Default: ${DEFAULT-VALUE}", order = 1) + private String databaseDirectory; + + @CommandLine.Option(names = {"--start-block", "-s"}, + description = "Start block number for backfill. Default: earliest block", order = 2) + private long startBlock; + + @CommandLine.Option(names = {"--end-block", "-e"}, + description = "End block number for backfill, inclusive. Default: latest solidity block", + order = 3) + private long endBlock; + + // sames as SectionBloomStore.BLOCK_PER_SECTION + private static final int BLOCKS_PER_SECTION = 2048; + private static final long PROGRESS_LOG_INTERVAL = 10_000L; + private static final String PROPERTIES_DB_NAME = "properties"; + private static final String TRANSACTION_RET_DB_NAME = "transactionRetStore"; + private static final String SECTION_BLOOM_DB_NAME = "section-bloom"; + private static final String LATEST_SOLIDIFIED_BLOCK_NUM = "LATEST_SOLIDIFIED_BLOCK_NUM"; + + @CommandLine.Option(names = {"--max-concurrency", "-c"}, defaultValue = "8", + description = "Maximum concurrency for processing. Default: ${DEFAULT-VALUE}. For SATA SSD " + + "use 4–8; for NVMe SSD use 8–16; for HDD use 1–2.", + order = 5) + private int maxConcurrency; + + @CommandLine.Option(names = {"--help", "-h"}, help = true, + description = "Display help message", order = 7) + private boolean help; + + // Statistics + // Number of blocks traversed (including failed ones) + private final AtomicLong processedBlocks = new AtomicLong(0); + // Number of successfully processed blocks + private final AtomicLong successfulBlocks = new AtomicLong(0); + // Number of blocks containing logs + private final AtomicLong blocksWithLogs = new AtomicLong(0); + // Number of failed blocks processed + private final AtomicLong errorCount = new AtomicLong(0); + // Total number of bloom writes + private final AtomicLong totalBloomWrites = new AtomicLong(0); + + private DBInterface transactionRetDb; + private DBInterface sectionBloomDb; + + private static class SectionRange { + + final long start; + final long end; + final long sectionId; + + SectionRange(long start, long end, long sectionId) { + this.start = start; + this.end = end; + this.sectionId = sectionId; + } + + @Override + public String toString() { + return String.format("Section %d: [%d-%d]", sectionId, start, end); + } + } + + @Override + public Integer call() { + if (help) { + logger.info("Displaying backfill-bloom help"); + spec.commandLine().usage(System.out); + return 0; + } + + try { + // Validate parameters + if (!validateParameters()) { + return 1; + } + + // Initialize database connections + if (!initializeDatabase()) { + return 1; + } + + // Determine end block if not specified + long latestSolidityBlockNumber; + try { + latestSolidityBlockNumber = getLatestSolidityBlockNumber(); + } catch (Exception e) { + printError(e, "Failed to read latest solidified block number"); + return 1; + } + if (latestSolidityBlockNumber < 0) { + printError("Latest solidified block number does not exist"); + return 1; + } + if (endBlock == 0) { + endBlock = latestSolidityBlockNumber; + } else if (endBlock > latestSolidityBlockNumber) { + printInfo("End block %d is larger than latest solidified block num %d; using %d instead.", + endBlock, latestSolidityBlockNumber, latestSolidityBlockNumber); + endBlock = latestSolidityBlockNumber; + } + + long minBlockNumber; + try { + minBlockNumber = getMinBlockNumber(); + } catch (Exception e) { + printError(e, "Failed to determine the first transaction result block"); + return 1; + } + if (minBlockNumber < 0) { + printError("Transaction result database does not contain any non-zero block"); + return 1; + } + if (startBlock == 0) { + startBlock = minBlockNumber; + } else if (startBlock < minBlockNumber) { + printInfo("Start block %d is earlier than the first available transaction result block %d; " + + "using %d instead.", startBlock, minBlockNumber, minBlockNumber); + startBlock = minBlockNumber; + } + + // Validate block range + if (endBlock < startBlock) { + printError("End block %d must be greater than or equal to start block %d", + endBlock, startBlock); + return 1; + } + + long totalBlocks = endBlock - startBlock + 1; + printInfo("Starting SectionBloom backfill for block number %d to %d (%d blocks)", + startBlock, endBlock, totalBlocks); + + // Process blocks with progress bar + long startTime = System.currentTimeMillis(); + int result = processBlocks(startTime); + long duration = (System.currentTimeMillis() - startTime) / 1000; + + // Print summary + printSummary(duration); + + return result; + + } catch (Exception e) { + printError(e, "Backfill failed"); + return 1; + } finally { + DbTool.close(); + } + } + + private boolean validateParameters() { + if (startBlock < 0) { + printError("Start block must be >= zero, it is %d", startBlock); + return false; + } + if (endBlock < 0) { + printError("End block must be > zero, it is %d", endBlock); + return false; + } + + if (maxConcurrency <= 0 || maxConcurrency > 128) { + printError("Max concurrency %d must be between 1 and 128", maxConcurrency); + return false; + } + + File dbDir = new File(databaseDirectory); + if (!dbDir.exists() || !dbDir.isDirectory()) { + printError("Database directory does not exist or is not a directory"); + return false; + } + if (!isDatabaseDirectory(dbDir, PROPERTIES_DB_NAME)) { + printError("Required database '%s' does not exist", PROPERTIES_DB_NAME); + return false; + } + if (!isDatabaseDirectory(dbDir, TRANSACTION_RET_DB_NAME)) { + printError("Required database '%s' does not exist", TRANSACTION_RET_DB_NAME); + return false; + } + + return true; + } + + private boolean isDatabaseDirectory(File databaseRoot, String databaseName) { + return new File(databaseRoot, databaseName).isDirectory(); + } + + private boolean initializeDatabase() { + try { + // Open both DBs here, single-threaded, before any worker thread starts. DbTool.getDB + // caches handles in a ConcurrentMap but its check-then-open is not atomic, so two + // threads opening the same LevelDB dir concurrently would hit the exclusive-lock error. + // Keep these handles for all worker threads instead of opening the same DB again. + transactionRetDb = DbTool.getDB(databaseDirectory, TRANSACTION_RET_DB_NAME); + sectionBloomDb = DbTool.getDB(databaseDirectory, SECTION_BLOOM_DB_NAME); + + printInfo("Database connections initialized successfully"); + return true; + } catch (Exception e) { + printError(e, "Failed to initialize database connections"); + return false; + } + } + + private long getLatestSolidityBlockNumber() throws IOException, RocksDBException { + DBInterface propertiesDb = DbTool.getDB(databaseDirectory, PROPERTIES_DB_NAME); + byte[] latestBlockKey = LATEST_SOLIDIFIED_BLOCK_NUM.getBytes(StandardCharsets.UTF_8); + byte[] latestBlockBytes = propertiesDb.get(latestBlockKey); + if (latestBlockBytes != null) { + return ByteArray.toLong(latestBlockBytes); + } + return -1; + } + + private long getMinBlockNumber() throws IOException { + try (DBIterator iterator = transactionRetDb.iterator()) { + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + return ByteArray.toLong(iterator.getKey()); + } + } + return -1; + } + + private int processBlocks(long startTime) { + long totalBlocks = endBlock - startBlock + 1; + // Calculate the section range to be processed + List sectionRanges = calculateSectionRanges(startBlock, endBlock); + + maxConcurrency = StrictMath.min(maxConcurrency, sectionRanges.size()); + ExecutorService executor = + ExecutorServiceManager.newFixedThreadPool("backfill-bloom", maxConcurrency); + List> futures = new ArrayList<>(); + + try (ProgressBar pb = new ProgressBar("Backfill section-bloom", totalBlocks)) { + printInfo("Processing %d sections with %d threads", sectionRanges.size(), maxConcurrency); + // Submit all section tasks to the thread pool + for (SectionRange range : sectionRanges) { + final long finalSectionStart = range.start; + final long finalSectionEnd = range.end; + + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + processSection(finalSectionStart, finalSectionEnd, totalBlocks, startTime, pb); + } catch (Exception e) { + printError(e, "Error processing section %d to %d", + finalSectionStart, finalSectionEnd); + } + }, executor); + + futures.add(future); + } + + // Wait for all tasks to complete + CompletableFuture allTasks = CompletableFuture.allOf(futures.toArray( + new CompletableFuture[0])); + + try { + allTasks.get(); + printInfo("All %d batch tasks completed", futures.size()); + } catch (Exception e) { + printError(e, "Error waiting for backfill tasks to complete"); + return 1; + } + + } catch (Exception e) { + printError(e, "Error in progress tracking"); + return 1; + } finally { + ExecutorServiceManager.shutdownAndAwaitTermination(executor, "backfill-bloom"); + } + + return errorCount.get() > 0 ? 1 : 0; + } + + /** + * Calculate the section range to be processed to ensure each thread processes a complete section. + * For example, startBlock=1000 and endBlock=4000 will generate: + * - SectionRange 0: [1000-2047] + * - SectionRange 1: [2048-4000] + */ + private List calculateSectionRanges(long startBlock, long endBlock) { + List ranges = new ArrayList<>(); + + long currentBlock = startBlock; + while (currentBlock <= endBlock) { + // Calculate the section to which the current block belongs + long sectionId = currentBlock / BLOCKS_PER_SECTION; + + // Calculate the boundaries of this section + long sectionStart = sectionId * BLOCKS_PER_SECTION; + long sectionEnd = sectionStart + BLOCKS_PER_SECTION - 1; + + // Adjust to the actual range that needs to be processed + long rangeStart = StrictMath.max(currentBlock, sectionStart); + long rangeEnd = StrictMath.min(endBlock, sectionEnd); + + ranges.add(new SectionRange(rangeStart, rangeEnd, sectionId)); + currentBlock = sectionEnd + 1; + } + + return ranges; + } + + private void processSection(long sectionStart, long sectionEnd, long totalBlocks, long startTime, + ProgressBar pb) { + for (long blockNum = sectionStart; blockNum <= sectionEnd; blockNum++) { + try { + processBlock(blockNum, transactionRetDb, sectionBloomDb); + successfulBlocks.incrementAndGet(); + } catch (Exception e) { + printError(e, "Error processing block %d", blockNum); + errorCount.incrementAndGet(); + } finally { + long processed = processedBlocks.incrementAndGet(); + logProgress(processed, totalBlocks, startTime); + pb.step(); + } + } + } + + private void logProgress(long processed, long totalBlocks, long startTime) { + if (processed % PROGRESS_LOG_INTERVAL != 0) { + return; + } + + long elapsedMillis = + StrictMath.max(System.currentTimeMillis() - startTime, 1L); + double progress = (double) processed / totalBlocks * 100; + double blocksPerSecond = (double) processed * 1000 / elapsedMillis; + long remainingSeconds = (long) ((totalBlocks - processed) / blocksPerSecond); + logger.info( + "Backfill progress: {}/{} blocks ({}%), elapsed={}, rate={} blocks/s, remaining={}", + processed, totalBlocks, String.format(Locale.ROOT, "%.2f", progress), + formatDuration(elapsedMillis / 1000), + String.format(Locale.ROOT, "%.2f", blocksPerSecond), + formatDuration(remainingSeconds)); + } + + private String formatDuration(long totalSeconds) { + long hours = totalSeconds / 3600; + long minutes = totalSeconds % 3600 / 60; + long seconds = totalSeconds % 60; + return String.format(Locale.ROOT, "%02d:%02d:%02d", hours, minutes, seconds); + } + + private void processBlock(long blockNum, DBInterface transactionRetDb, DBInterface sectionBloomDb) + throws BadItemException, EventBloomException { + + // Get transaction info for this block + byte[] blockKey = ByteArray.fromLong(blockNum); + byte[] transactionRetData = transactionRetDb.get(blockKey); + + if (transactionRetData == null) { + return; + } + + TransactionRetCapsule transactionRetCapsule = new TransactionRetCapsule(transactionRetData); + + // Create bloom filter for this block using the same logic as SectionBloomStore + Bloom blockBloom = Bloom.createBloom(transactionRetCapsule); + + if (blockBloom != null) { + // Extract bit positions from bloom filter + List bitList = extractBitPositions(blockBloom); + + if (!CollectionUtils.isEmpty(bitList)) { + // Write to section bloom store using the same logic as SectionBloomStore.write + writeSectionBloom(blockNum, bitList, sectionBloomDb); + blocksWithLogs.incrementAndGet(); + } + } + } + + private List extractBitPositions(Bloom blockBloom) { + List bitList = new ArrayList<>(); + BitSet bs = BitSet.valueOf(blockBloom.getData()); + for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i + 1)) { + // operate on index i here + if (i == Integer.MAX_VALUE) { + break; // or (i+1) would overflow + } + bitList.add(i); + } + return bitList; + } + + private void writeSectionBloom(long blockNum, List bitList, DBInterface sectionBloomDb) + throws EventBloomException { + + int section = (int) (blockNum / BLOCKS_PER_SECTION); + int blockNumOffset = (int) (blockNum % BLOCKS_PER_SECTION); + + for (int bitIndex : bitList) { + // Get existing BitSet from database + BitSet bitSet = getSectionBloomBitSet(section, bitIndex, sectionBloomDb); + if (Objects.isNull(bitSet)) { + bitSet = new BitSet(BLOCKS_PER_SECTION); + } + + // Update the bit for this block + bitSet.set(blockNumOffset); + + // Put back into database + putSectionBloomBitSet(section, bitIndex, bitSet, sectionBloomDb); + totalBloomWrites.incrementAndGet(); + } + } + + private long combineKey(int section, int bitIndex) { + return section * 1_000_000L + bitIndex; + } + + private BitSet getSectionBloomBitSet(int section, int bitIndex, DBInterface sectionBloomDb) + throws EventBloomException { + long keyLong = combineKey(section, bitIndex); + byte[] key = Long.toHexString(keyLong).getBytes(); + byte[] data = sectionBloomDb.get(key); + + if (data == null) { + return null; + } + + try { + byte[] decompressedData = ByteUtil.decompress(data); + return BitSet.valueOf(decompressedData); + } catch (Exception e) { + throw new EventBloomException("decompress byte failed"); + } + } + + private void putSectionBloomBitSet(int section, int bitIndex, BitSet bitSet, + DBInterface sectionBloomDb) + throws EventBloomException { + long keyLong = combineKey(section, bitIndex); + byte[] key = Long.toHexString(keyLong).getBytes(); + + try { + byte[] compressedData = ByteUtil.compress(bitSet.toByteArray()); + sectionBloomDb.put(key, compressedData); + } catch (Exception e) { + throw new EventBloomException("compress byte failed"); + } + } + + private void printSummary(long duration) { + spec.commandLine().getOut().println(); + printInfo("=== Backfill Summary ==="); + + printInfo("Total blocks scanned: %d", processedBlocks.get()); + printInfo("Successfully processed: %d", successfulBlocks.get()); + printInfo("Blocks with logs: %d", blocksWithLogs.get()); + printInfo("Errors encountered: %d", errorCount.get()); + printInfo("Duration: %d seconds", duration); + + // Success rate statistics + if (processedBlocks.get() > 0) { + double successRate = (double) successfulBlocks.get() / processedBlocks.get() * 100; + double logRate = (double) blocksWithLogs.get() / processedBlocks.get() * 100; + printInfo("Success rate: %.2f%% (%d/%d)", + successRate, successfulBlocks.get(), processedBlocks.get()); + printInfo("Blocks with logs rate: %.2f%% (%d/%d)", + logRate, blocksWithLogs.get(), processedBlocks.get()); + } + + // Performance statistics + printInfo("Total bloom writes: %d", totalBloomWrites.get()); + printInfo("Max concurrency used: %d threads", maxConcurrency); + printInfo("Section-based processing: No locks needed"); + + if (duration > 0) { + printInfo("Scanning rate: %.2f blocks/second", (double) processedBlocks.get() / duration); + printInfo("Processing rate: %.2f blocks/second", (double) successfulBlocks.get() / duration); + if (totalBloomWrites.get() > 0) { + printInfo("Bloom write rate: %.2f writes/second", + (double) totalBloomWrites.get() / duration); + } + } + + // Result judgment + if (errorCount.get() == 0) { + printInfo("✓ Backfill completed successfully!"); + } else { + printWarning("⚠ Backfill completed with %d errors.", errorCount.get()); + } + } + + private void printInfo(String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.info(message); + spec.commandLine().getOut().println(message); + } + + private void printWarning(String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.warn(message); + spec.commandLine().getOut().println(message); + } + + private void printError(String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.error(message); + spec.commandLine().getErr().println(message); + } + + private void printError(Throwable cause, String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.error(message, cause); + spec.commandLine().getErr().println(message); + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java new file mode 100644 index 00000000000..e48105128be --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java @@ -0,0 +1,741 @@ +package org.tron.plugins; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.AdditionalMatchers.aryEq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; +import org.slf4j.LoggerFactory; +import org.tron.common.utils.ByteArray; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; +import picocli.CommandLine; + +public class DbBackfillBloomTest { + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private String databaseDirectory; + private CommandLine cli; + private ByteArrayOutputStream outputStream; + private ByteArrayOutputStream errorStream; + private PrintStream originalOut; + private PrintStream originalErr; + private MockedStatic dbToolMock; + + @Before + public void setUp() throws IOException { + // Create temporary database directory + databaseDirectory = temporaryFolder.newFolder().toString(); + assertTrue(new File(databaseDirectory, "properties").mkdir()); + assertTrue(new File(databaseDirectory, "transactionRetStore").mkdir()); + + // Create the command line interface using Toolkit as the root command + cli = new CommandLine(new Toolkit()); + + // Capture output streams + outputStream = new ByteArrayOutputStream(); + errorStream = new ByteArrayOutputStream(); + originalOut = System.out; + originalErr = System.err; + System.setOut(new PrintStream(outputStream)); + System.setErr(new PrintStream(errorStream)); + + // Mock DbTool static methods + dbToolMock = mockStatic(DbTool.class); + } + + private DBIterator mockMinBlock(DBInterface transactionRetDb, long blockNumber) { + DBIterator iterator = mock(DBIterator.class); + when(transactionRetDb.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(true); + when(iterator.getKey()).thenReturn(ByteArray.fromLong(blockNumber)); + return iterator; + } + + @After + public void tearDown() { + // Restore original streams + System.setOut(originalOut); + System.setErr(originalErr); + + // Close static mock + if (dbToolMock != null) { + dbToolMock.close(); + } + } + + @Test + public void testHelp() { + String[] args = new String[] { "db", "backfill-bloom", "-h" }; + assertEquals(0, cli.execute(args)); + assertTrue(outputStream.toString().contains( + "The same block range can be safely rerun after interruption.")); + } + + @Test + public void testValidParametersWithMockedDatabase() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + // Mock DbTool.getDB calls + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock latest block number + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + mockMinBlock(transactionRetDb, 1L); + + // Mock empty transaction data (no transactions to process) + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-e", "200", + "-c", "2" + }; + + assertEquals(0, cli.execute(args)); + } + + @Test + public void testSummaryReportsScannedBlocks() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + mockMinBlock(transactionRetDb, 1L); + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + // Capture the command's output writer (summary is printed via + // spec.commandLine().getOut()). Follow the repo pattern: set the writer on a + // fresh root CommandLine, which picocli propagates to the subcommand on execute. + StringWriter out = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + + // Blocks 100..200 inclusive = 101 blocks scanned. + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-e", "200", + "-c", "2" + }; + + assertEquals(0, cmd.execute(args)); + + // Guards against the regression where processedBlocks was never incremented, + // which made the whole summary report 0 scanned blocks and skip the rates. + String output = out.toString(); + assertTrue(output.contains("Total blocks scanned: 101")); + assertTrue(output.contains("Success rate: 100.00%")); + } + + @Test + public void testLogsProgressEveryTenThousandBlocksWithoutWritingToCommandOutput() + throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(10_000L)); + mockMinBlock(transactionRetDb, 1L); + when(transactionRetDb.get(any(byte[].class))).thenReturn(null); + + StringWriter out = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + + Logger progressLogger = (Logger) LoggerFactory.getLogger("backfill-bloom"); + ListAppender appender = new ListAppender<>(); + appender.start(); + progressLogger.addAppender(appender); + try { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "10000", + "-c", "1" + }; + + assertEquals(0, cmd.execute(args)); + + long progressLogCount = appender.list.stream() + .map(ILoggingEvent::getFormattedMessage) + .filter(message -> message.startsWith( + "Backfill progress: 10000/10000 blocks (100.00%)")) + .count(); + assertEquals(1L, progressLogCount); + assertFalse(out.toString().contains("Backfill progress:")); + } finally { + progressLogger.detachAppender(appender); + appender.stop(); + } + } + + @Test + public void testInvalidStartBlock() { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "-1" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testInvalidConcurrency() { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-c", "0" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testInvalidConcurrencyTooHigh() { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-c", "200" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testNonExistentDatabaseDirectory() { + String nonExistentDir = databaseDirectory + File.separator + UUID.randomUUID(); + String[] args = new String[] { + "db", "backfill-bloom", + "-d", nonExistentDir, + "-s", "100" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testMissingPropertiesDatabaseIsRejectedBeforeOpeningDatabases() { + File databaseRoot = temporaryFolder.getRoot(); + File transactionRetDirectory = new File(databaseRoot, "only-transaction-ret"); + assertTrue(transactionRetDirectory.mkdir()); + assertTrue(new File(transactionRetDirectory, "transactionRetStore").mkdir()); + + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", transactionRetDirectory.toString(), + "-e", "100" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Required database 'properties' does not exist")); + dbToolMock.verify(() -> DbTool.close()); + dbToolMock.verifyNoMoreInteractions(); + } + + @Test + public void testMissingTransactionRetDatabaseIsRejectedBeforeOpeningDatabases() { + File databaseRoot = temporaryFolder.getRoot(); + File propertiesDirectory = new File(databaseRoot, "only-properties"); + assertTrue(propertiesDirectory.mkdir()); + assertTrue(new File(propertiesDirectory, "properties").mkdir()); + + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", propertiesDirectory.toString(), + "-e", "100" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains( + "Required database 'transactionRetStore' does not exist")); + dbToolMock.verify(() -> DbTool.close()); + dbToolMock.verifyNoMoreInteractions(); + } + + @Test + public void testEndBlockLessThanStartBlock() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + mockMinBlock(transactionRetDb, 1L); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "200", + "-e", "100" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testDatabaseInitializationFailure() { + // Mock DbTool to throw exception + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenThrow(new RuntimeException("Database connection failed")); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testAutoDetectEndBlock() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock latest block number + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(5000L)); + mockMinBlock(transactionRetDb, 1L); + + // Mock empty transaction data + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100" + // No end block specified - should auto-detect + }; + + assertEquals(0, cli.execute(args)); + } + + @Test + public void testAdjustsStartBlockToMinimumNonZeroTransactionResultBlock() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + DBIterator iterator = mock(DBIterator.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(12L)); + when(transactionRetDb.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(true); + when(iterator.getKey()).thenReturn(ByteArray.fromLong(10L)); + when(transactionRetDb.get(any(byte[].class))).thenReturn(null); + + StringWriter out = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "5", + "-e", "12" + }; + + assertEquals(0, cmd.execute(args)); + assertTrue(out.toString().contains( + "Start block 5 is earlier than the first available transaction result block 10")); + assertTrue(out.toString().contains( + "Starting SectionBloom backfill for block number 10 to 12 (3 blocks)")); + verify(iterator).seek(aryEq(ByteArray.fromLong(1))); + verify(iterator).close(); + } + + @Test + public void testFailsWhenTransactionResultStoreIsEmpty() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + DBIterator iterator = mock(DBIterator.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(6L)); + when(transactionRetDb.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(false); + + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "5", + "-e", "6" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains( + "Transaction result database does not contain any non-zero block")); + assertFalse(out.toString().contains("Starting SectionBloom backfill")); + verify(iterator).seek(aryEq(ByteArray.fromLong(1))); + verify(iterator).close(); + } + + @Test + public void testFailsWhenMinimumBlockCannotBeRead() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(6L)); + when(transactionRetDb.iterator()) + .thenThrow(new RuntimeException("iterator failed")); + + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "6" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains( + "Failed to determine the first transaction result block")); + } + + @Test + public void testProcessingErrorIsWrittenToStderr() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1L)); + mockMinBlock(transactionRetDb, 1L); + when(transactionRetDb.get(any(byte[].class))) + .thenThrow(new RuntimeException("read failed")); + + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "1" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Error processing block 1")); + assertFalse(out.toString().contains("Error processing block 1")); + } + + @Test + public void testMissingLatestSolidityBlockNumber() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock null latest block number (failed to get) + when(propertiesDb.get(any(byte[].class))) + .thenReturn(null); + + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100" + // No end block specified - should fail to auto-detect + }; + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Latest solidified block number does not exist")); + verify(propertiesDb).get(aryEq( + "LATEST_SOLIDIFIED_BLOCK_NUM".getBytes(StandardCharsets.UTF_8))); + } + + @Test + public void testFailsWhenLatestSolidityBlockNumberCannotBeRead() { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + when(propertiesDb.get(any(byte[].class))) + .thenThrow(new RuntimeException("read failed")); + + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "6" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Failed to read latest solidified block number")); + assertFalse(out.toString().contains("using -1 instead")); + } + + @Test + public void testDefaultParameters() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock latest block number + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + + // Mock empty transaction data + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + // Test with default database directory + String[] args = new String[] { + "db", "backfill-bloom", + "-s", "100", + "-e", "200" + }; + + // This should fail because default directory doesn't exist + assertEquals(1, cli.execute(args)); + } +}