From 8bdc11fce8ba274894eb61b0de61e5e7c2ad8a26 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Mon, 8 Jun 2026 00:43:19 +0800 Subject: [PATCH 1/5] fill section bloom using exist db --- plugins/README.md | 30 ++ plugins/build.gradle | 1 + .../main/java/common/org/tron/plugins/Db.java | 1 + .../org/tron/plugins/DbBackfillBloom.java | 463 ++++++++++++++++++ .../org/tron/plugins/DbBackfillBloomTest.java | 374 ++++++++++++++ 5 files changed, 869 insertions(+) create mode 100644 plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java diff --git a/plugins/README.md b/plugins/README.md index f14e070c01a..452cad873dd 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -148,6 +148,36 @@ NOTE: large db may GC overhead limit exceeded. - `--db`: db name. - `-h | --help`: provide the help info +## DB Backfill-Bloom + +DB backfill bloom provides the ability to backfill SectionBloom data for historical blocks to enable `eth_getLogs` address/topics filtering. This is useful when `isJsonRpcFilterEnabled` was disabled during block processing and later enabled, causing historical blocks to lack SectionBloom data. + +### Available parameters: + +- `-d | --database-directory`: Specify the database directory path, it is used to open the database to get the transaction log and write the SectionBloom data back, default: output-directory/database. +- `-s | --start-block`: Specify the start block number for backfill (required). +- `-e | --end-block`: Specify the end block number for backfill (optional, default: latest block). +- `-c | --max-concurrency`: Specify the maximum concurrency for processing, default: 8. +- `-h | --help`: Provide the help info. + +### Examples: + +```shell script +# full command + java -jar Toolkit.jar db backfill-bloom [-h] -s= [-e=] [-d=] [-c=] +# examples + java -jar Toolkit.jar db backfill-bloom -s 1000000 -e 2000000 #1. backfill blocks 1000000 to 2000000 + java -jar Toolkit.jar db backfill-bloom -s 1000000 -d /path/to/database #2. specify custom database directory + java -jar Toolkit.jar db backfill-bloom -s 1000000 -c 8 #3. use higher concurrency (8 threads) +``` + +### Backfill speed + +The time required to process different block ranges varies. It is recommended to increase `--max-concurrency` appropriately to speed up the backfill process. + +- 0-10000000: It's done almost instantly because there are no logs inside. +- 10000000-70000000: Takes about 3-4 hours/10,000,000 blocks with `--max-concurrency` set to 32. + ## Keystore Keystore provides commands for managing account keystore files (Web3 Secret Storage format). diff --git a/plugins/build.gradle b/plugins/build.gradle index 09a13a19b1b..15ab420eaa9 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -70,6 +70,7 @@ dependencies { implementation 'io.github.tronprotocol:leveldb:1.18.2' } implementation project(":protocol") + implementation project(":chainbase") } check.dependsOn 'lint' diff --git a/plugins/src/main/java/common/org/tron/plugins/Db.java b/plugins/src/main/java/common/org/tron/plugins/Db.java index 84654dca934..0918d939dc5 100644 --- a/plugins/src/main/java/common/org/tron/plugins/Db.java +++ b/plugins/src/main/java/common/org/tron/plugins/Db.java @@ -12,6 +12,7 @@ DbConvert.class, DbLite.class, DbCopy.class, + DbBackfillBloom.class, DbRoot.class }, commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n" diff --git a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java new file mode 100644 index 00000000000..4dc630526e8 --- /dev/null +++ b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java @@ -0,0 +1,463 @@ +package org.tron.plugins; + +import java.io.File; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import me.tongfei.progressbar.ProgressBar; +import org.apache.commons.collections4.CollectionUtils; +import org.tron.common.bloom.Bloom; +import org.tron.common.es.ExecutorServiceManager; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.ByteUtil; +import org.tron.core.capsule.TransactionRetCapsule; +import org.tron.core.exception.BadItemException; +import org.tron.core.exception.EventBloomException; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; +import picocli.CommandLine; + +@Slf4j(topic = "backfill-bloom") +@CommandLine.Command(name = "backfill-bloom", + description = "Backfill SectionBloom for historical blocks to enable eth_getLogs filtering.", + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "1:Internal error: exception occurred, please check toolkit.log"}) +public class DbBackfillBloom implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"--database-directory", "-d"}, + defaultValue = "output-directory/database", + description = "Database directory path. Default: ${DEFAULT-VALUE}", order = 1) + private String databaseDirectory; + + @CommandLine.Option(names = {"--start-block", "-s"}, required = true, + description = "Start block number for backfill", order = 2) + private long startBlock; + + @CommandLine.Option(names = {"--end-block", "-e"}, + description = "End block number for backfill (default: latest block)", order = 3) + private Long endBlock; + + private static final int BLOCKS_PER_SECTION = 2048; + + @CommandLine.Option(names = {"--max-concurrency", "-c"}, defaultValue = "8", + description = "Maximum concurrency for processing. Default: ${DEFAULT-VALUE}", + order = 5) + private int maxConcurrency; + + @CommandLine.Option(names = {"--help", "-h"}, help = true, + description = "Display help message", order = 7) + private boolean help; + + // Statistics + // Number of blocks traversed (including failed ones) + private final AtomicLong processedBlocks = new AtomicLong(0); + // Number of successfully processed blocks + private final AtomicLong successfulBlocks = new AtomicLong(0); + // Number of blocks containing logs + private final AtomicLong blocksWithLogs = new AtomicLong(0); + // Number of failed blocks processed + private final AtomicLong errorCount = new AtomicLong(0); + // Total number of bloom writes + private final AtomicLong totalBloomWrites = new AtomicLong(0); + + private static class SectionRange { + + final long start; + final long end; + final long sectionId; + + SectionRange(long start, long end, long sectionId) { + this.start = start; + this.end = end; + this.sectionId = sectionId; + } + + @Override + public String toString() { + return String.format("Section %d: [%d-%d]", sectionId, start, end); + } + } + + @Override + public Integer call() { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + + try { + // Validate parameters + if (!validateParameters()) { + return 1; + } + + // Initialize database connections + if (!initializeDatabase()) { + return 1; + } + + // Determine end block if not specified + if (endBlock == null) { + endBlock = getLatestBlockNumber(); + if (endBlock == null || endBlock == 0) { + spec.commandLine().getErr().println("Failed to determine latest block number"); + return 1; + } + } + + // Validate block range + if (endBlock < startBlock) { + spec.commandLine().getErr().println("End block must be >= start block"); + return 1; + } + + long totalBlocks = endBlock - startBlock + 1; + spec.commandLine().getOut().printf( + "Starting SectionBloom backfill for blocks %d to %d (%d blocks)%n", + startBlock, endBlock, totalBlocks); + + // Process blocks with progress bar + long startTime = System.currentTimeMillis(); + int result = processBlocks(); + long duration = (System.currentTimeMillis() - startTime) / 1000; + + // Print summary + printSummary(duration); + + return result; + + } catch (Exception e) { + logger.error("Backfill failed", e); + spec.commandLine().getErr().println("Backfill failed: " + e.getMessage()); + return 1; + } finally { + DbTool.close(); + } + } + + private boolean validateParameters() { + if (startBlock < 0) { + spec.commandLine().getErr().println("Start block must be >= 0"); + return false; + } + + if (maxConcurrency <= 0 || maxConcurrency > 128) { + spec.commandLine().getErr().println("Max concurrency must be between 1 and 128"); + return false; + } + + File dbDir = new File(databaseDirectory); + if (!dbDir.exists() || !dbDir.isDirectory()) { + spec.commandLine().getErr().println("Database directory does not exist: " + + databaseDirectory); + return false; + } + + return true; + } + + private boolean initializeDatabase() { + try { + // Open both DBs here, single-threaded, before any worker thread starts. DbTool.getDB + // caches handles in a ConcurrentMap but its check-then-open is not atomic, so two + // threads opening the same LevelDB dir concurrently would hit the exclusive-lock error. + // Pre-warming the cache means the worker threads in processSection() only ever read the + // cached handles. Do NOT remove this pre-warm. + DbTool.getDB(databaseDirectory, "transactionRetStore"); + DbTool.getDB(databaseDirectory, "section-bloom"); + + spec.commandLine().getOut().println("Database connections initialized successfully"); + return true; + } catch (Exception e) { + logger.error("Failed to initialize database connections: {}", e.getMessage()); + spec.commandLine().getErr().println("Failed to initialize database connections: " + + e.getMessage()); + return false; + } + } + + private Long getLatestBlockNumber() { + try { + DBInterface propertiesDb = DbTool.getDB(databaseDirectory, "properties"); + byte[] latestBlockKey = "latest_block_header_number".getBytes(); + byte[] latestBlockBytes = propertiesDb.get(latestBlockKey); + + if (latestBlockBytes != null) { + return ByteArray.toLong(latestBlockBytes); + } + return null; + } catch (Exception e) { + logger.error("Failed to get latest block number: {}", e.getMessage()); + return null; + } + } + + private int processBlocks() { + long totalBlocks = endBlock - startBlock + 1; + // Calculate the section range to be processed + List sectionRanges = calculateSectionRanges(startBlock, endBlock); + + maxConcurrency = Math.min(maxConcurrency, sectionRanges.size()); + ExecutorService executor = + ExecutorServiceManager.newFixedThreadPool("backfill-bloom", maxConcurrency); + List> futures = new ArrayList<>(); + + try (ProgressBar pb = new ProgressBar("Scanning blocks for SectionBloom backfill", + totalBlocks)) { + spec.commandLine().getOut().printf("Processing %d sections with %d threads\n", + sectionRanges.size(), maxConcurrency); + // Submit all section tasks to the thread pool + for (SectionRange range : sectionRanges) { + final long finalSectionStart = range.start; + final long finalSectionEnd = range.end; + + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + processSection(finalSectionStart, finalSectionEnd, pb); + } catch (Exception e) { + spec.commandLine().getOut().printf("Error processing section %d to %d, %s\n", + finalSectionStart, finalSectionEnd, e); + } + }, executor); + + futures.add(future); + } + + // Wait for all tasks to complete + CompletableFuture allTasks = CompletableFuture.allOf(futures.toArray( + new CompletableFuture[0])); + + try { + allTasks.get(); + spec.commandLine().getOut().printf("All %d batch tasks completed\n", futures.size()); + } catch (Exception e) { + spec.commandLine().getOut().printf("Error waiting for tasks to complete: %s\n", + e.getMessage()); + return 1; + } + + } catch (Exception e) { + spec.commandLine().getOut().printf("Error in progress tracking %s\n", e); + return 1; + } finally { + ExecutorServiceManager.shutdownAndAwaitTermination(executor, "backfill-bloom"); + } + + return errorCount.get() > 0 ? 1 : 0; + } + + /** + * Calculate the section range to be processed to ensure each thread processes a complete section. + * For example, startBlock=1000 and endBlock=4000 will generate: + * - SectionRange 0: [1000-2047] + * - SectionRange 1: [2048-4000] + */ + private List calculateSectionRanges(long startBlock, long endBlock) { + List ranges = new ArrayList<>(); + + long currentBlock = startBlock; + while (currentBlock <= endBlock) { + // Calculate the section to which the current block belongs + long sectionId = currentBlock / BLOCKS_PER_SECTION; + + // Calculate the boundaries of this section + long sectionStart = sectionId * BLOCKS_PER_SECTION; + long sectionEnd = sectionStart + BLOCKS_PER_SECTION - 1; + + // Adjust to the actual range that needs to be processed + long rangeStart = Math.max(currentBlock, sectionStart); + long rangeEnd = Math.min(endBlock, sectionEnd); + + ranges.add(new SectionRange(rangeStart, rangeEnd, sectionId)); + currentBlock = sectionEnd + 1; + } + + return ranges; + } + + private void processSection(long sectionStart, long sectionEnd, ProgressBar pb) { + long sectionId = sectionStart / BLOCKS_PER_SECTION; + try { + // Cache hit only: these handles were pre-warmed single-threaded in initializeDatabase() + // to avoid a concurrent-open race on the same LevelDB dir. + DBInterface transactionRetDb = DbTool.getDB(databaseDirectory, "transactionRetStore"); + DBInterface sectionBloomDb = DbTool.getDB(databaseDirectory, "section-bloom"); + + for (long blockNum = sectionStart; blockNum <= sectionEnd; blockNum++) { + try { + processBlock(blockNum, transactionRetDb, sectionBloomDb); + successfulBlocks.incrementAndGet(); + } catch (Exception e) { + spec.commandLine().getOut().printf("Error processing block %d, %s\n", blockNum, e); + errorCount.incrementAndGet(); + } finally { + processedBlocks.incrementAndGet(); + pb.step(); + } + } + } catch (Exception e) { + spec.commandLine().getOut().printf("Error in section %d processing: %s\n", sectionId, e); + throw new RuntimeException(e); + } + } + + private void processBlock(long blockNum, DBInterface transactionRetDb, DBInterface sectionBloomDb) + throws BadItemException, EventBloomException { + + // Get transaction info for this block + byte[] blockKey = ByteArray.fromLong(blockNum); + byte[] transactionRetData = transactionRetDb.get(blockKey); + + if (transactionRetData == null) { + return; + } + + try { + TransactionRetCapsule transactionRetCapsule = new TransactionRetCapsule(transactionRetData); + + // Create bloom filter for this block using the same logic as SectionBloomStore + Bloom blockBloom = Bloom.createBloom(transactionRetCapsule); + + if (blockBloom != null) { + // Extract bit positions from bloom filter + List bitList = extractBitPositions(blockBloom); + + if (!CollectionUtils.isEmpty(bitList)) { + // Write to section bloom store using the same logic as SectionBloomStore.write + writeSectionBloom(blockNum, bitList, sectionBloomDb); + blocksWithLogs.incrementAndGet(); + } + } + } catch (Exception e) { + spec.commandLine().getOut().printf("Error processing block %d: %s\n", blockNum, + e.getMessage()); + throw e; + } + } + + private List extractBitPositions(Bloom blockBloom) { + List bitList = new ArrayList<>(); + BitSet bs = BitSet.valueOf(blockBloom.getData()); + for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i + 1)) { + // operate on index i here + if (i == Integer.MAX_VALUE) { + break; // or (i+1) would overflow + } + bitList.add(i); + } + return bitList; + } + + private void writeSectionBloom(long blockNum, List bitList, DBInterface sectionBloomDb) + throws EventBloomException { + + int section = (int) (blockNum / BLOCKS_PER_SECTION); + int blockNumOffset = (int) (blockNum % BLOCKS_PER_SECTION); + + for (int bitIndex : bitList) { + // Get existing BitSet from database + BitSet bitSet = getSectionBloomBitSet(section, bitIndex, sectionBloomDb); + if (Objects.isNull(bitSet)) { + bitSet = new BitSet(BLOCKS_PER_SECTION); + } + + // Update the bit for this block + bitSet.set(blockNumOffset); + + // Put back into database + putSectionBloomBitSet(section, bitIndex, bitSet, sectionBloomDb); + totalBloomWrites.incrementAndGet(); + } + } + + private long combineKey(int section, int bitIndex) { + return section * 1_000_000L + bitIndex; + } + + private BitSet getSectionBloomBitSet(int section, int bitIndex, DBInterface sectionBloomDb) + throws EventBloomException { + long keyLong = combineKey(section, bitIndex); + byte[] key = Long.toHexString(keyLong).getBytes(); + byte[] data = sectionBloomDb.get(key); + + if (data == null) { + return null; + } + + try { + byte[] decompressedData = ByteUtil.decompress(data); + return BitSet.valueOf(decompressedData); + } catch (Exception e) { + throw new EventBloomException("decompress byte failed"); + } + } + + private void putSectionBloomBitSet(int section, int bitIndex, BitSet bitSet, + DBInterface sectionBloomDb) + throws EventBloomException { + long keyLong = combineKey(section, bitIndex); + byte[] key = Long.toHexString(keyLong).getBytes(); + + try { + byte[] compressedData = ByteUtil.compress(bitSet.toByteArray()); + sectionBloomDb.put(key, compressedData); + } catch (Exception e) { + throw new EventBloomException("compress byte failed"); + } + } + + private void printSummary(long duration) { + spec.commandLine().getOut().println("\n=== Backfill Summary ==="); + + spec.commandLine().getOut().printf("Total blocks scanned: %d%n", processedBlocks.get()); + spec.commandLine().getOut().printf("Successfully processed: %d%n", successfulBlocks.get()); + spec.commandLine().getOut().printf("Blocks with logs: %d%n", blocksWithLogs.get()); + spec.commandLine().getOut().printf("Errors encountered: %d%n", errorCount.get()); + spec.commandLine().getOut().printf("Duration: %d seconds%n", duration); + + // Success rate statistics + if (processedBlocks.get() > 0) { + double successRate = (double) successfulBlocks.get() / processedBlocks.get() * 100; + double logRate = (double) blocksWithLogs.get() / processedBlocks.get() * 100; + spec.commandLine().getOut().printf("Success rate: %.2f%% (%d/%d)%n", + successRate, successfulBlocks.get(), processedBlocks.get()); + spec.commandLine().getOut().printf("Blocks with logs rate: %.2f%% (%d/%d)%n", + logRate, blocksWithLogs.get(), processedBlocks.get()); + } + + // Performance statistics + spec.commandLine().getOut().printf("Total bloom writes: %d%n", totalBloomWrites.get()); + spec.commandLine().getOut().printf("Max concurrency used: %d threads%n", maxConcurrency); + spec.commandLine().getOut().printf("Section-based processing: No locks needed%n"); + + if (duration > 0) { + spec.commandLine().getOut().printf("Scanning rate: %.2f blocks/second%n", + (double) processedBlocks.get() / duration); + spec.commandLine().getOut().printf("Processing rate: %.2f blocks/second%n", + (double) successfulBlocks.get() / duration); + if (totalBloomWrites.get() > 0) { + spec.commandLine().getOut().printf("Bloom write rate: %.2f writes/second%n", + (double) totalBloomWrites.get() / duration); + } + } + + // Result judgment + if (errorCount.get() == 0) { + spec.commandLine().getOut().println("✓ Backfill completed successfully!"); + } else { + spec.commandLine().getOut().printf("⚠ Backfill completed with %d errors.%n", + errorCount.get()); + } + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java new file mode 100644 index 00000000000..9198c10e924 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java @@ -0,0 +1,374 @@ +package org.tron.plugins; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.UUID; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; +import org.tron.common.utils.ByteArray; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; +import picocli.CommandLine; + +public class DbBackfillBloomTest { + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private String databaseDirectory; + private CommandLine cli; + private ByteArrayOutputStream outputStream; + private ByteArrayOutputStream errorStream; + private PrintStream originalOut; + private PrintStream originalErr; + private MockedStatic dbToolMock; + + @Before + public void setUp() throws IOException { + // Create temporary database directory + databaseDirectory = temporaryFolder.newFolder().toString(); + + // Create the command line interface using Toolkit as the root command + cli = new CommandLine(new Toolkit()); + + // Capture output streams + outputStream = new ByteArrayOutputStream(); + errorStream = new ByteArrayOutputStream(); + originalOut = System.out; + originalErr = System.err; + System.setOut(new PrintStream(outputStream)); + System.setErr(new PrintStream(errorStream)); + + // Mock DbTool static methods + dbToolMock = mockStatic(DbTool.class); + } + + @After + public void tearDown() { + // Restore original streams + System.setOut(originalOut); + System.setErr(originalErr); + + // Close static mock + if (dbToolMock != null) { + dbToolMock.close(); + } + } + + @Test + public void testHelp() { + String[] args = new String[] { "db", "backfill-bloom", "-h" }; + assertEquals(0, cli.execute(args)); + } + + @Test + public void testValidParametersWithMockedDatabase() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + // Mock DbTool.getDB calls + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock latest block number + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + + // Mock empty transaction data (no transactions to process) + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-e", "200", + "-c", "2" + }; + + assertEquals(0, cli.execute(args)); + } + + @Test + public void testSummaryReportsScannedBlocks() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + // Capture the command's output writer (summary is printed via + // spec.commandLine().getOut()). Follow the repo pattern: set the writer on a + // fresh root CommandLine, which picocli propagates to the subcommand on execute. + StringWriter out = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + + // Blocks 100..200 inclusive = 101 blocks scanned. + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-e", "200", + "-c", "2" + }; + + assertEquals(0, cmd.execute(args)); + + // Guards against the regression where processedBlocks was never incremented, + // which made the whole summary report 0 scanned blocks and skip the rates. + String output = out.toString(); + assertTrue(output.contains("Total blocks scanned: 101")); + assertTrue(output.contains("Success rate: 100.00%")); + } + + @Test + public void testInvalidStartBlock() { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "-1" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testInvalidConcurrency() { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-c", "0" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testInvalidConcurrencyTooHigh() { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100", + "-c", "200" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testNonExistentDatabaseDirectory() { + String nonExistentDir = databaseDirectory + File.separator + UUID.randomUUID(); + String[] args = new String[] { + "db", "backfill-bloom", + "-d", nonExistentDir, + "-s", "100" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testEndBlockLessThanStartBlock() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + default: + return mock(DBInterface.class); + } + }); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "200", + "-e", "100" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testDatabaseInitializationFailure() { + // Mock DbTool to throw exception + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenThrow(new RuntimeException("Database connection failed")); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100" + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testAutoDetectEndBlock() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock latest block number + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(5000L)); + + // Mock empty transaction data + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100" + // No end block specified - should auto-detect + }; + + assertEquals(0, cli.execute(args)); + } + + @Test + public void testFailedToGetLatestBlockNumber() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock null latest block number (failed to get) + when(propertiesDb.get(any(byte[].class))) + .thenReturn(null); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "100" + // No end block specified - should fail to auto-detect + }; + assertEquals(1, cli.execute(args)); + } + + @Test + public void testDefaultParameters() throws Exception { + // Mock database interfaces + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + // Mock latest block number + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + + // Mock empty transaction data + when(transactionRetDb.get(any(byte[].class))) + .thenReturn(null); + + // Test with default database directory + String[] args = new String[] { + "db", "backfill-bloom", + "-s", "100", + "-e", "200" + }; + + // This should fail because default directory doesn't exist + assertEquals(1, cli.execute(args)); + } +} From 633958b46dd54ec38bf029a917a8f7d7a3e8751b Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Thu, 23 Jul 2026 19:16:23 +0800 Subject: [PATCH 2/5] add getMinNonZeroBlockNumber and getLatestSolidityBlockNumber for DbBackfillBloom --- plugins/build.gradle | 2 +- .../org/tron/plugins/DbBackfillBloom.java | 204 ++++++++++-------- .../org/tron/plugins/DbBackfillBloomTest.java | 133 +++++++++++- 3 files changed, 249 insertions(+), 90 deletions(-) diff --git a/plugins/build.gradle b/plugins/build.gradle index 15ab420eaa9..1530c6bcf06 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -150,7 +150,7 @@ def binaryRelease(taskName, jarName, mainClass) { // (and so partial / parallel builds cannot run binaryRelease before the // dependency jars exist). dependsOn (project(':protocol').jar, project(':platform').jar, - project(':crypto').jar, project(':common').jar) // explicit_dependency + project(':crypto').jar, project(':common').jar, project(':chainbase').jar) // explicit_dependency from { configurations.runtimeClasspath.collect { // https://docs.gradle.org/current/userguide/upgrading_version_6.html#changes_6.3 it.isDirectory() ? it : zipTree(it) diff --git a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java index 4dc630526e8..f8de0670ca5 100644 --- a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; @@ -20,6 +21,7 @@ import org.tron.core.exception.BadItemException; import org.tron.core.exception.EventBloomException; import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @@ -40,14 +42,15 @@ public class DbBackfillBloom implements Callable { description = "Database directory path. Default: ${DEFAULT-VALUE}", order = 1) private String databaseDirectory; - @CommandLine.Option(names = {"--start-block", "-s"}, required = true, - description = "Start block number for backfill", order = 2) + @CommandLine.Option(names = {"--start-block", "-s"}, + description = "Start block number for backfill(default: earliest block)", order = 2) private long startBlock; @CommandLine.Option(names = {"--end-block", "-e"}, description = "End block number for backfill (default: latest block)", order = 3) private Long endBlock; + // sames as SectionBloomStore.BLOCK_PER_SECTION private static final int BLOCKS_PER_SECTION = 2048; @CommandLine.Option(names = {"--max-concurrency", "-c"}, defaultValue = "8", @@ -71,6 +74,9 @@ public class DbBackfillBloom implements Callable { // Total number of bloom writes private final AtomicLong totalBloomWrites = new AtomicLong(0); + private DBInterface transactionRetDb; + private DBInterface sectionBloomDb; + private static class SectionRange { final long start; @@ -92,6 +98,7 @@ public String toString() { @Override public Integer call() { if (help) { + logger.info("Displaying backfill-bloom help"); spec.commandLine().usage(System.out); return 0; } @@ -109,22 +116,31 @@ public Integer call() { // Determine end block if not specified if (endBlock == null) { - endBlock = getLatestBlockNumber(); + endBlock = getLatestSolidityBlockNumber(); if (endBlock == null || endBlock == 0) { - spec.commandLine().getErr().println("Failed to determine latest block number"); + printError("Failed to determine latest block number"); return 1; } } + Long minNonZeroBlockNumber = getMinNonZeroBlockNumber(); + if (minNonZeroBlockNumber != null && startBlock < minNonZeroBlockNumber) { + printInfo( + "Start block %d is earlier than the first available transaction result block %d; " + + "using %d instead.", + startBlock, minNonZeroBlockNumber, minNonZeroBlockNumber); + startBlock = minNonZeroBlockNumber; + } + // Validate block range if (endBlock < startBlock) { - spec.commandLine().getErr().println("End block must be >= start block"); + printError("End block %d must be greater than or equal to start block %d", + endBlock, startBlock); return 1; } long totalBlocks = endBlock - startBlock + 1; - spec.commandLine().getOut().printf( - "Starting SectionBloom backfill for blocks %d to %d (%d blocks)%n", + printInfo("Starting SectionBloom backfill for blocks %d to %d (%d blocks)", startBlock, endBlock, totalBlocks); // Process blocks with progress bar @@ -138,8 +154,7 @@ public Integer call() { return result; } catch (Exception e) { - logger.error("Backfill failed", e); - spec.commandLine().getErr().println("Backfill failed: " + e.getMessage()); + printError(e, "Backfill failed"); return 1; } finally { DbTool.close(); @@ -148,19 +163,18 @@ public Integer call() { private boolean validateParameters() { if (startBlock < 0) { - spec.commandLine().getErr().println("Start block must be >= 0"); + printError("Start block %d must be greater than or equal to zero", startBlock); return false; } if (maxConcurrency <= 0 || maxConcurrency > 128) { - spec.commandLine().getErr().println("Max concurrency must be between 1 and 128"); + printError("Max concurrency %d must be between 1 and 128", maxConcurrency); return false; } File dbDir = new File(databaseDirectory); if (!dbDir.exists() || !dbDir.isDirectory()) { - spec.commandLine().getErr().println("Database directory does not exist: " - + databaseDirectory); + printError("Database directory does not exist or is not a directory"); return false; } @@ -172,25 +186,22 @@ private boolean initializeDatabase() { // Open both DBs here, single-threaded, before any worker thread starts. DbTool.getDB // caches handles in a ConcurrentMap but its check-then-open is not atomic, so two // threads opening the same LevelDB dir concurrently would hit the exclusive-lock error. - // Pre-warming the cache means the worker threads in processSection() only ever read the - // cached handles. Do NOT remove this pre-warm. - DbTool.getDB(databaseDirectory, "transactionRetStore"); - DbTool.getDB(databaseDirectory, "section-bloom"); + // Keep these handles for all worker threads instead of opening the same DB again. + transactionRetDb = DbTool.getDB(databaseDirectory, "transactionRetStore"); + sectionBloomDb = DbTool.getDB(databaseDirectory, "section-bloom"); - spec.commandLine().getOut().println("Database connections initialized successfully"); + printInfo("Database connections initialized successfully"); return true; } catch (Exception e) { - logger.error("Failed to initialize database connections: {}", e.getMessage()); - spec.commandLine().getErr().println("Failed to initialize database connections: " - + e.getMessage()); + printError(e, "Failed to initialize database connections"); return false; } } - private Long getLatestBlockNumber() { + private Long getLatestSolidityBlockNumber() { try { DBInterface propertiesDb = DbTool.getDB(databaseDirectory, "properties"); - byte[] latestBlockKey = "latest_block_header_number".getBytes(); + byte[] latestBlockKey = "LATEST_SOLIDIFIED_BLOCK_NUM".getBytes(); byte[] latestBlockBytes = propertiesDb.get(latestBlockKey); if (latestBlockBytes != null) { @@ -198,11 +209,25 @@ private Long getLatestBlockNumber() { } return null; } catch (Exception e) { - logger.error("Failed to get latest block number: {}", e.getMessage()); + logger.error("Failed to get latest block number", e); return null; } } + private Long getMinNonZeroBlockNumber() { + try { + try (DBIterator iterator = transactionRetDb.iterator()) { + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + return ByteArray.toLong(iterator.getKey()); + } + } + } catch (Exception e) { + logger.error("Failed to get minimum non-zero block number", e); + } + return null; + } + private int processBlocks() { long totalBlocks = endBlock - startBlock + 1; // Calculate the section range to be processed @@ -215,8 +240,7 @@ private int processBlocks() { try (ProgressBar pb = new ProgressBar("Scanning blocks for SectionBloom backfill", totalBlocks)) { - spec.commandLine().getOut().printf("Processing %d sections with %d threads\n", - sectionRanges.size(), maxConcurrency); + printInfo("Processing %d sections with %d threads", sectionRanges.size(), maxConcurrency); // Submit all section tasks to the thread pool for (SectionRange range : sectionRanges) { final long finalSectionStart = range.start; @@ -226,8 +250,8 @@ private int processBlocks() { try { processSection(finalSectionStart, finalSectionEnd, pb); } catch (Exception e) { - spec.commandLine().getOut().printf("Error processing section %d to %d, %s\n", - finalSectionStart, finalSectionEnd, e); + printError(e, "Error processing section %d to %d", + finalSectionStart, finalSectionEnd); } }, executor); @@ -240,15 +264,14 @@ private int processBlocks() { try { allTasks.get(); - spec.commandLine().getOut().printf("All %d batch tasks completed\n", futures.size()); + printInfo("All %d batch tasks completed", futures.size()); } catch (Exception e) { - spec.commandLine().getOut().printf("Error waiting for tasks to complete: %s\n", - e.getMessage()); + printError(e, "Error waiting for backfill tasks to complete"); return 1; } } catch (Exception e) { - spec.commandLine().getOut().printf("Error in progress tracking %s\n", e); + printError(e, "Error in progress tracking"); return 1; } finally { ExecutorServiceManager.shutdownAndAwaitTermination(executor, "backfill-bloom"); @@ -287,28 +310,17 @@ private List calculateSectionRanges(long startBlock, long endBlock } private void processSection(long sectionStart, long sectionEnd, ProgressBar pb) { - long sectionId = sectionStart / BLOCKS_PER_SECTION; - try { - // Cache hit only: these handles were pre-warmed single-threaded in initializeDatabase() - // to avoid a concurrent-open race on the same LevelDB dir. - DBInterface transactionRetDb = DbTool.getDB(databaseDirectory, "transactionRetStore"); - DBInterface sectionBloomDb = DbTool.getDB(databaseDirectory, "section-bloom"); - - for (long blockNum = sectionStart; blockNum <= sectionEnd; blockNum++) { - try { - processBlock(blockNum, transactionRetDb, sectionBloomDb); - successfulBlocks.incrementAndGet(); - } catch (Exception e) { - spec.commandLine().getOut().printf("Error processing block %d, %s\n", blockNum, e); - errorCount.incrementAndGet(); - } finally { - processedBlocks.incrementAndGet(); - pb.step(); - } + for (long blockNum = sectionStart; blockNum <= sectionEnd; blockNum++) { + try { + processBlock(blockNum, transactionRetDb, sectionBloomDb); + successfulBlocks.incrementAndGet(); + } catch (Exception e) { + printError(e, "Error processing block %d", blockNum); + errorCount.incrementAndGet(); + } finally { + processedBlocks.incrementAndGet(); + pb.step(); } - } catch (Exception e) { - spec.commandLine().getOut().printf("Error in section %d processing: %s\n", sectionId, e); - throw new RuntimeException(e); } } @@ -323,26 +335,20 @@ private void processBlock(long blockNum, DBInterface transactionRetDb, DBInterfa return; } - try { - TransactionRetCapsule transactionRetCapsule = new TransactionRetCapsule(transactionRetData); + TransactionRetCapsule transactionRetCapsule = new TransactionRetCapsule(transactionRetData); - // Create bloom filter for this block using the same logic as SectionBloomStore - Bloom blockBloom = Bloom.createBloom(transactionRetCapsule); + // Create bloom filter for this block using the same logic as SectionBloomStore + Bloom blockBloom = Bloom.createBloom(transactionRetCapsule); - if (blockBloom != null) { - // Extract bit positions from bloom filter - List bitList = extractBitPositions(blockBloom); + if (blockBloom != null) { + // Extract bit positions from bloom filter + List bitList = extractBitPositions(blockBloom); - if (!CollectionUtils.isEmpty(bitList)) { - // Write to section bloom store using the same logic as SectionBloomStore.write - writeSectionBloom(blockNum, bitList, sectionBloomDb); - blocksWithLogs.incrementAndGet(); - } + if (!CollectionUtils.isEmpty(bitList)) { + // Write to section bloom store using the same logic as SectionBloomStore.write + writeSectionBloom(blockNum, bitList, sectionBloomDb); + blocksWithLogs.incrementAndGet(); } - } catch (Exception e) { - spec.commandLine().getOut().printf("Error processing block %d: %s\n", blockNum, - e.getMessage()); - throw e; } } @@ -418,46 +424,68 @@ private void putSectionBloomBitSet(int section, int bitIndex, BitSet bitSet, } private void printSummary(long duration) { - spec.commandLine().getOut().println("\n=== Backfill Summary ==="); + spec.commandLine().getOut().println(); + printInfo("=== Backfill Summary ==="); - spec.commandLine().getOut().printf("Total blocks scanned: %d%n", processedBlocks.get()); - spec.commandLine().getOut().printf("Successfully processed: %d%n", successfulBlocks.get()); - spec.commandLine().getOut().printf("Blocks with logs: %d%n", blocksWithLogs.get()); - spec.commandLine().getOut().printf("Errors encountered: %d%n", errorCount.get()); - spec.commandLine().getOut().printf("Duration: %d seconds%n", duration); + printInfo("Total blocks scanned: %d", processedBlocks.get()); + printInfo("Successfully processed: %d", successfulBlocks.get()); + printInfo("Blocks with logs: %d", blocksWithLogs.get()); + printInfo("Errors encountered: %d", errorCount.get()); + printInfo("Duration: %d seconds", duration); // Success rate statistics if (processedBlocks.get() > 0) { double successRate = (double) successfulBlocks.get() / processedBlocks.get() * 100; double logRate = (double) blocksWithLogs.get() / processedBlocks.get() * 100; - spec.commandLine().getOut().printf("Success rate: %.2f%% (%d/%d)%n", + printInfo("Success rate: %.2f%% (%d/%d)", successRate, successfulBlocks.get(), processedBlocks.get()); - spec.commandLine().getOut().printf("Blocks with logs rate: %.2f%% (%d/%d)%n", + printInfo("Blocks with logs rate: %.2f%% (%d/%d)", logRate, blocksWithLogs.get(), processedBlocks.get()); } // Performance statistics - spec.commandLine().getOut().printf("Total bloom writes: %d%n", totalBloomWrites.get()); - spec.commandLine().getOut().printf("Max concurrency used: %d threads%n", maxConcurrency); - spec.commandLine().getOut().printf("Section-based processing: No locks needed%n"); + printInfo("Total bloom writes: %d", totalBloomWrites.get()); + printInfo("Max concurrency used: %d threads", maxConcurrency); + printInfo("Section-based processing: No locks needed"); if (duration > 0) { - spec.commandLine().getOut().printf("Scanning rate: %.2f blocks/second%n", - (double) processedBlocks.get() / duration); - spec.commandLine().getOut().printf("Processing rate: %.2f blocks/second%n", - (double) successfulBlocks.get() / duration); + printInfo("Scanning rate: %.2f blocks/second", (double) processedBlocks.get() / duration); + printInfo("Processing rate: %.2f blocks/second", (double) successfulBlocks.get() / duration); if (totalBloomWrites.get() > 0) { - spec.commandLine().getOut().printf("Bloom write rate: %.2f writes/second%n", + printInfo("Bloom write rate: %.2f writes/second", (double) totalBloomWrites.get() / duration); } } // Result judgment if (errorCount.get() == 0) { - spec.commandLine().getOut().println("✓ Backfill completed successfully!"); + printInfo("✓ Backfill completed successfully!"); } else { - spec.commandLine().getOut().printf("⚠ Backfill completed with %d errors.%n", - errorCount.get()); + printWarning("⚠ Backfill completed with %d errors.", errorCount.get()); } } + + private void printInfo(String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.info(message); + spec.commandLine().getOut().println(message); + } + + private void printWarning(String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.warn(message); + spec.commandLine().getOut().println(message); + } + + private void printError(String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.error(message); + spec.commandLine().getErr().println(message); + } + + private void printError(Throwable cause, String format, Object... args) { + String message = String.format(Locale.ROOT, format, args); + logger.error(message, cause); + spec.commandLine().getErr().println(message); + } } diff --git a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java index 9198c10e924..89156d63600 100644 --- a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java @@ -1,11 +1,14 @@ package org.tron.plugins; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.AdditionalMatchers.aryEq; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; @@ -23,6 +26,7 @@ import org.mockito.MockedStatic; import org.tron.common.utils.ByteArray; import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @@ -297,7 +301,134 @@ public void testAutoDetectEndBlock() throws Exception { } @Test - public void testFailedToGetLatestBlockNumber() throws Exception { + public void testAdjustsStartBlockToMinimumNonZeroTransactionResultBlock() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBIterator iterator = mock(DBIterator.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + default: + return mock(DBInterface.class); + } + }); + + when(transactionRetDb.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(true); + when(iterator.getKey()).thenReturn(ByteArray.fromLong(10L)); + when(transactionRetDb.get(any(byte[].class))).thenReturn(null); + + StringWriter out = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "0", + "-e", "12" + }; + + assertEquals(0, cmd.execute(args)); + assertTrue(out.toString().contains( + "Start block 0 is earlier than the first available transaction result block 10")); + assertTrue(out.toString().contains( + "Starting SectionBloom backfill for blocks 10 to 12 (3 blocks)")); + verify(iterator).seek(aryEq(ByteArray.fromLong(1))); + verify(iterator).close(); + } + + @Test + public void testKeepsStartBlockWhenTransactionResultStoreIsEmpty() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBIterator iterator = mock(DBIterator.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + default: + return mock(DBInterface.class); + } + }); + + when(transactionRetDb.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(false); + when(transactionRetDb.get(any(byte[].class))).thenReturn(null); + + StringWriter out = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "5", + "-e", "6" + }; + + assertEquals(0, cmd.execute(args)); + assertTrue(out.toString().contains( + "Starting SectionBloom backfill for blocks 5 to 6 (2 blocks)")); + verify(iterator).seek(aryEq(ByteArray.fromLong(1))); + verify(iterator).close(); + } + + @Test + public void testProcessingErrorIsWrittenToStderr() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBIterator iterator = mock(DBIterator.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + default: + return mock(DBInterface.class); + } + }); + + when(transactionRetDb.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(false); + when(transactionRetDb.get(any(byte[].class))) + .thenThrow(new RuntimeException("read failed")); + + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "1" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Error processing block 1")); + assertFalse(out.toString().contains("Error processing block 1")); + } + + @Test + public void testFailedToGetLatestSolidityBlockNumber() throws Exception { // Mock database interfaces DBInterface transactionRetDb = mock(DBInterface.class); DBInterface sectionBloomDb = mock(DBInterface.class); From d2021a4b4c13daddd83e4f4e59bb831cfe5a11a2 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Fri, 24 Jul 2026 18:35:44 +0800 Subject: [PATCH 3/5] add more check for parameters --- plugins/README.md | 7 +- .../org/tron/plugins/DbBackfillBloom.java | 118 +++++++---- .../org/tron/plugins/DbBackfillBloomTest.java | 196 ++++++++++++++++-- 3 files changed, 266 insertions(+), 55 deletions(-) diff --git a/plugins/README.md b/plugins/README.md index 452cad873dd..83a13613e73 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -152,11 +152,14 @@ NOTE: large db may GC overhead limit exceeded. DB backfill bloom provides the ability to backfill SectionBloom data for historical blocks to enable `eth_getLogs` address/topics filtering. This is useful when `isJsonRpcFilterEnabled` was disabled during block processing and later enabled, causing historical blocks to lack SectionBloom data. +The source database must contain a non-empty `transactionRetStore`. Ensure +`storage.transHistory.switch` was enabled while the historical blocks were processed. + ### Available parameters: - `-d | --database-directory`: Specify the database directory path, it is used to open the database to get the transaction log and write the SectionBloom data back, default: output-directory/database. -- `-s | --start-block`: Specify the start block number for backfill (required). -- `-e | --end-block`: Specify the end block number for backfill (optional, default: latest block). +- `-s | --start-block`: Specify the start block number for backfill. +- `-e | --end-block`: Specify the end block number for backfill (optional, default: latest solidity block). - `-c | --max-concurrency`: Specify the maximum concurrency for processing, default: 8. - `-h | --help`: Provide the help info. diff --git a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java index f8de0670ca5..6659249131b 100644 --- a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java @@ -1,6 +1,8 @@ package org.tron.plugins; import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.BitSet; import java.util.List; @@ -13,6 +15,7 @@ import lombok.extern.slf4j.Slf4j; import me.tongfei.progressbar.ProgressBar; import org.apache.commons.collections4.CollectionUtils; +import org.rocksdb.RocksDBException; import org.tron.common.bloom.Bloom; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.utils.ByteArray; @@ -47,11 +50,15 @@ public class DbBackfillBloom implements Callable { private long startBlock; @CommandLine.Option(names = {"--end-block", "-e"}, - description = "End block number for backfill (default: latest block)", order = 3) - private Long endBlock; + description = "End block number for backfill (default: latest solidity block)", order = 3) + private long endBlock; // sames as SectionBloomStore.BLOCK_PER_SECTION private static final int BLOCKS_PER_SECTION = 2048; + private static final String PROPERTIES_DB_NAME = "properties"; + private static final String TRANSACTION_RET_DB_NAME = "transactionRetStore"; + private static final String SECTION_BLOOM_DB_NAME = "section-bloom"; + private static final String LATEST_SOLIDIFIED_BLOCK_NUM = "LATEST_SOLIDIFIED_BLOCK_NUM"; @CommandLine.Option(names = {"--max-concurrency", "-c"}, defaultValue = "8", description = "Maximum concurrency for processing. Default: ${DEFAULT-VALUE}", @@ -115,21 +122,44 @@ public Integer call() { } // Determine end block if not specified - if (endBlock == null) { - endBlock = getLatestSolidityBlockNumber(); - if (endBlock == null || endBlock == 0) { - printError("Failed to determine latest block number"); - return 1; - } + long latestSolidityBlockNumber; + try { + latestSolidityBlockNumber = getLatestSolidityBlockNumber(); + } catch (Exception e) { + printError(e, "Failed to read latest solidified block number"); + return 1; + } + if (latestSolidityBlockNumber < 0) { + printError("Latest solidified block number does not exist"); + return 1; + } + if (endBlock == 0) { + endBlock = latestSolidityBlockNumber; + } else if (endBlock > latestSolidityBlockNumber) { + printInfo("End block %d is larger than latest solidified block num %d; using %d instead.", + endBlock, latestSolidityBlockNumber, latestSolidityBlockNumber); + endBlock = latestSolidityBlockNumber; } - Long minNonZeroBlockNumber = getMinNonZeroBlockNumber(); - if (minNonZeroBlockNumber != null && startBlock < minNonZeroBlockNumber) { + long minBlockNumber; + try { + minBlockNumber = getMinBlockNumber(); + } catch (Exception e) { + printError(e, "Failed to determine the first transaction result block"); + return 1; + } + if (minBlockNumber < 0) { + printError("Transaction result database does not contain any non-zero block"); + return 1; + } + if (startBlock == 0) { + startBlock = minBlockNumber; + } else if (startBlock < minBlockNumber) { printInfo( "Start block %d is earlier than the first available transaction result block %d; " + "using %d instead.", - startBlock, minNonZeroBlockNumber, minNonZeroBlockNumber); - startBlock = minNonZeroBlockNumber; + startBlock, minBlockNumber, minBlockNumber); + startBlock = minBlockNumber; } // Validate block range @@ -163,7 +193,11 @@ public Integer call() { private boolean validateParameters() { if (startBlock < 0) { - printError("Start block %d must be greater than or equal to zero", startBlock); + printError("Start block must be >= zero, it is %d", startBlock); + return false; + } + if (endBlock < 0) { + printError("End block must be > zero, it is %d", endBlock); return false; } @@ -177,18 +211,30 @@ private boolean validateParameters() { printError("Database directory does not exist or is not a directory"); return false; } + if (!isDatabaseDirectory(dbDir, PROPERTIES_DB_NAME)) { + printError("Required database '%s' does not exist", PROPERTIES_DB_NAME); + return false; + } + if (!isDatabaseDirectory(dbDir, TRANSACTION_RET_DB_NAME)) { + printError("Required database '%s' does not exist", TRANSACTION_RET_DB_NAME); + return false; + } return true; } + private boolean isDatabaseDirectory(File databaseRoot, String databaseName) { + return new File(databaseRoot, databaseName).isDirectory(); + } + private boolean initializeDatabase() { try { // Open both DBs here, single-threaded, before any worker thread starts. DbTool.getDB // caches handles in a ConcurrentMap but its check-then-open is not atomic, so two // threads opening the same LevelDB dir concurrently would hit the exclusive-lock error. // Keep these handles for all worker threads instead of opening the same DB again. - transactionRetDb = DbTool.getDB(databaseDirectory, "transactionRetStore"); - sectionBloomDb = DbTool.getDB(databaseDirectory, "section-bloom"); + transactionRetDb = DbTool.getDB(databaseDirectory, TRANSACTION_RET_DB_NAME); + sectionBloomDb = DbTool.getDB(databaseDirectory, SECTION_BLOOM_DB_NAME); printInfo("Database connections initialized successfully"); return true; @@ -198,34 +244,24 @@ private boolean initializeDatabase() { } } - private Long getLatestSolidityBlockNumber() { - try { - DBInterface propertiesDb = DbTool.getDB(databaseDirectory, "properties"); - byte[] latestBlockKey = "LATEST_SOLIDIFIED_BLOCK_NUM".getBytes(); - byte[] latestBlockBytes = propertiesDb.get(latestBlockKey); - - if (latestBlockBytes != null) { - return ByteArray.toLong(latestBlockBytes); - } - return null; - } catch (Exception e) { - logger.error("Failed to get latest block number", e); - return null; + private long getLatestSolidityBlockNumber() throws IOException, RocksDBException { + DBInterface propertiesDb = DbTool.getDB(databaseDirectory, PROPERTIES_DB_NAME); + byte[] latestBlockKey = LATEST_SOLIDIFIED_BLOCK_NUM.getBytes(StandardCharsets.UTF_8); + byte[] latestBlockBytes = propertiesDb.get(latestBlockKey); + if (latestBlockBytes != null) { + return ByteArray.toLong(latestBlockBytes); } + return -1; } - private Long getMinNonZeroBlockNumber() { - try { - try (DBIterator iterator = transactionRetDb.iterator()) { - iterator.seek(ByteArray.fromLong(1)); - if (iterator.hasNext()) { - return ByteArray.toLong(iterator.getKey()); - } + private long getMinBlockNumber() throws IOException { + try (DBIterator iterator = transactionRetDb.iterator()) { + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + return ByteArray.toLong(iterator.getKey()); } - } catch (Exception e) { - logger.error("Failed to get minimum non-zero block number", e); } - return null; + return -1; } private int processBlocks() { @@ -233,7 +269,7 @@ private int processBlocks() { // Calculate the section range to be processed List sectionRanges = calculateSectionRanges(startBlock, endBlock); - maxConcurrency = Math.min(maxConcurrency, sectionRanges.size()); + maxConcurrency = StrictMath.min(maxConcurrency, sectionRanges.size()); ExecutorService executor = ExecutorServiceManager.newFixedThreadPool("backfill-bloom", maxConcurrency); List> futures = new ArrayList<>(); @@ -299,8 +335,8 @@ private List calculateSectionRanges(long startBlock, long endBlock long sectionEnd = sectionStart + BLOCKS_PER_SECTION - 1; // Adjust to the actual range that needs to be processed - long rangeStart = Math.max(currentBlock, sectionStart); - long rangeEnd = Math.min(endBlock, sectionEnd); + long rangeStart = StrictMath.max(currentBlock, sectionStart); + long rangeEnd = StrictMath.min(endBlock, sectionEnd); ranges.add(new SectionRange(rangeStart, rangeEnd, sectionId)); currentBlock = sectionEnd + 1; diff --git a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java index 89156d63600..2c3ca18a669 100644 --- a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java @@ -17,6 +17,7 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; +import java.nio.charset.StandardCharsets; import java.util.UUID; import org.junit.After; import org.junit.Before; @@ -47,6 +48,8 @@ public class DbBackfillBloomTest { public void setUp() throws IOException { // Create temporary database directory databaseDirectory = temporaryFolder.newFolder().toString(); + assertTrue(new File(databaseDirectory, "properties").mkdir()); + assertTrue(new File(databaseDirectory, "transactionRetStore").mkdir()); // Create the command line interface using Toolkit as the root command cli = new CommandLine(new Toolkit()); @@ -63,6 +66,14 @@ public void setUp() throws IOException { dbToolMock = mockStatic(DbTool.class); } + private DBIterator mockMinBlock(DBInterface transactionRetDb, long blockNumber) { + DBIterator iterator = mock(DBIterator.class); + when(transactionRetDb.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(true); + when(iterator.getKey()).thenReturn(ByteArray.fromLong(blockNumber)); + return iterator; + } + @After public void tearDown() { // Restore original streams @@ -107,6 +118,7 @@ public void testValidParametersWithMockedDatabase() throws Exception { // Mock latest block number when(propertiesDb.get(any(byte[].class))) .thenReturn(ByteArray.fromLong(1000L)); + mockMinBlock(transactionRetDb, 1L); // Mock empty transaction data (no transactions to process) when(transactionRetDb.get(any(byte[].class))) @@ -147,6 +159,7 @@ public void testSummaryReportsScannedBlocks() throws Exception { when(propertiesDb.get(any(byte[].class))) .thenReturn(ByteArray.fromLong(1000L)); + mockMinBlock(transactionRetDb, 1L); when(transactionRetDb.get(any(byte[].class))) .thenReturn(null); @@ -218,11 +231,59 @@ public void testNonExistentDatabaseDirectory() { assertEquals(1, cli.execute(args)); } + @Test + public void testMissingPropertiesDatabaseIsRejectedBeforeOpeningDatabases() { + File databaseRoot = temporaryFolder.getRoot(); + File transactionRetDirectory = new File(databaseRoot, "only-transaction-ret"); + assertTrue(transactionRetDirectory.mkdir()); + assertTrue(new File(transactionRetDirectory, "transactionRetStore").mkdir()); + + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", transactionRetDirectory.toString(), + "-e", "100" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Required database 'properties' does not exist")); + dbToolMock.verify(() -> DbTool.close()); + dbToolMock.verifyNoMoreInteractions(); + } + + @Test + public void testMissingTransactionRetDatabaseIsRejectedBeforeOpeningDatabases() { + File databaseRoot = temporaryFolder.getRoot(); + File propertiesDirectory = new File(databaseRoot, "only-properties"); + assertTrue(propertiesDirectory.mkdir()); + assertTrue(new File(propertiesDirectory, "properties").mkdir()); + + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", propertiesDirectory.toString(), + "-e", "100" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains( + "Required database 'transactionRetStore' does not exist")); + dbToolMock.verify(() -> DbTool.close()); + dbToolMock.verifyNoMoreInteractions(); + } + @Test public void testEndBlockLessThanStartBlock() throws Exception { // Mock database interfaces DBInterface transactionRetDb = mock(DBInterface.class); DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) .thenAnswer(invocation -> { @@ -232,10 +293,15 @@ public void testEndBlockLessThanStartBlock() throws Exception { return transactionRetDb; case "section-bloom": return sectionBloomDb; + case "properties": + return propertiesDb; default: return mock(DBInterface.class); } }); + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1000L)); + mockMinBlock(transactionRetDb, 1L); String[] args = new String[] { "db", "backfill-bloom", @@ -285,6 +351,7 @@ public void testAutoDetectEndBlock() throws Exception { // Mock latest block number when(propertiesDb.get(any(byte[].class))) .thenReturn(ByteArray.fromLong(5000L)); + mockMinBlock(transactionRetDb, 1L); // Mock empty transaction data when(transactionRetDb.get(any(byte[].class))) @@ -304,6 +371,7 @@ public void testAutoDetectEndBlock() throws Exception { public void testAdjustsStartBlockToMinimumNonZeroTransactionResultBlock() throws Exception { DBInterface transactionRetDb = mock(DBInterface.class); DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); DBIterator iterator = mock(DBIterator.class); dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) @@ -314,11 +382,15 @@ public void testAdjustsStartBlockToMinimumNonZeroTransactionResultBlock() throws return transactionRetDb; case "section-bloom": return sectionBloomDb; + case "properties": + return propertiesDb; default: return mock(DBInterface.class); } }); + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(12L)); when(transactionRetDb.iterator()).thenReturn(iterator); when(iterator.hasNext()).thenReturn(true); when(iterator.getKey()).thenReturn(ByteArray.fromLong(10L)); @@ -331,13 +403,13 @@ public void testAdjustsStartBlockToMinimumNonZeroTransactionResultBlock() throws String[] args = new String[] { "db", "backfill-bloom", "-d", databaseDirectory, - "-s", "0", + "-s", "5", "-e", "12" }; assertEquals(0, cmd.execute(args)); assertTrue(out.toString().contains( - "Start block 0 is earlier than the first available transaction result block 10")); + "Start block 5 is earlier than the first available transaction result block 10")); assertTrue(out.toString().contains( "Starting SectionBloom backfill for blocks 10 to 12 (3 blocks)")); verify(iterator).seek(aryEq(ByteArray.fromLong(1))); @@ -345,9 +417,10 @@ public void testAdjustsStartBlockToMinimumNonZeroTransactionResultBlock() throws } @Test - public void testKeepsStartBlockWhenTransactionResultStoreIsEmpty() throws Exception { + public void testFailsWhenTransactionResultStoreIsEmpty() throws Exception { DBInterface transactionRetDb = mock(DBInterface.class); DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); DBIterator iterator = mock(DBIterator.class); dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) @@ -358,18 +431,23 @@ public void testKeepsStartBlockWhenTransactionResultStoreIsEmpty() throws Except return transactionRetDb; case "section-bloom": return sectionBloomDb; + case "properties": + return propertiesDb; default: return mock(DBInterface.class); } }); + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(6L)); when(transactionRetDb.iterator()).thenReturn(iterator); when(iterator.hasNext()).thenReturn(false); - when(transactionRetDb.get(any(byte[].class))).thenReturn(null); StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); CommandLine cmd = new CommandLine(new Toolkit()); cmd.setOut(new PrintWriter(out)); + cmd.setErr(new PrintWriter(err)); String[] args = new String[] { "db", "backfill-bloom", @@ -378,18 +456,61 @@ public void testKeepsStartBlockWhenTransactionResultStoreIsEmpty() throws Except "-e", "6" }; - assertEquals(0, cmd.execute(args)); - assertTrue(out.toString().contains( - "Starting SectionBloom backfill for blocks 5 to 6 (2 blocks)")); + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains( + "Transaction result database does not contain any non-zero block")); + assertFalse(out.toString().contains("Starting SectionBloom backfill")); verify(iterator).seek(aryEq(ByteArray.fromLong(1))); verify(iterator).close(); } + @Test + public void testFailsWhenMinimumBlockCannotBeRead() throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(6L)); + when(transactionRetDb.iterator()) + .thenThrow(new RuntimeException("iterator failed")); + + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "6" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains( + "Failed to determine the first transaction result block")); + } + @Test public void testProcessingErrorIsWrittenToStderr() throws Exception { DBInterface transactionRetDb = mock(DBInterface.class); DBInterface sectionBloomDb = mock(DBInterface.class); - DBIterator iterator = mock(DBIterator.class); + DBInterface propertiesDb = mock(DBInterface.class); dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) .thenAnswer(invocation -> { @@ -399,13 +520,16 @@ public void testProcessingErrorIsWrittenToStderr() throws Exception { return transactionRetDb; case "section-bloom": return sectionBloomDb; + case "properties": + return propertiesDb; default: return mock(DBInterface.class); } }); - when(transactionRetDb.iterator()).thenReturn(iterator); - when(iterator.hasNext()).thenReturn(false); + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(1L)); + mockMinBlock(transactionRetDb, 1L); when(transactionRetDb.get(any(byte[].class))) .thenThrow(new RuntimeException("read failed")); @@ -428,7 +552,7 @@ public void testProcessingErrorIsWrittenToStderr() throws Exception { } @Test - public void testFailedToGetLatestSolidityBlockNumber() throws Exception { + public void testMissingLatestSolidityBlockNumber() throws Exception { // Mock database interfaces DBInterface transactionRetDb = mock(DBInterface.class); DBInterface sectionBloomDb = mock(DBInterface.class); @@ -453,13 +577,61 @@ public void testFailedToGetLatestSolidityBlockNumber() throws Exception { when(propertiesDb.get(any(byte[].class))) .thenReturn(null); + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setErr(new PrintWriter(err)); + String[] args = new String[] { "db", "backfill-bloom", "-d", databaseDirectory, "-s", "100" // No end block specified - should fail to auto-detect }; - assertEquals(1, cli.execute(args)); + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Latest solidified block number does not exist")); + verify(propertiesDb).get(aryEq( + "LATEST_SOLIDIFIED_BLOCK_NUM".getBytes(StandardCharsets.UTF_8))); + } + + @Test + public void testFailsWhenLatestSolidityBlockNumberCannotBeRead() { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + when(propertiesDb.get(any(byte[].class))) + .thenThrow(new RuntimeException("read failed")); + + StringWriter out = new StringWriter(); + StringWriter err = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + cmd.setErr(new PrintWriter(err)); + + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "6" + }; + + assertEquals(1, cmd.execute(args)); + assertTrue(err.toString().contains("Failed to read latest solidified block number")); + assertFalse(out.toString().contains("using -1 instead")); } @Test From a0614b778ab00df6c25e7251fb3bc9ea8952ee1e Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Mon, 27 Jul 2026 11:14:38 +0800 Subject: [PATCH 4/5] add progress log --- plugins/README.md | 2 + .../org/tron/plugins/DbBackfillBloom.java | 62 ++++++++++++----- .../org/tron/plugins/DbBackfillBloomTest.java | 66 ++++++++++++++++++- 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/plugins/README.md b/plugins/README.md index 83a13613e73..c892fabd381 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -155,6 +155,8 @@ DB backfill bloom provides the ability to backfill SectionBloom data for histori The source database must contain a non-empty `transactionRetStore`. Ensure `storage.transHistory.switch` was enabled while the historical blocks were processed. +The backfill operation is idempotent. If it is interrupted, you can safely rerun the same block range. Existing SectionBloom bits are preserved and set again. Do not run multiple backfill processes concurrently or run the tool while another process is using the same database. + ### Available parameters: - `-d | --database-directory`: Specify the database directory path, it is used to open the database to get the transaction log and write the SectionBloom data back, default: output-directory/database. diff --git a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java index 6659249131b..8f6efcb7149 100644 --- a/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbBackfillBloom.java @@ -30,7 +30,10 @@ @Slf4j(topic = "backfill-bloom") @CommandLine.Command(name = "backfill-bloom", - description = "Backfill SectionBloom for historical blocks to enable eth_getLogs filtering.", + description = { + "Backfill SectionBloom for historical blocks to enable eth_getLogs filtering.", + "The same block range can be safely rerun after interruption." + }, exitCodeListHeading = "Exit Codes:%n", exitCodeList = { "0:Successful", @@ -46,22 +49,25 @@ public class DbBackfillBloom implements Callable { private String databaseDirectory; @CommandLine.Option(names = {"--start-block", "-s"}, - description = "Start block number for backfill(default: earliest block)", order = 2) + description = "Start block number for backfill. Default: earliest block", order = 2) private long startBlock; @CommandLine.Option(names = {"--end-block", "-e"}, - description = "End block number for backfill (default: latest solidity block)", order = 3) + description = "End block number for backfill, inclusive. Default: latest solidity block", + order = 3) private long endBlock; // sames as SectionBloomStore.BLOCK_PER_SECTION private static final int BLOCKS_PER_SECTION = 2048; + private static final long PROGRESS_LOG_INTERVAL = 10_000L; private static final String PROPERTIES_DB_NAME = "properties"; private static final String TRANSACTION_RET_DB_NAME = "transactionRetStore"; private static final String SECTION_BLOOM_DB_NAME = "section-bloom"; private static final String LATEST_SOLIDIFIED_BLOCK_NUM = "LATEST_SOLIDIFIED_BLOCK_NUM"; @CommandLine.Option(names = {"--max-concurrency", "-c"}, defaultValue = "8", - description = "Maximum concurrency for processing. Default: ${DEFAULT-VALUE}", + description = "Maximum concurrency for processing. Default: ${DEFAULT-VALUE}. For SATA SSD " + + "use 4–8; for NVMe SSD use 8–16; for HDD use 1–2.", order = 5) private int maxConcurrency; @@ -155,10 +161,8 @@ public Integer call() { if (startBlock == 0) { startBlock = minBlockNumber; } else if (startBlock < minBlockNumber) { - printInfo( - "Start block %d is earlier than the first available transaction result block %d; " - + "using %d instead.", - startBlock, minBlockNumber, minBlockNumber); + printInfo("Start block %d is earlier than the first available transaction result block %d; " + + "using %d instead.", startBlock, minBlockNumber, minBlockNumber); startBlock = minBlockNumber; } @@ -170,12 +174,12 @@ public Integer call() { } long totalBlocks = endBlock - startBlock + 1; - printInfo("Starting SectionBloom backfill for blocks %d to %d (%d blocks)", + printInfo("Starting SectionBloom backfill for block number %d to %d (%d blocks)", startBlock, endBlock, totalBlocks); // Process blocks with progress bar long startTime = System.currentTimeMillis(); - int result = processBlocks(); + int result = processBlocks(startTime); long duration = (System.currentTimeMillis() - startTime) / 1000; // Print summary @@ -264,7 +268,7 @@ private long getMinBlockNumber() throws IOException { return -1; } - private int processBlocks() { + private int processBlocks(long startTime) { long totalBlocks = endBlock - startBlock + 1; // Calculate the section range to be processed List sectionRanges = calculateSectionRanges(startBlock, endBlock); @@ -274,8 +278,7 @@ private int processBlocks() { ExecutorServiceManager.newFixedThreadPool("backfill-bloom", maxConcurrency); List> futures = new ArrayList<>(); - try (ProgressBar pb = new ProgressBar("Scanning blocks for SectionBloom backfill", - totalBlocks)) { + try (ProgressBar pb = new ProgressBar("Backfill section-bloom", totalBlocks)) { printInfo("Processing %d sections with %d threads", sectionRanges.size(), maxConcurrency); // Submit all section tasks to the thread pool for (SectionRange range : sectionRanges) { @@ -284,7 +287,7 @@ private int processBlocks() { CompletableFuture future = CompletableFuture.runAsync(() -> { try { - processSection(finalSectionStart, finalSectionEnd, pb); + processSection(finalSectionStart, finalSectionEnd, totalBlocks, startTime, pb); } catch (Exception e) { printError(e, "Error processing section %d to %d", finalSectionStart, finalSectionEnd); @@ -345,7 +348,8 @@ private List calculateSectionRanges(long startBlock, long endBlock return ranges; } - private void processSection(long sectionStart, long sectionEnd, ProgressBar pb) { + private void processSection(long sectionStart, long sectionEnd, long totalBlocks, long startTime, + ProgressBar pb) { for (long blockNum = sectionStart; blockNum <= sectionEnd; blockNum++) { try { processBlock(blockNum, transactionRetDb, sectionBloomDb); @@ -354,12 +358,38 @@ private void processSection(long sectionStart, long sectionEnd, ProgressBar pb) printError(e, "Error processing block %d", blockNum); errorCount.incrementAndGet(); } finally { - processedBlocks.incrementAndGet(); + long processed = processedBlocks.incrementAndGet(); + logProgress(processed, totalBlocks, startTime); pb.step(); } } } + private void logProgress(long processed, long totalBlocks, long startTime) { + if (processed % PROGRESS_LOG_INTERVAL != 0) { + return; + } + + long elapsedMillis = + StrictMath.max(System.currentTimeMillis() - startTime, 1L); + double progress = (double) processed / totalBlocks * 100; + double blocksPerSecond = (double) processed * 1000 / elapsedMillis; + long remainingSeconds = (long) ((totalBlocks - processed) / blocksPerSecond); + logger.info( + "Backfill progress: {}/{} blocks ({}%), elapsed={}, rate={} blocks/s, remaining={}", + processed, totalBlocks, String.format(Locale.ROOT, "%.2f", progress), + formatDuration(elapsedMillis / 1000), + String.format(Locale.ROOT, "%.2f", blocksPerSecond), + formatDuration(remainingSeconds)); + } + + private String formatDuration(long totalSeconds) { + long hours = totalSeconds / 3600; + long minutes = totalSeconds % 3600 / 60; + long seconds = totalSeconds % 60; + return String.format(Locale.ROOT, "%02d:%02d:%02d", hours, minutes, seconds); + } + private void processBlock(long blockNum, DBInterface transactionRetDb, DBInterface sectionBloomDb) throws BadItemException, EventBloomException { diff --git a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java index 2c3ca18a669..e48105128be 100644 --- a/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbBackfillBloomTest.java @@ -11,6 +11,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -25,6 +28,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.mockito.MockedStatic; +import org.slf4j.LoggerFactory; import org.tron.common.utils.ByteArray; import org.tron.plugins.utils.db.DBInterface; import org.tron.plugins.utils.db.DBIterator; @@ -90,6 +94,8 @@ public void tearDown() { public void testHelp() { String[] args = new String[] { "db", "backfill-bloom", "-h" }; assertEquals(0, cli.execute(args)); + assertTrue(outputStream.toString().contains( + "The same block range can be safely rerun after interruption.")); } @Test @@ -188,6 +194,64 @@ public void testSummaryReportsScannedBlocks() throws Exception { assertTrue(output.contains("Success rate: 100.00%")); } + @Test + public void testLogsProgressEveryTenThousandBlocksWithoutWritingToCommandOutput() + throws Exception { + DBInterface transactionRetDb = mock(DBInterface.class); + DBInterface sectionBloomDb = mock(DBInterface.class); + DBInterface propertiesDb = mock(DBInterface.class); + + dbToolMock.when(() -> DbTool.getDB(anyString(), anyString())) + .thenAnswer(invocation -> { + String dbName = invocation.getArgument(1); + switch (dbName) { + case "transactionRetStore": + return transactionRetDb; + case "section-bloom": + return sectionBloomDb; + case "properties": + return propertiesDb; + default: + return mock(DBInterface.class); + } + }); + when(propertiesDb.get(any(byte[].class))) + .thenReturn(ByteArray.fromLong(10_000L)); + mockMinBlock(transactionRetDb, 1L); + when(transactionRetDb.get(any(byte[].class))).thenReturn(null); + + StringWriter out = new StringWriter(); + CommandLine cmd = new CommandLine(new Toolkit()); + cmd.setOut(new PrintWriter(out)); + + Logger progressLogger = (Logger) LoggerFactory.getLogger("backfill-bloom"); + ListAppender appender = new ListAppender<>(); + appender.start(); + progressLogger.addAppender(appender); + try { + String[] args = new String[] { + "db", "backfill-bloom", + "-d", databaseDirectory, + "-s", "1", + "-e", "10000", + "-c", "1" + }; + + assertEquals(0, cmd.execute(args)); + + long progressLogCount = appender.list.stream() + .map(ILoggingEvent::getFormattedMessage) + .filter(message -> message.startsWith( + "Backfill progress: 10000/10000 blocks (100.00%)")) + .count(); + assertEquals(1L, progressLogCount); + assertFalse(out.toString().contains("Backfill progress:")); + } finally { + progressLogger.detachAppender(appender); + appender.stop(); + } + } + @Test public void testInvalidStartBlock() { String[] args = new String[] { @@ -411,7 +475,7 @@ public void testAdjustsStartBlockToMinimumNonZeroTransactionResultBlock() throws assertTrue(out.toString().contains( "Start block 5 is earlier than the first available transaction result block 10")); assertTrue(out.toString().contains( - "Starting SectionBloom backfill for blocks 10 to 12 (3 blocks)")); + "Starting SectionBloom backfill for block number 10 to 12 (3 blocks)")); verify(iterator).seek(aryEq(ByteArray.fromLong(1))); verify(iterator).close(); } From 3f4b9c49a96139c608ed5ad44520d0af01454a3e Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Mon, 27 Jul 2026 11:29:55 +0800 Subject: [PATCH 5/5] update document of DB Backfill-Bloom --- plugins/README.md | 50 +++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/plugins/README.md b/plugins/README.md index c892fabd381..ece19006b58 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -150,38 +150,46 @@ NOTE: large db may GC overhead limit exceeded. ## DB Backfill-Bloom -DB backfill bloom provides the ability to backfill SectionBloom data for historical blocks to enable `eth_getLogs` address/topics filtering. This is useful when `isJsonRpcFilterEnabled` was disabled during block processing and later enabled, causing historical blocks to lack SectionBloom data. +DB backfill bloom rebuilds SectionBloom data for historical blocks so that `eth_getLogs` can filter by address and topics. This is useful when `isJsonRpcFilterEnabled` was disabled during historical block processing and was enabled later. -The source database must contain a non-empty `transactionRetStore`. Ensure -`storage.transHistory.switch` was enabled while the historical blocks were processed. +### Prerequisites and behavior -The backfill operation is idempotent. If it is interrupted, you can safely rerun the same block range. Existing SectionBloom bits are preserved and set again. Do not run multiple backfill processes concurrently or run the tool while another process is using the same database. +- Stop the node and any other process using the database before running the command. +- The database directory must contain the `properties` and `transactionRetStore` databases. `transactionRetStore` must contain at least one non-zero block. +- Ensure `storage.transHistory.switch` was enabled while the historical blocks were processed. +- The start and end block numbers are inclusive. +- The command creates or updates the `section-bloom` database in the specified database directory. +- The operation is idempotent. If it is interrupted, safely rerun the same block range. Existing SectionBloom bits are preserved and set again. Do not run multiple backfill processes concurrently. -### Available parameters: +### Available parameters -- `-d | --database-directory`: Specify the database directory path, it is used to open the database to get the transaction log and write the SectionBloom data back, default: output-directory/database. -- `-s | --start-block`: Specify the start block number for backfill. -- `-e | --end-block`: Specify the end block number for backfill (optional, default: latest solidity block). -- `-c | --max-concurrency`: Specify the maximum concurrency for processing, default: 8. -- `-h | --help`: Provide the help info. +- `-d | --database-directory`: Parent directory containing the source databases and the destination `section-bloom` database. Default: `output-directory/database`. +- `-s | --start-block`: Inclusive start block. Optional; defaults to the earliest non-zero block in `transactionRetStore`. A lower value is automatically raised to the earliest available block. +- `-e | --end-block`: Inclusive end block. Optional; defaults to the latest solidified block in `properties`. A higher value is automatically reduced to the latest solidified block. +- `-c | --max-concurrency`: Maximum number of processing threads, from 1 to 128. Default: 8. Use 4–8 for SATA SSD, 8–16 for NVMe SSD, or 1–2 for HDD. The actual concurrency does not exceed the number of sections being processed. +- `-h | --help`: Display the help message. -### Examples: +### Examples ```shell script -# full command - java -jar Toolkit.jar db backfill-bloom [-h] -s= [-e=] [-d=] [-c=] -# examples - java -jar Toolkit.jar db backfill-bloom -s 1000000 -e 2000000 #1. backfill blocks 1000000 to 2000000 - java -jar Toolkit.jar db backfill-bloom -s 1000000 -d /path/to/database #2. specify custom database directory - java -jar Toolkit.jar db backfill-bloom -s 1000000 -c 8 #3. use higher concurrency (8 threads) +# Full command +java -jar Toolkit.jar db backfill-bloom [-d ] [-s ] [-e ] [-c ] [-h] + +# Backfill the complete available range in the default database directory +java -jar Toolkit.jar db backfill-bloom + +# Backfill blocks 1,000,000 through 2,000,000, inclusive +java -jar Toolkit.jar db backfill-bloom -s 1000000 -e 2000000 + +# Use a custom database directory and eight processing threads +java -jar Toolkit.jar db backfill-bloom -d /path/to/database -c 8 ``` -### Backfill speed +### Progress and performance -The time required to process different block ranges varies. It is recommended to increase `--max-concurrency` appropriately to speed up the backfill process. +The terminal progress bar displays completed blocks, elapsed time, and estimated remaining time. `toolkit.log` records progress every 10,000 scanned blocks and includes the percentage, elapsed time, average rate, and estimated remaining time. The final summary reports scanned and successful blocks, blocks containing logs, errors, Bloom writes, duration, rates, and the concurrency used. -- 0-10000000: It's done almost instantly because there are no logs inside. -- 10000000-70000000: Takes about 3-4 hours/10,000,000 blocks with `--max-concurrency` set to 32. +Performance depends on the number of logs, storage engine, disk, CPU, and database compaction. Increase `--max-concurrency` gradually while monitoring disk latency and CPU usage. ## Keystore