diff --git a/config-example.json b/config-example.json index 91e99672..b6e2279e 100644 --- a/config-example.json +++ b/config-example.json @@ -24,5 +24,14 @@ "blocks": { "enableMetrics": true, "metricsPort": 4001 + }, + "bitcoin": { + "mempoolApiUrl": "https://mempool.space/api", + "blocksSample": 1000, + "hashratePeriod": "1w", + "minCoverage": 0.9, + "requestDelayMs": 250, + "requestTimeoutMs": 15000, + "maxRetries": 3 } } \ No newline at end of file diff --git a/package.json b/package.json index 674ca526..e884f90d 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@rsksmart/rsk-contract-parser": "^2.2.0", "@rsksmart/rsk-js-cli": "^1.0.0", "@rsksmart/rsk-utils": "^1.1.0", + "axios": "^1.8.4", "bignumber.js": "^7.2.1", "body-parser": "^1.19.0", "bunyan": "^2.0.4", @@ -60,7 +61,6 @@ "@babel/node": "^7.10.4", "@babel/preset-env": "^7.11.5", "@babel/register": "^7.11.5", - "axios": "^1.8.4", "chai": "^4.2.0", "deep-equal-in-any-order": "^2.0.6", "eslint": "^4.19.1", diff --git a/prisma/rsk-explorer-database.sql b/prisma/rsk-explorer-database.sql index d252d3ce..1a65d383 100644 --- a/prisma/rsk-explorer-database.sql +++ b/prisma/rsk-explorer-database.sql @@ -1,6 +1,9 @@ --- RSK Explorer Database Schema V1.2.5 +-- RSK Explorer Database Schema V1.2.6 /* +V1.2.6 Notes: +- Added bitcoin merge-mining daily stats table (btc_merge_mining_stats) + V1.2.5 Notes: - Added new index in address_in_event table @@ -681,4 +684,15 @@ CREATE TABLE bo_active_addresses_daily_aggregated ( CREATE TABLE bo_number_transactions_daily_aggregated ( date_1 DATE PRIMARY KEY, number_of_transactions INT +); + +-- Daily Bitcoin merge-mining stats +CREATE TABLE btc_merge_mining_stats ( + date DATE PRIMARY KEY, + bitcoin_hashrate VARCHAR NOT NULL, + rootstock_secured_hashrate VARCHAR NOT NULL, + merge_mining_percentage NUMERIC NOT NULL, + bitcoin_blocks_sampled INT NOT NULL, + merge_mined_blocks INT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 155f4151..b2d4fc5d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -610,6 +610,16 @@ model bo_number_transactions_daily_aggregated { numberOfTransactions Int? @map("number_of_transactions") } +model btc_merge_mining_stats { + date DateTime @id @db.Date + bitcoinHashrate String @map("bitcoin_hashrate") @db.VarChar + rootstockSecuredHashrate String @map("rootstock_secured_hashrate") @db.VarChar + mergeMiningPercentage Decimal @map("merge_mining_percentage") @db.Decimal + bitcoinBlocksSampled Int @map("bitcoin_blocks_sampled") + mergeMinedBlocks Int @map("merge_mined_blocks") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) +} + model receipt { transactionHash String @id @map("transaction_hash") @db.VarChar(66) contractAddress String? @map("contract_address") @db.VarChar(42) diff --git a/src/lib/btcMempoolClient.js b/src/lib/btcMempoolClient.js new file mode 100644 index 00000000..5e402edc --- /dev/null +++ b/src/lib/btcMempoolClient.js @@ -0,0 +1,69 @@ +import axios from 'axios' +import Logger from './Logger' + +const DEFAULTS = { + baseUrl: 'https://mempool.space/api', + requestDelayMs: 250, + requestTimeoutMs: 15000, + maxRetries: 3, + retryDelayMs: 1000 +} + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) + +// Read-only client for the mempool.space REST API (Esplora-compatible), +// treating the provider as untrusted: bounded timeout, retries on network/ +// timeout and 429/5xx errors (not other 4xx), and a throttle between calls. +// Uses axios rather than a fetch global, which is absent before Node 18. +export function createBtcMempoolClient (options = {}) { + const cfg = { ...DEFAULTS, ...options } + const baseUrl = cfg.baseUrl.replace(/\/+$/, '') + const log = cfg.log || Logger('[btc-mempool-client]') + + async function fetchOnce (path) { + try { + const { data } = await axios.get(`${baseUrl}${path}`, { timeout: cfg.requestTimeoutMs }) + return data + } catch (error) { + const status = error.response && error.response.status + // Network/timeout errors and 429/5xx are worth retrying; other 4xx are not + error.retryable = !status || status === 429 || status >= 500 + throw error + } + } + + async function request (path) { + let lastError + for (let attempt = 0; attempt <= cfg.maxRetries; attempt++) { + try { + return await fetchOnce(path) + } catch (error) { + lastError = error + if (!error.retryable || attempt === cfg.maxRetries) break + const backoff = cfg.retryDelayMs * (attempt + 1) + log.warn(`Request to ${path} failed (${error.message}); retry ${attempt + 1}/${cfg.maxRetries} in ${backoff} ms`) + await sleep(backoff) + } + } + throw lastError + } + + return { + getTipHeight: async () => Number(await request('/blocks/tip/height')), + getBlockHash: async height => { + const hash = String(await request(`/block-height/${height}`)).trim() + // Validated before being interpolated into later request paths + if (!/^[0-9a-f]{64}$/i.test(hash)) throw new Error(`Invalid block hash for height ${height}`) + return hash + }, + getCoinbase: async blockHash => { + const txs = await request(`/block/${blockHash}/txs/0`) + return Array.isArray(txs) ? txs[0] : null + }, + getNetworkHashrate: async (period = '1w') => { + const { currentHashrate } = await request(`/v1/mining/hashrate/${period}`) + return currentHashrate + }, + throttle: () => sleep(cfg.requestDelayMs) + } +} diff --git a/src/lib/cronJobs.js b/src/lib/cronJobs.js index c222bb38..a8cfeae2 100644 --- a/src/lib/cronJobs.js +++ b/src/lib/cronJobs.js @@ -1,6 +1,9 @@ import { CronJob } from 'cron' import { prismaClient } from './prismaClient' import Logger from './Logger' +import config from './config' +import { createBtcMempoolClient } from './btcMempoolClient' +import { getMergeMiningStats } from './getMergeMiningStats' /* Supported Ranges @@ -70,6 +73,10 @@ const cronsConfig = { daily3_15AM: { value: '15 3 * * *', description: 'every day at 3:15 AM' + }, + daily3_20AM: { + value: '20 3 * * *', + description: 'every day at 3:20 AM' } } } @@ -265,11 +272,67 @@ function dailyNumberOfTransactionsUpdater () { } } +// Daily Bitcoin merge-mining stats snapshot +function dailyMergeMiningStatsUpdater () { + const name = 'daily-merge-mining-stats' + const log = Logger(`[${name}]`) + const schedule = cronsConfig.schedules.daily3_20AM + const action = async () => { + try { + log.info(`Started at ${new Date().toISOString()} (${cronsConfig.timeZone}). Job Schedule: ${schedule.value} (${schedule.description})`) + log.info(`Updating daily merge-mining stats...`) + + const started = Date.now() + const { bitcoin } = config + const client = createBtcMempoolClient({ + baseUrl: bitcoin.mempoolApiUrl, + requestDelayMs: bitcoin.requestDelayMs, + requestTimeoutMs: bitcoin.requestTimeoutMs, + maxRetries: bitcoin.maxRetries, + log + }) + + const stats = await getMergeMiningStats({ + client, + sampleSize: bitcoin.blocksSample, + hashratePeriod: bitcoin.hashratePeriod, + minCoverage: bitcoin.minCoverage, + log + }) + + const date = new Date() + date.setUTCHours(0, 0, 0, 0) + + await prismaClient.btc_merge_mining_stats.upsert({ + where: { date }, + update: stats, + create: { date, ...stats } + }) + + log.info(`Daily merge-mining stats updated (${Date.now() - started} ms)`) + log.info(`Finished at ${new Date().toISOString()} (${cronsConfig.timeZone})`) + } catch (error) { + // Leaves the previous snapshot in place so the API keeps serving last-good data + log.error(`Error updating daily merge-mining stats: ${error.message}`) + log.error(error.stack) + } + } + const cronJob = createCronJob({ schedule, action }) + + return { + schedule, + name, + cronJob, + start: () => cronJob.start() + } +} + const cronJobs = { dailyGasFeesUpdater: dailyGasFeesUpdater(), newAddressesUpdater: newAddressesUpdater(), dailyActiveAddressesUpdater: dailyActiveAddressesUpdater(), - dailyNumberOfTransactionsUpdater: dailyNumberOfTransactionsUpdater() + dailyNumberOfTransactionsUpdater: dailyNumberOfTransactionsUpdater(), + dailyMergeMiningStatsUpdater: dailyMergeMiningStatsUpdater() } export function startCronJobs ({ log = Logger('[CronJobs]') } = {}) { diff --git a/src/lib/defaultConfig.js b/src/lib/defaultConfig.js index 273fccf3..2c969787 100644 --- a/src/lib/defaultConfig.js +++ b/src/lib/defaultConfig.js @@ -76,6 +76,16 @@ export default { address: '127.0.0.1', services }, + bitcoin: { + // Testnet deployments override with 'https://mempool.space/testnet/api' + mempoolApiUrl: 'https://mempool.space/api', + blocksSample: 1000, + hashratePeriod: '1w', + minCoverage: 0.9, + requestDelayMs: 250, + requestTimeoutMs: 15000, + maxRetries: 3 + }, forceSaveBcStats: true, enableTxPoolFromApi: true } diff --git a/src/lib/getMergeMiningStats.js b/src/lib/getMergeMiningStats.js new file mode 100644 index 00000000..9c1528bf --- /dev/null +++ b/src/lib/getMergeMiningStats.js @@ -0,0 +1,64 @@ +import { BigNumber } from 'bignumber.js' +import { isRskMergeMined } from './rskMergeMiningTag' +import Logger from './Logger' + +export async function getMergeMiningStats ({ + client, + sampleSize = 1000, + hashratePeriod = '1w', + minCoverage = 0.9, + log = Logger('[merge-mining-stats]') +}) { + if (!client) throw new Error('Missing Bitcoin client') + if (!Number.isInteger(sampleSize) || sampleSize < 1) throw new Error(`Invalid sampleSize: ${sampleSize}`) + if (!(minCoverage > 0) || minCoverage > 1) throw new Error(`Invalid minCoverage: ${minCoverage}`) + + // Fetched first so a bad hashrate fails fast, before the long block walk + const bitcoinHashrate = new BigNumber(await client.getNetworkHashrate(hashratePeriod)) + if (!bitcoinHashrate.isFinite() || bitcoinHashrate.lte(0)) { + throw new Error(`Invalid Bitcoin hashrate: ${bitcoinHashrate.toString()}`) + } + + const tipHeight = await client.getTipHeight() + if (!Number.isInteger(tipHeight) || tipHeight < 0) throw new Error(`Invalid tip height: ${tipHeight}`) + + const lowestHeight = Math.max(0, tipHeight - sampleSize + 1) + const requested = tipHeight - lowestHeight + 1 + let mergeMinedBlocks = 0 + let bitcoinBlocksSampled = 0 + let failed = 0 + + for (let height = tipHeight; height >= lowestHeight; height--) { + try { + const coinbase = await client.getCoinbase(await client.getBlockHash(height)) + if (!coinbase) throw new Error(`Missing coinbase for block ${height}`) + if (isRskMergeMined(coinbase)) mergeMinedBlocks++ + bitcoinBlocksSampled++ + } catch (error) { + failed++ + log.warn(`Skipping BTC block ${height}: ${error.message}`) + } + // Stop early once the remaining blocks can no longer restore minimum coverage + // (e.g. a provider outage), instead of walking the whole window for nothing + if (bitcoinBlocksSampled + (height - lowestHeight) < requested * minCoverage) break + await client.throttle() + } + + // A failing or hostile provider must not skew the ratio by starving us of blocks + if (bitcoinBlocksSampled < requested * minCoverage) { + throw new Error(`Insufficient block coverage: ${bitcoinBlocksSampled}/${requested} sampled (${failed} failed)`) + } + + const mergeMiningPercentage = new BigNumber(mergeMinedBlocks).dividedBy(bitcoinBlocksSampled) + const rootstockSecuredHashrate = bitcoinHashrate.times(mergeMiningPercentage) + + log.info(`Sampled ${bitcoinBlocksSampled}/${requested} BTC blocks (${failed} failed); ${mergeMinedBlocks} merge-mined (${mergeMiningPercentage.times(100).toFixed(2)}%)`) + + return { + bitcoinHashrate: bitcoinHashrate.toFixed(0), + rootstockSecuredHashrate: rootstockSecuredHashrate.toFixed(0), + mergeMiningPercentage: mergeMiningPercentage.toFixed(6), + bitcoinBlocksSampled, + mergeMinedBlocks + } +} diff --git a/src/lib/rskMergeMiningTag.js b/src/lib/rskMergeMiningTag.js new file mode 100644 index 00000000..04b82943 --- /dev/null +++ b/src/lib/rskMergeMiningTag.js @@ -0,0 +1,23 @@ +import { remove0x } from './utils' + +// 'RSKBLOCK:' marker (RSKIP-110) merge-mining pools embed in the Bitcoin +// coinbase, in an OP_RETURN output or the input scriptSig — so both are scanned. +export const RSK_TAG_HEX = '52534b424c4f434b3a' + +const hexHasTag = hex => + typeof hex === 'string' && remove0x(hex).toLowerCase().includes(RSK_TAG_HEX) + +// Handles both the mempool/Esplora shape (`scriptsig` / `scriptpubkey`) and the +// Bitcoin Core RPC shape (`scriptSig.hex` / `scriptPubKey.hex`). +export function isRskMergeMined (coinbase) { + if (!coinbase) return false + + const inputScripts = (coinbase.vin || []).map( + input => input && (input.scriptsig || (input.scriptSig && input.scriptSig.hex)) + ) + const outputScripts = (coinbase.vout || []).map( + output => output && (output.scriptpubkey || (output.scriptPubKey && output.scriptPubKey.hex)) + ) + + return [...inputScripts, ...outputScripts].some(hexHasTag) +} diff --git a/test/btcMempoolClient.spec.js b/test/btcMempoolClient.spec.js new file mode 100644 index 00000000..4c2350d8 --- /dev/null +++ b/test/btcMempoolClient.spec.js @@ -0,0 +1,59 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import axios from 'axios' +import { createBtcMempoolClient } from '../src/lib/btcMempoolClient' + +const silentLog = { info () {}, warn () {}, error () {} } +const httpError = status => { + const error = new Error(`HTTP ${status}`) + error.response = { status } + return error +} +const client = () => createBtcMempoolClient({ retryDelayMs: 1, maxRetries: 2, log: silentLog }) + +describe('# btcMempoolClient', function () { + afterEach(() => sinon.restore()) + + it('retries on server errors and returns once the request succeeds', async () => { + const get = sinon.stub(axios, 'get') + get.onCall(0).rejects(httpError(503)) + get.onCall(1).resolves({ data: 512345 }) + + const height = await client().getTipHeight() + expect(height).to.equal(512345) + expect(get.callCount).to.equal(2) + }) + + it('fails fast on a non-retryable 4xx without exhausting retries', async () => { + const get = sinon.stub(axios, 'get').rejects(httpError(404)) + + let threw = false + try { + await client().getNetworkHashrate('1w') + } catch (error) { + threw = true + } + expect(threw).to.equal(true) + expect(get.callCount).to.equal(1) + }) + + it('rejects a block hash that is not 64 hex chars', async () => { + sinon.stub(axios, 'get').resolves({ data: 'not-a-valid-hash' }) + + let threw = false + try { + await client().getBlockHash(800000) + } catch (error) { + threw = true + } + expect(threw).to.equal(true) + }) + + it('returns the coinbase (first tx) of a block', async () => { + const coinbase = { vin: [{ scriptsig: '03aa' }], vout: [] } + sinon.stub(axios, 'get').resolves({ data: [coinbase] }) + + const tx = await client().getCoinbase('a'.repeat(64)) + expect(tx).to.deep.equal(coinbase) + }) +}) diff --git a/test/getMergeMiningStats.spec.js b/test/getMergeMiningStats.spec.js new file mode 100644 index 00000000..feeb42f9 --- /dev/null +++ b/test/getMergeMiningStats.spec.js @@ -0,0 +1,76 @@ +import { expect } from 'chai' +import { getMergeMiningStats } from '../src/lib/getMergeMiningStats' +import { RSK_TAG_HEX } from '../src/lib/rskMergeMiningTag' + +const silentLog = { info () {}, warn () {}, error () {} } +const taggedCoinbase = { vin: [{ scriptsig: '03aa' }], vout: [{ scriptpubkey: `6a29${RSK_TAG_HEX}${'ab'.repeat(32)}` }] } +const plainCoinbase = { vin: [{ scriptsig: '03bb' }], vout: [{ scriptpubkey: '76a90088ac' }] } + +// Heights in `taggedHeights` are merge-mined; heights in `failHeights` throw on fetch. +function fakeClient ({ tipHeight, taggedHeights = [], failHeights = [], hashrate = 900 }) { + return { + getTipHeight: async () => tipHeight, + getBlockHash: async height => { + if (failHeights.includes(height)) throw new Error(`boom ${height}`) + return `hash-${height}` + }, + getCoinbase: async hash => { + const height = Number(hash.replace('hash-', '')) + return taggedHeights.includes(height) ? taggedCoinbase : plainCoinbase + }, + getNetworkHashrate: async () => hashrate, + throttle: async () => {} + } +} + +describe('# getMergeMiningStats', function () { + it('counts merge-mined blocks and derives the secured hashrate', async () => { + const client = fakeClient({ tipHeight: 9, taggedHeights: [9, 7, 5], hashrate: 1000 }) + const stats = await getMergeMiningStats({ client, sampleSize: 10, log: silentLog }) + + expect(stats.bitcoinBlocksSampled).to.equal(10) + expect(stats.mergeMinedBlocks).to.equal(3) + expect(stats.mergeMiningPercentage).to.equal('0.300000') + expect(stats.bitcoinHashrate).to.equal('1000') + expect(stats.rootstockSecuredHashrate).to.equal('300') + }) + + it('caps the window at the genesis block when the chain is shorter than the sample', async () => { + const client = fakeClient({ tipHeight: 2, taggedHeights: [2] }) + const stats = await getMergeMiningStats({ client, sampleSize: 1000, log: silentLog }) + + expect(stats.bitcoinBlocksSampled).to.equal(3) + expect(stats.mergeMinedBlocks).to.equal(1) + }) + + it('skips unreachable blocks and computes the ratio over those actually sampled', async () => { + const client = fakeClient({ tipHeight: 9, taggedHeights: [9, 8], failHeights: [7], hashrate: 1000 }) + const stats = await getMergeMiningStats({ client, sampleSize: 10, minCoverage: 0.5, log: silentLog }) + + expect(stats.bitcoinBlocksSampled).to.equal(9) + expect(stats.mergeMinedBlocks).to.equal(2) + expect(stats.mergeMiningPercentage).to.equal('0.222222') + }) + + it('aborts rather than publish a ratio when block coverage is too low', async () => { + const client = fakeClient({ tipHeight: 9, failHeights: [9, 8, 7, 6, 5] }) + let threw = false + try { + await getMergeMiningStats({ client, sampleSize: 10, minCoverage: 0.9, log: silentLog }) + } catch (error) { + threw = true + } + expect(threw).to.equal(true) + }) + + it('rejects an invalid Bitcoin hashrate before walking blocks', async () => { + const client = fakeClient({ tipHeight: 1, hashrate: 0 }) + let threw = false + try { + await getMergeMiningStats({ client, sampleSize: 2, log: silentLog }) + } catch (error) { + threw = true + } + expect(threw).to.equal(true) + }) +}) diff --git a/test/rskMergeMiningTag.spec.js b/test/rskMergeMiningTag.spec.js new file mode 100644 index 00000000..15178d6f --- /dev/null +++ b/test/rskMergeMiningTag.spec.js @@ -0,0 +1,47 @@ +import { expect } from 'chai' +import { isRskMergeMined, RSK_TAG_HEX } from '../src/lib/rskMergeMiningTag' + +// 'RSKBLOCK:' + a 32-byte payload, as embedded by merge-mining pools. +const taggedScript = `6a29${RSK_TAG_HEX}${'ab'.repeat(32)}` +const plainScript = '6a14' + 'cd'.repeat(20) + +describe('# rskMergeMiningTag', function () { + describe('isRskMergeMined()', function () { + it('detects the RSK tag in a coinbase OP_RETURN output (mempool shape)', () => { + const coinbase = { + vin: [{ scriptsig: '03abcdef' }], + vout: [{ scriptpubkey: '76a914' + '00'.repeat(20) + '88ac' }, { scriptpubkey: taggedScript }] + } + expect(isRskMergeMined(coinbase)).to.equal(true) + }) + + it('detects the RSK tag in the coinbase input scriptSig', () => { + const coinbase = { vin: [{ scriptsig: `03abcd${taggedScript}` }], vout: [{ scriptpubkey: plainScript }] } + expect(isRskMergeMined(coinbase)).to.equal(true) + }) + + it('detects the RSK tag in the Bitcoin Core RPC shape', () => { + const coinbase = { + vin: [{ scriptSig: { hex: '03abcdef' } }], + vout: [{ scriptPubKey: { hex: taggedScript } }] + } + expect(isRskMergeMined(coinbase)).to.equal(true) + }) + + it('matches the tag regardless of hex casing', () => { + const coinbase = { vin: [], vout: [{ scriptpubkey: taggedScript.toUpperCase() }] } + expect(isRskMergeMined(coinbase)).to.equal(true) + }) + + it('returns false for a coinbase without the RSK tag', () => { + const coinbase = { vin: [{ scriptsig: '03abcdef' }], vout: [{ scriptpubkey: plainScript }] } + expect(isRskMergeMined(coinbase)).to.equal(false) + }) + + it('returns false for missing or malformed coinbase data', () => { + expect(isRskMergeMined(null)).to.equal(false) + expect(isRskMergeMined({})).to.equal(false) + expect(isRskMergeMined({ vin: [null], vout: [{}] })).to.equal(false) + }) + }) +})