feat(stats): add daily Bitcoin merge-mining metrics snapshot

config-example.json

@@ -24,5 +24,14 @@
   "blocks": {
     "enableMetrics": true,
     "metricsPort": 4001
+  },
+  "bitcoin": {
+    "mempoolApiUrl": "https://mempool.space/api",
+    "blocksSample": 1000,
+    "hashratePeriod": "1w",
+    "minCoverage": 0.9,
+    "requestDelayMs": 250,
+    "requestTimeoutMs": 15000,
+    "maxRetries": 3
   }
 }

package.json

@@ -42,6 +42,7 @@
     "@rsksmart/rsk-contract-parser": "^2.2.0",
     "@rsksmart/rsk-js-cli": "^1.0.0",
     "@rsksmart/rsk-utils": "^1.1.0",
+    "axios": "^1.8.4",
     "bignumber.js": "^7.2.1",
     "body-parser": "^1.19.0",
     "bunyan": "^2.0.4",
@@ -60,7 +61,6 @@
     "@babel/node": "^7.10.4",
     "@babel/preset-env": "^7.11.5",
     "@babel/register": "^7.11.5",
-    "axios": "^1.8.4",
     "chai": "^4.2.0",
     "deep-equal-in-any-order": "^2.0.6",
     "eslint": "^4.19.1",

prisma/rsk-explorer-database.sql

@@ -1,6 +1,9 @@
--- RSK Explorer Database Schema V1.2.5
+-- RSK Explorer Database Schema V1.2.6
 /*
 
+V1.2.6 Notes:
+- Added bitcoin merge-mining daily stats table (btc_merge_mining_stats)
+
 V1.2.5 Notes:
 - Added new index in address_in_event table
 
@@ -681,4 +684,15 @@ CREATE TABLE bo_active_addresses_daily_aggregated (
 CREATE TABLE bo_number_transactions_daily_aggregated (
     date_1 DATE PRIMARY KEY,
     number_of_transactions INT
+);
+
+-- Daily Bitcoin merge-mining stats
+CREATE TABLE btc_merge_mining_stats (
+    date DATE PRIMARY KEY,
+    bitcoin_hashrate VARCHAR NOT NULL,
+    rootstock_secured_hashrate VARCHAR NOT NULL,
+    merge_mining_percentage NUMERIC NOT NULL,
+    bitcoin_blocks_sampled INT NOT NULL,
+    merge_mined_blocks INT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
 );

prisma/schema.prisma

@@ -610,6 +610,16 @@ model bo_number_transactions_daily_aggregated {
   numberOfTransactions Int?     @map("number_of_transactions")
 }
 
+model btc_merge_mining_stats {
+  date                     DateTime @id @db.Date
+  bitcoinHashrate          String   @map("bitcoin_hashrate") @db.VarChar
+  rootstockSecuredHashrate String   @map("rootstock_secured_hashrate") @db.VarChar
+  mergeMiningPercentage    Decimal  @map("merge_mining_percentage") @db.Decimal
+  bitcoinBlocksSampled     Int      @map("bitcoin_blocks_sampled")
+  mergeMinedBlocks         Int      @map("merge_mined_blocks")
+  createdAt                DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
+}
+
 model receipt {
   transactionHash                           String      @id @map("transaction_hash") @db.VarChar(66)
   contractAddress                           String?     @map("contract_address") @db.VarChar(42)

src/lib/btcMempoolClient.js

@@ -0,0 +1,69 @@
+import axios from 'axios'
+import Logger from './Logger'
+
+const DEFAULTS = {
+  baseUrl: 'https://mempool.space/api',
+  requestDelayMs: 250,
+  requestTimeoutMs: 15000,
+  maxRetries: 3,
+  retryDelayMs: 1000
+}
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+
+// Read-only client for the mempool.space REST API (Esplora-compatible),
+// treating the provider as untrusted: bounded timeout, retries on network/
+// timeout and 429/5xx errors (not other 4xx), and a throttle between calls.
+// Uses axios rather than a fetch global, which is absent before Node 18.
+export function createBtcMempoolClient (options = {}) {
+  const cfg = { ...DEFAULTS, ...options }
+  const baseUrl = cfg.baseUrl.replace(/\/+$/, '')
+  const log = cfg.log || Logger('[btc-mempool-client]')
+
+  async function fetchOnce (path) {
+    try {
+      const { data } = await axios.get(`${baseUrl}${path}`, { timeout: cfg.requestTimeoutMs })
+      return data
+    } catch (error) {
+      const status = error.response && error.response.status
+      // Network/timeout errors and 429/5xx are worth retrying; other 4xx are not
+      error.retryable = !status || status === 429 || status >= 500
+      throw error
+    }
+  }
+
+  async function request (path) {
+    let lastError
+    for (let attempt = 0; attempt <= cfg.maxRetries; attempt++) {
+      try {
+        return await fetchOnce(path)
+      } catch (error) {
+        lastError = error
+        if (!error.retryable || attempt === cfg.maxRetries) break
+        const backoff = cfg.retryDelayMs * (attempt + 1)
+        log.warn(`Request to ${path} failed (${error.message}); retry ${attempt + 1}/${cfg.maxRetries} in ${backoff} ms`)
+        await sleep(backoff)
+      }
+    }
+    throw lastError
+  }
+
+  return {
+    getTipHeight: async () => Number(await request('/blocks/tip/height')),
+    getBlockHash: async height => {
+      const hash = String(await request(`/block-height/${height}`)).trim()
+      // Validated before being interpolated into later request paths
+      if (!/^[0-9a-f]{64}$/i.test(hash)) throw new Error(`Invalid block hash for height ${height}`)
+      return hash
+    },
+    getCoinbase: async blockHash => {
+      const txs = await request(`/block/${blockHash}/txs/0`)
+      return Array.isArray(txs) ? txs[0] : null
+    },
+    getNetworkHashrate: async (period = '1w') => {
+      const { currentHashrate } = await request(`/v1/mining/hashrate/${period}`)
+      return currentHashrate
+    },
+    throttle: () => sleep(cfg.requestDelayMs)
+  }
+}

src/lib/cronJobs.js

@@ -1,6 +1,9 @@
 import { CronJob } from 'cron'
 import { prismaClient } from './prismaClient'
 import Logger from './Logger'
+import config from './config'
+import { createBtcMempoolClient } from './btcMempoolClient'
+import { getMergeMiningStats } from './getMergeMiningStats'
 
 /*
 Supported Ranges
@@ -70,6 +73,10 @@ const cronsConfig = {
     daily3_15AM: {
       value: '15 3 * * *',
       description: 'every day at 3:15 AM'
+    },
+    daily3_20AM: {
+      value: '20 3 * * *',
+      description: 'every day at 3:20 AM'
     }
   }
 }
@@ -265,11 +272,67 @@ function dailyNumberOfTransactionsUpdater () {
   }
 }
 
+// Daily Bitcoin merge-mining stats snapshot
+function dailyMergeMiningStatsUpdater () {
+  const name = 'daily-merge-mining-stats'
+  const log = Logger(`[${name}]`)
+  const schedule = cronsConfig.schedules.daily3_20AM
+  const action = async () => {
+    try {
+      log.info(`Started at ${new Date().toISOString()} (${cronsConfig.timeZone}). Job Schedule: ${schedule.value} (${schedule.description})`)
+      log.info(`Updating daily merge-mining stats...`)
+
+      const started = Date.now()
+      const { bitcoin } = config
+      const client = createBtcMempoolClient({
+        baseUrl: bitcoin.mempoolApiUrl,
+        requestDelayMs: bitcoin.requestDelayMs,
+        requestTimeoutMs: bitcoin.requestTimeoutMs,
+        maxRetries: bitcoin.maxRetries,
+        log
+      })
+
+      const stats = await getMergeMiningStats({
+        client,
+        sampleSize: bitcoin.blocksSample,
+        hashratePeriod: bitcoin.hashratePeriod,
+        minCoverage: bitcoin.minCoverage,
+        log
+      })
+
+      const date = new Date()
+      date.setUTCHours(0, 0, 0, 0)
+
+      await prismaClient.btc_merge_mining_stats.upsert({
+        where: { date },
+        update: stats,
+        create: { date, ...stats }
+      })
+
+      log.info(`Daily merge-mining stats updated (${Date.now() - started} ms)`)
+      log.info(`Finished at ${new Date().toISOString()} (${cronsConfig.timeZone})`)
+    } catch (error) {
+      // Leaves the previous snapshot in place so the API keeps serving last-good data
+      log.error(`Error updating daily merge-mining stats: ${error.message}`)
+      log.error(error.stack)
+    }
+  }
+  const cronJob = createCronJob({ schedule, action })
+
+  return {
+    schedule,
+    name,
+    cronJob,
+    start: () => cronJob.start()
+  }
+}
+
 const cronJobs = {
   dailyGasFeesUpdater: dailyGasFeesUpdater(),
   newAddressesUpdater: newAddressesUpdater(),
   dailyActiveAddressesUpdater: dailyActiveAddressesUpdater(),
-  dailyNumberOfTransactionsUpdater: dailyNumberOfTransactionsUpdater()
+  dailyNumberOfTransactionsUpdater: dailyNumberOfTransactionsUpdater(),
+  dailyMergeMiningStatsUpdater: dailyMergeMiningStatsUpdater()
 }
 
 export function startCronJobs ({ log = Logger('[CronJobs]') } = {}) {

src/lib/defaultConfig.js

@@ -76,6 +76,16 @@ export default {
     address: '127.0.0.1',
     services
   },
+  bitcoin: {
+    // Testnet deployments override with 'https://mempool.space/testnet/api'
+    mempoolApiUrl: 'https://mempool.space/api',
+    blocksSample: 1000,
+    hashratePeriod: '1w',
+    minCoverage: 0.9,
+    requestDelayMs: 250,
+    requestTimeoutMs: 15000,
+    maxRetries: 3
+  },
   forceSaveBcStats: true,
   enableTxPoolFromApi: true
 }

src/lib/getMergeMiningStats.js

@@ -0,0 +1,64 @@
+import { BigNumber } from 'bignumber.js'
+import { isRskMergeMined } from './rskMergeMiningTag'
+import Logger from './Logger'
+
+export async function getMergeMiningStats ({
+  client,
+  sampleSize = 1000,
+  hashratePeriod = '1w',
+  minCoverage = 0.9,
+  log = Logger('[merge-mining-stats]')
+}) {
+  if (!client) throw new Error('Missing Bitcoin client')
+  if (!Number.isInteger(sampleSize) || sampleSize < 1) throw new Error(`Invalid sampleSize: ${sampleSize}`)
+  if (!(minCoverage > 0) || minCoverage > 1) throw new Error(`Invalid minCoverage: ${minCoverage}`)
+
+  // Fetched first so a bad hashrate fails fast, before the long block walk
+  const bitcoinHashrate = new BigNumber(await client.getNetworkHashrate(hashratePeriod))
+  if (!bitcoinHashrate.isFinite() || bitcoinHashrate.lte(0)) {
+    throw new Error(`Invalid Bitcoin hashrate: ${bitcoinHashrate.toString()}`)
+  }
+
+  const tipHeight = await client.getTipHeight()
+  if (!Number.isInteger(tipHeight) || tipHeight < 0) throw new Error(`Invalid tip height: ${tipHeight}`)
+
+  const lowestHeight = Math.max(0, tipHeight - sampleSize + 1)
+  const requested = tipHeight - lowestHeight + 1
+  let mergeMinedBlocks = 0
+  let bitcoinBlocksSampled = 0
+  let failed = 0
+
+  for (let height = tipHeight; height >= lowestHeight; height--) {
+    try {
+      const coinbase = await client.getCoinbase(await client.getBlockHash(height))
+      if (!coinbase) throw new Error(`Missing coinbase for block ${height}`)
+      if (isRskMergeMined(coinbase)) mergeMinedBlocks++
+      bitcoinBlocksSampled++
+    } catch (error) {
+      failed++
+      log.warn(`Skipping BTC block ${height}: ${error.message}`)
+    }
+    // Stop early once the remaining blocks can no longer restore minimum coverage
+    // (e.g. a provider outage), instead of walking the whole window for nothing
+    if (bitcoinBlocksSampled + (height - lowestHeight) < requested * minCoverage) break
+    await client.throttle()
+  }
+
+  // A failing or hostile provider must not skew the ratio by starving us of blocks
+  if (bitcoinBlocksSampled < requested * minCoverage) {
+    throw new Error(`Insufficient block coverage: ${bitcoinBlocksSampled}/${requested} sampled (${failed} failed)`)
+  }
+
+  const mergeMiningPercentage = new BigNumber(mergeMinedBlocks).dividedBy(bitcoinBlocksSampled)
+  const rootstockSecuredHashrate = bitcoinHashrate.times(mergeMiningPercentage)
+
+  log.info(`Sampled ${bitcoinBlocksSampled}/${requested} BTC blocks (${failed} failed); ${mergeMinedBlocks} merge-mined (${mergeMiningPercentage.times(100).toFixed(2)}%)`)
+
+  return {
+    bitcoinHashrate: bitcoinHashrate.toFixed(0),
+    rootstockSecuredHashrate: rootstockSecuredHashrate.toFixed(0),
+    mergeMiningPercentage: mergeMiningPercentage.toFixed(6),
+    bitcoinBlocksSampled,
+    mergeMinedBlocks
+  }
+}

src/lib/rskMergeMiningTag.js

@@ -0,0 +1,23 @@
+import { remove0x } from './utils'
+
+// 'RSKBLOCK:' marker (RSKIP-110) merge-mining pools embed in the Bitcoin
+// coinbase, in an OP_RETURN output or the input scriptSig — so both are scanned.
+export const RSK_TAG_HEX = '52534b424c4f434b3a'
+
+const hexHasTag = hex =>
+  typeof hex === 'string' && remove0x(hex).toLowerCase().includes(RSK_TAG_HEX)
+
+// Handles both the mempool/Esplora shape (`scriptsig` / `scriptpubkey`) and the
+// Bitcoin Core RPC shape (`scriptSig.hex` / `scriptPubKey.hex`).
+export function isRskMergeMined (coinbase) {
+  if (!coinbase) return false
+
+  const inputScripts = (coinbase.vin || []).map(
+    input => input && (input.scriptsig || (input.scriptSig && input.scriptSig.hex))
+  )
+  const outputScripts = (coinbase.vout || []).map(
+    output => output && (output.scriptpubkey || (output.scriptPubKey && output.scriptPubKey.hex))
+  )
+
+  return [...inputScripts, ...outputScripts].some(hexHasTag)
+}