diff --git a/Cargo.lock b/Cargo.lock index dda0d4d5356a..056cabc75078 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4977,6 +4977,7 @@ dependencies = [ "sc-client-api 28.0.0", "sc-consensus", "sc-network 0.34.0", + "sc-network-bitswap", "sc-network-sync", "sc-network-transactions", "sc-rpc", @@ -17550,6 +17551,7 @@ dependencies = [ "sc-keystore", "sc-mixnet", "sc-network 0.34.0", + "sc-network-bitswap", "sc-network-common 0.33.0", "sc-network-gossip", "sc-network-light", @@ -21472,14 +21474,11 @@ dependencies = [ "litep2p 0.14.3", "log", "mockall", - "multihash-codetable", "multistream-select", "parity-scale-codec", "parking_lot 0.12.3", "partial_sort", "pin-project", - "prost 0.12.6", - "prost-build 0.13.2", "rand 0.8.5", "rustls 0.23.31", "sc-block-builder", @@ -21495,7 +21494,6 @@ dependencies = [ "sp-blockchain 28.0.0", "sp-consensus 0.32.0", "sp-core 28.0.0", - "sp-crypto-hashing 0.1.0", "sp-runtime 31.0.1", "sp-tracing 16.0.0", "substrate-prometheus-endpoint 0.17.0", @@ -21563,6 +21561,37 @@ dependencies = [ "zeroize", ] +[[package]] +name = "sc-network-bitswap" +version = "0.1.0" +dependencies = [ + "async-trait", + "cid", + "futures", + "litep2p 0.14.3", + "log", + "proptest", + "rand 0.8.5", + "rstest", + "sc-block-builder", + "sc-client-api 28.0.0", + "sc-network-common 0.33.0", + "sc-network-sync", + "sc-network-types 0.10.0", + "slotmap", + "smallvec", + "sp-consensus 0.32.0", + "sp-core 28.0.0", + "sp-crypto-hashing 0.1.0", + "sp-runtime 31.0.1", + "substrate-prometheus-endpoint 0.17.0", + "substrate-test-runtime", + "substrate-test-runtime-client", + "thiserror 1.0.65", + "tokio", + "tokio-stream", +] + [[package]] name = "sc-network-common" version = "0.33.0" @@ -22036,6 +22065,7 @@ dependencies = [ "sc-informant", "sc-keystore", "sc-network 0.34.0", + "sc-network-bitswap", "sc-network-common 0.33.0", "sc-network-light", "sc-network-sync", @@ -22160,18 +22190,13 @@ version = "0.1.0" dependencies = [ "async-trait", "cid", - "futures", - "futures-timer", "log", "parity-scale-codec", - "prost 0.12.6", - "rand 0.8.5", "rstest", "sc-client-api 28.0.0", "sc-client-db", "sc-consensus", - "sc-network 0.34.0", - "sc-network-sync", + "sc-network-bitswap", "sp-api 26.0.0", "sp-blockchain 28.0.0", "sp-consensus 0.32.0", diff --git a/Cargo.toml b/Cargo.toml index 073f0eaf4270..2fd15a54c960 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -292,6 +292,7 @@ members = [ "substrate/client/mixnet", "substrate/client/network", "substrate/client/network-gossip", + "substrate/client/network/bitswap", "substrate/client/network/common", "substrate/client/network/light", "substrate/client/network/statement", @@ -1268,6 +1269,7 @@ sc-informant = { path = "substrate/client/informant", default-features = false } sc-keystore = { path = "substrate/client/keystore", default-features = false } sc-mixnet = { path = "substrate/client/mixnet", default-features = false } sc-network = { path = "substrate/client/network", default-features = false } +sc-network-bitswap = { path = "substrate/client/network/bitswap", default-features = false } sc-network-common = { path = "substrate/client/network/common", default-features = false } sc-network-gossip = { path = "substrate/client/network-gossip", default-features = false } sc-network-light = { path = "substrate/client/network/light", default-features = false } diff --git a/cumulus/client/service/Cargo.toml b/cumulus/client/service/Cargo.toml index 11f965cc93ca..31e5408fccd9 100644 --- a/cumulus/client/service/Cargo.toml +++ b/cumulus/client/service/Cargo.toml @@ -21,6 +21,7 @@ prometheus = { workspace = true } sc-client-api = { workspace = true, default-features = true } sc-consensus = { workspace = true, default-features = true } sc-network = { workspace = true, default-features = true } +sc-network-bitswap = { workspace = true, default-features = true } sc-network-sync = { workspace = true, default-features = true } sc-network-transactions = { workspace = true, default-features = true } sc-rpc = { workspace = true, default-features = true } diff --git a/cumulus/client/service/src/lib.rs b/cumulus/client/service/src/lib.rs index 3f7efee15ea1..e44ad017eb50 100644 --- a/cumulus/client/service/src/lib.rs +++ b/cumulus/client/service/src/lib.rs @@ -323,6 +323,7 @@ pub async fn build_network<'a, Block, Client, RCInterface, IQ, Network>( TracingUnboundedSender>, TransactionsHandlerController, Arc>, + Option, )> where Block: BlockT, diff --git a/cumulus/polkadot-omni-node/lib/src/common/spec.rs b/cumulus/polkadot-omni-node/lib/src/common/spec.rs index 538fa9a48920..1cbb11b06eaf 100644 --- a/cumulus/polkadot-omni-node/lib/src/common/spec.rs +++ b/cumulus/polkadot-omni-node/lib/src/common/spec.rs @@ -52,7 +52,7 @@ use sc_network::{ use sc_service::{Configuration, ImportQueue, PartialComponents, TaskManager}; use sc_statement_store::Store; use sc_storage_chain_sync::{ - IndexedTransactionFetcher, NetworkHandle, StorageChainBlockImport, SyncingHandle, + BitswapHandleSlot, IndexedTransactionFetcher, StorageChainBlockImport, }; use sc_sysinfo::HwBench; use sc_telemetry::{TelemetryHandle, TelemetryWorker}; @@ -298,13 +298,9 @@ pub(crate) trait BaseNodeSpec { .build(), ); - let network_handle: NetworkHandle = Arc::new(OnceLock::new()); - let syncing_handle: SyncingHandle = Arc::new(OnceLock::new()); + let bitswap_slot: BitswapHandleSlot = Arc::new(OnceLock::new()); - let fetcher = IndexedTransactionFetcher::new( - Arc::clone(&network_handle), - Arc::clone(&syncing_handle), - ); + let fetcher = IndexedTransactionFetcher::new(Arc::clone(&bitswap_slot)); let storage_chain_block_import = StorageChainBlockImport::new(client.clone(), client.clone(), fetcher); @@ -335,8 +331,7 @@ pub(crate) trait BaseNodeSpec { telemetry, telemetry_worker_handle, block_import_auxiliary_data, - network_handle, - syncing_handle, + bitswap_slot, ), }) } @@ -404,8 +399,7 @@ pub(crate) trait NodeSpec: BaseNodeSpec { mut telemetry, telemetry_worker_handle, block_import_auxiliary_data, - network_handle, - syncing_handle, + bitswap_slot, ) = params.other; let client = params.client.clone(); let backend = params.backend.clone(); @@ -450,7 +444,7 @@ pub(crate) trait NodeSpec: BaseNodeSpec { (proto, config) }); - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, bitswap_handle) = build_network(BuildNetworkParams { parachain_config: ¶chain_config, net_config, @@ -466,10 +460,9 @@ pub(crate) trait NodeSpec: BaseNodeSpec { }) .await?; - let _ = network_handle - .set(network.clone() as Arc); - let _ = syncing_handle.set(sync_service.clone() - as Arc); + if let Some(handle) = bitswap_handle { + let _ = bitswap_slot.set(Arc::new(handle)); + } let peer_id = relay_chain_network.local_peer_id(); diff --git a/cumulus/polkadot-omni-node/lib/src/common/types.rs b/cumulus/polkadot-omni-node/lib/src/common/types.rs index 62b2e2898407..b90e935fe59f 100644 --- a/cumulus/polkadot-omni-node/lib/src/common/types.rs +++ b/cumulus/polkadot-omni-node/lib/src/common/types.rs @@ -60,7 +60,6 @@ pub type ParachainService = PartialCo Option, Option, BIExtraReturnValue, - sc_storage_chain_sync::NetworkHandle, - sc_storage_chain_sync::SyncingHandle, + sc_storage_chain_sync::BitswapHandleSlot, ), >; diff --git a/cumulus/polkadot-omni-node/lib/src/nodes/aura.rs b/cumulus/polkadot-omni-node/lib/src/nodes/aura.rs index 536c1a995347..3015da80145a 100644 --- a/cumulus/polkadot-omni-node/lib/src/nodes/aura.rs +++ b/cumulus/polkadot-omni-node/lib/src/nodes/aura.rs @@ -252,7 +252,7 @@ where keystore_container, select_chain: _, transaction_pool, - other: (_, mut telemetry, _, _, _, _), + other: (_, mut telemetry, _, _, _), } = Self::new_partial(&config)?; // Since this is a dev node, prevent it from connecting to peers. @@ -276,7 +276,7 @@ where (proto, *ss_config) }); - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 1031f9c93025..8d2f190bcde3 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -361,7 +361,7 @@ where .map_err(|e| sc_service::Error::Application(Box::new(e) as Box<_>))?; tracing::info!("Parachain id: {:?}", para_id); - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = build_network(BuildNetworkParams { parachain_config: ¶chain_config, net_config, diff --git a/polkadot/node/service/src/builder/mod.rs b/polkadot/node/service/src/builder/mod.rs index 403f7e342606..855861656045 100644 --- a/polkadot/node/service/src/builder/mod.rs +++ b/polkadot/node/service/src/builder/mod.rs @@ -468,7 +468,7 @@ where }) }; - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, net_config, diff --git a/prdoc/pr_12686.prdoc b/prdoc/pr_12686.prdoc new file mode 100644 index 000000000000..a69f5fb4dc7b --- /dev/null +++ b/prdoc/pr_12686.prdoc @@ -0,0 +1,31 @@ +title: Extract and refactor the Bitswap service +doc: +- audience: Node Dev + description: |- + Moves the Bitswap client and server from `sc-network` into the new + `sc-network-bitswap` crate. The new actor exposes a streaming `BitswapHandle` and + centralizes CID deduplication, peer selection, retries, response handling, and + backpressure. Storage-chain sync and node service wiring now use this handle. + + The backend-specific Bitswap shims and local protobuf plumbing are removed. Bitswap + is now supported through Litep2p only; starting a node with `--ipfs-server` and a + different network backend returns an error. +crates: +- name: sc-network-bitswap + bump: patch +- name: sc-network + bump: major +- name: sc-service + bump: major + validate: false +- name: sc-storage-chain-sync + bump: major +- name: cumulus-client-service + bump: major + validate: false +- name: polkadot-service + bump: patch +- name: polkadot-omni-node-lib + bump: major +- name: polkadot-sdk + bump: minor diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index 012234dc5032..f6289853e439 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -534,7 +534,7 @@ pub fn new_full_base::Hash>>( Vec::default(), )); - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, net_config, diff --git a/substrate/client/network/Cargo.toml b/substrate/client/network/Cargo.toml index 7ae8682a268e..6d2b62eddf59 100644 --- a/substrate/client/network/Cargo.toml +++ b/substrate/client/network/Cargo.toml @@ -46,7 +46,6 @@ parking_lot = { workspace = true, default-features = true } partial_sort = { workspace = true } pin-project = { workspace = true } prometheus-endpoint = { workspace = true, default-features = true } -prost = { workspace = true } rand = { workspace = true, default-features = true } rustls = { workspace = true } sc-client-api = { workspace = true, default-features = true } @@ -60,7 +59,6 @@ smallvec = { workspace = true, default-features = true } sp-arithmetic = { workspace = true, default-features = true } sp-blockchain = { workspace = true, default-features = true } sp-core = { workspace = true, default-features = true } -sp-crypto-hashing = { workspace = true, default-features = true } sp-runtime = { workspace = true, default-features = true } thiserror = { workspace = true } tokio = { features = ["macros", "sync"], workspace = true, default-features = true } @@ -72,7 +70,6 @@ zeroize = { workspace = true, default-features = true } [dev-dependencies] assert_matches = { workspace = true } -multihash-codetable = { workspace = true } multistream-select = { workspace = true } sc-block-builder = { workspace = true, default-features = true } sp-consensus = { workspace = true, default-features = true } @@ -85,8 +82,5 @@ tokio-util = { features = ["compat"], workspace = true } criterion = { workspace = true, default-features = true, features = ["async_tokio"] } -[build-dependencies] -prost-build = { workspace = true } - [features] default = [] diff --git a/substrate/client/network/bitswap/Cargo.toml b/substrate/client/network/bitswap/Cargo.toml new file mode 100644 index 000000000000..d2247f876bda --- /dev/null +++ b/substrate/client/network/bitswap/Cargo.toml @@ -0,0 +1,46 @@ +[package] +description = "Substrate Bitswap client/server service" +name = "sc-network-bitswap" +version = "0.1.0" +license = "GPL-3.0-or-later WITH Classpath-exception-2.0" +authors.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true +documentation = "https://docs.rs/sc-network-bitswap" + +[lints] +workspace = true + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +async-trait = { workspace = true } +cid = { workspace = true } +futures = { workspace = true } +litep2p = { workspace = true } +log = { workspace = true, default-features = true } +prometheus-endpoint = { workspace = true, default-features = true } +rand = { workspace = true, default-features = true } +sc-client-api = { workspace = true, default-features = true } +sc-network-common = { workspace = true, default-features = true } +sc-network-sync = { workspace = true, default-features = true } +slotmap = { workspace = true } +smallvec = { workspace = true, default-features = true } +sp-core = { workspace = true, default-features = true } +sp-runtime = { workspace = true, default-features = true } +thiserror = { workspace = true } +tokio = { features = ["macros", "rt", "sync", "time"], workspace = true, default-features = true } + +[dev-dependencies] +proptest = { workspace = true } +rstest = { workspace = true } +sc-block-builder = { workspace = true, default-features = true } +sc-network-types = { workspace = true, default-features = true } +sp-consensus = { workspace = true, default-features = true } +sp-crypto-hashing = { workspace = true, default-features = true } +substrate-test-runtime = { workspace = true } +substrate-test-runtime-client = { workspace = true } +tokio = { features = ["macros", "rt-multi-thread", "test-util"], workspace = true, default-features = true } +tokio-stream = { workspace = true } diff --git a/substrate/client/network/bitswap/src/handle.rs b/substrate/client/network/bitswap/src/handle.rs new file mode 100644 index 000000000000..302b1335a8d0 --- /dev/null +++ b/substrate/client/network/bitswap/src/handle.rs @@ -0,0 +1,95 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Bitswap request API. + +use super::{is_cid_supported, Cid}; + +use tokio::sync::mpsc; + +/// Bitswap request errors. +#[derive(Debug, thiserror::Error)] +pub enum BitswapError { + /// The service is unavailable. + #[error("Bitswap service is closed")] + ServiceClosed, + /// A CID is unsupported. + #[error("invalid CID for Bitswap: {cid}")] + InvalidCid { + /// Unsupported CID. + cid: Cid, + }, + /// The service is at capacity. + #[error("Bitswap service is overloaded")] + Overloaded, +} + +/// A fetched block or request error. +pub type FetchItem = Result<(Cid, Vec), BitswapError>; + +/// Handle for submitting Bitswap requests. +#[derive(Debug, Clone)] +pub struct BitswapHandle { + cmd_tx: mpsc::Sender, +} + +impl BitswapHandle { + pub(crate) fn new(cmd_tx: mpsc::Sender) -> Self { + Self { cmd_tx } + } + + /// Submit a wantlist. Returns a receiver that yields `Ok((cid, bytes))` with + /// hash-verified bytes for each requested CID, in the order they resolve. + /// + /// The stream closes once every CID has been delivered. Unresolved CIDs are retried + /// until the receiver is dropped. + /// + /// `Err(BitswapError::ServiceClosed)` is yielded once, as the final item, if the + /// service shuts down mid-request. `Err(BitswapError::Overloaded)` is yielded once as + /// the only item if too many concurrent requests want one of the CIDs. + pub fn request_stream( + &self, + cids: Vec, + ) -> Result, BitswapError> { + if cids.is_empty() { + let (_tx, rx) = mpsc::channel(1); + return Ok(rx); + } + + for cid in &cids { + if !is_cid_supported(cid) { + return Err(BitswapError::InvalidCid { cid: *cid }); + } + } + + let (sink, rx) = mpsc::channel(cids.len() + 1); + + self.cmd_tx.try_send(BitswapCommand::RequestStream { cids, sink }).map_err( + |e| match e { + mpsc::error::TrySendError::Full(_) => BitswapError::Overloaded, + mpsc::error::TrySendError::Closed(_) => BitswapError::ServiceClosed, + }, + )?; + + Ok(rx) + } +} + +#[derive(Debug)] +pub(crate) enum BitswapCommand { + RequestStream { cids: Vec, sink: mpsc::Sender }, +} diff --git a/substrate/client/network/bitswap/src/lib.rs b/substrate/client/network/bitswap/src/lib.rs new file mode 100644 index 000000000000..bfc8aaa6fa98 --- /dev/null +++ b/substrate/client/network/bitswap/src/lib.rs @@ -0,0 +1,83 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Bitswap client and server. + +use cid::Version as CidVersion; + +mod handle; +mod metrics; +mod service; + +pub use cid::Cid; +pub(crate) use handle::BitswapCommand; +pub use handle::{BitswapError, BitswapHandle, FetchItem}; +pub use service::start; + +pub(crate) const LOG_TARGET: &str = "sub-libp2p::bitswap"; + +/// Maximum entries per Bitswap message. +pub const MAX_WANTED_BLOCKS: usize = 16; + +/// IPFS raw multicodec used for indexed transaction payload bytes. +pub const RAW_CODEC: u64 = 0x55; + +/// Multihash code for BLAKE2b-256, per the multicodec table. +pub const BLAKE2B_256_MULTIHASH_CODE: u64 = 0xb220; + +/// Multihash code for SHA2-256, per the multicodec table. +pub const SHA2_256_MULTIHASH_CODE: u64 = 0x12; + +/// Multihash code for Keccak-256, per the multicodec table. +pub const KECCAK_256_MULTIHASH_CODE: u64 = 0x1b; + +/// Returns whether Bitswap supports a CID. +pub fn is_cid_supported(cid: &Cid) -> bool { + cid.version() != CidVersion::V0 && + cid.hash().size() == 32 && + matches!( + cid.hash().code(), + BLAKE2B_256_MULTIHASH_CODE | SHA2_256_MULTIHASH_CODE | KECCAK_256_MULTIHASH_CODE + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_cid_supported_accepts_all_three_supported_hashings() { + use cid::multihash::Multihash; + for multihash_code in + [BLAKE2B_256_MULTIHASH_CODE, SHA2_256_MULTIHASH_CODE, KECCAK_256_MULTIHASH_CODE] + { + let digest = [9u8; 32]; + let mh = Multihash::<64>::wrap(multihash_code, &digest).unwrap(); + let cid = Cid::new_v1(RAW_CODEC, mh); + assert!(is_cid_supported(&cid), "{multihash_code} CID should be supported"); + } + } + + #[test] + fn is_cid_supported_rejects_unknown_multihash_code() { + use cid::multihash::Multihash; + let digest = [9u8; 32]; + let mh = Multihash::<64>::wrap(0x99, &digest).unwrap(); + let cid = Cid::new_v1(RAW_CODEC, mh); + assert!(!is_cid_supported(&cid)); + } +} diff --git a/substrate/client/network/bitswap/src/metrics.rs b/substrate/client/network/bitswap/src/metrics.rs new file mode 100644 index 000000000000..2d5fcf4d8351 --- /dev/null +++ b/substrate/client/network/bitswap/src/metrics.rs @@ -0,0 +1,274 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use litep2p::protocol::libp2p::bitswap::{BlockPresenceType, ResponseType}; +use prometheus_endpoint::{ + exponential_buckets, register, Counter, CounterVec, Gauge, Histogram, HistogramOpts, Opts, + PrometheusError, Registry, U64, +}; +use std::{sync::Arc, time::Duration}; + +pub(crate) mod outcomes { + pub(crate) const BLOCK_SERVED: &str = "block_served"; + pub(crate) const HAVE: &str = "have"; + pub(crate) const DONT_HAVE: &str = "dont_have"; + pub(crate) const UNSUPPORTED_CID: &str = "unsupported_cid"; + pub(crate) const DROPPED_OVERFLOW: &str = "dropped_overflow"; +} + +pub(crate) mod errors { + pub(crate) const CLIENT: &str = "client"; +} + +pub(crate) mod outbound_events { + pub(crate) const REQUESTED: &str = "requested"; + pub(crate) const DELIVERED: &str = "delivered"; + pub(crate) const TIMED_OUT: &str = "timed_out"; + pub(crate) const ROUND_RESTARTED: &str = "round_restarted"; + pub(crate) const ABANDONED: &str = "abandoned"; + pub(crate) const OVERLOADED: &str = "overloaded"; +} + +struct Inner { + entries_total: CounterVec, + request_errors_total: CounterVec, + inbound_request_duration_seconds: Histogram, + response_bytes_total: Counter, + outbound_events_total: CounterVec, + live_cids: Gauge, + queued_cids: Gauge, + waiters: Gauge, +} + +impl Inner { + fn register(registry: &Registry) -> Result { + Ok(Self { + entries_total: register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_bitswap_entries_total", + "Total number of bitswap wantlist entries processed, by outcome", + ), + &["outcome"], + )?, + registry, + )?, + request_errors_total: register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_bitswap_request_errors_total", + "Total number of bitswap inbound requests rejected, by reason", + ), + &["reason"], + )?, + registry, + )?, + inbound_request_duration_seconds: register( + Histogram::with_opts(HistogramOpts { + common_opts: Opts::new( + "substrate_sub_libp2p_bitswap_inbound_request_duration_seconds", + "Duration of handling an inbound bitswap wantlist, in seconds", + ), + buckets: exponential_buckets(0.001, 2.0, 16) + .expect("valid histogram parameters"), + })?, + registry, + )?, + response_bytes_total: register( + Counter::new( + "substrate_sub_libp2p_bitswap_response_bytes_total", + "Total payload bytes sent in bitswap responses to inbound wantlists", + )?, + registry, + )?, + outbound_events_total: register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_bitswap_outbound_events_total", + "Total number of outbound bitswap events, by event", + ), + &["event"], + )?, + registry, + )?, + live_cids: register( + Gauge::new( + "substrate_sub_libp2p_bitswap_live_cids", + "Number of CIDs with an in-flight peer request", + )?, + registry, + )?, + queued_cids: register( + Gauge::new( + "substrate_sub_libp2p_bitswap_queued_cids", + "Number of CIDs waiting for a dispatch-window slot", + )?, + registry, + )?, + waiters: register( + Gauge::new( + "substrate_sub_libp2p_bitswap_waiters", + "Number of active outbound bitswap requests", + )?, + registry, + )?, + }) + } +} + +#[derive(Clone, Default)] +pub(crate) struct BitswapMetrics { + inner: Option>, +} + +impl BitswapMetrics { + pub(crate) fn new(registry: Option<&Registry>) -> Result { + Ok(Self { inner: registry.map(Inner::register).transpose()?.map(Arc::new) }) + } + + pub(crate) fn record_entry(&self, outcome: &str) { + self.record_entries(outcome, 1); + } + + pub(crate) fn record_entries(&self, outcome: &str, count: usize) { + if let Some(inner) = &self.inner { + inner.entries_total.with_label_values(&[outcome]).inc_by(count as u64); + } + } + + pub(crate) fn record_response(&self, response: &ResponseType) { + let outcome = match response { + ResponseType::Block { .. } => outcomes::BLOCK_SERVED, + ResponseType::Presence { presence: BlockPresenceType::Have, .. } => outcomes::HAVE, + ResponseType::Presence { presence: BlockPresenceType::DontHave, .. } => { + outcomes::DONT_HAVE + }, + }; + self.record_entry(outcome); + } + + pub(crate) fn record_error(&self, reason: &str) { + if let Some(inner) = &self.inner { + inner.request_errors_total.with_label_values(&[reason]).inc(); + } + } + + pub(crate) fn record_duration(&self, duration: Duration) { + if let Some(inner) = &self.inner { + inner.inbound_request_duration_seconds.observe(duration.as_secs_f64()); + } + } + + pub(crate) fn record_responses(&self, responses: &[ResponseType]) { + for response in responses { + self.record_response(response); + } + self.record_response_bytes(responses); + } + + pub(crate) fn record_response_bytes(&self, responses: &[ResponseType]) { + self.add_response_bytes(response_payload_bytes(responses)); + } + + pub(crate) fn record_outbound(&self, event: &str, count: usize) { + if let Some(inner) = &self.inner { + inner.outbound_events_total.with_label_values(&[event]).inc_by(count as u64); + } + } + + pub(crate) fn set_state(&self, live_cids: usize, queued_cids: usize, waiters: usize) { + if let Some(inner) = &self.inner { + inner.live_cids.set(live_cids as u64); + inner.queued_cids.set(queued_cids as u64); + inner.waiters.set(waiters as u64); + } + } + + fn add_response_bytes(&self, bytes: u64) { + if let Some(inner) = &self.inner { + inner.response_bytes_total.inc_by(bytes); + } + } +} + +fn response_payload_bytes(responses: &[ResponseType]) -> u64 { + responses + .iter() + .map(|response| match response { + ResponseType::Block { cid, block } => cid.to_bytes().len() + block.len(), + ResponseType::Presence { cid, .. } => cid.to_bytes().len(), + }) + .sum::() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use cid::{multihash::Multihash as CidMultihash, Cid}; + + fn make_cid() -> Cid { + let mh = CidMultihash::<64>::wrap(0xb220, &[0u8; 32]).unwrap(); + Cid::new_v1(0x55, mh) + } + + #[test] + fn disabled_metrics_are_no_ops() { + let metrics = BitswapMetrics::default(); + metrics.record_entry(outcomes::BLOCK_SERVED); + metrics.record_error(errors::CLIENT); + metrics.record_duration(Duration::from_millis(1)); + metrics.record_outbound(outbound_events::REQUESTED, 1); + metrics.set_state(1, 2, 3); + } + + #[test] + fn enabled_metrics_record_inbound_and_outbound_activity() { + let registry = Registry::new(); + let metrics = BitswapMetrics::new(Some(®istry)).unwrap(); + let cid = make_cid(); + let responses = [ + ResponseType::Block { cid, block: vec![1, 2, 3] }, + ResponseType::Presence { cid, presence: BlockPresenceType::DontHave }, + ]; + + metrics.record_responses(&responses); + metrics.record_error(errors::CLIENT); + metrics.record_entries(outcomes::DROPPED_OVERFLOW, 7); + metrics.record_duration(Duration::from_millis(5)); + metrics.record_outbound(outbound_events::REQUESTED, 2); + metrics.set_state(1, 2, 3); + + let inner = metrics.inner.as_ref().unwrap(); + assert_eq!(inner.entries_total.with_label_values(&[outcomes::BLOCK_SERVED]).get(), 1); + assert_eq!(inner.entries_total.with_label_values(&[outcomes::DONT_HAVE]).get(), 1); + assert_eq!(inner.entries_total.with_label_values(&[outcomes::DROPPED_OVERFLOW]).get(), 7); + assert_eq!(inner.request_errors_total.with_label_values(&[errors::CLIENT]).get(), 1); + assert_eq!( + inner + .outbound_events_total + .with_label_values(&[outbound_events::REQUESTED]) + .get(), + 2 + ); + assert_eq!(inner.live_cids.get(), 1); + assert_eq!(inner.queued_cids.get(), 2); + assert_eq!(inner.waiters.get(), 3); + assert_eq!(inner.inbound_request_duration_seconds.get_sample_count(), 1); + assert_eq!(inner.response_bytes_total.get(), response_payload_bytes(&responses)); + } +} diff --git a/substrate/client/network/bitswap/src/service.rs b/substrate/client/network/bitswap/src/service.rs new file mode 100644 index 000000000000..4680326028ac --- /dev/null +++ b/substrate/client/network/bitswap/src/service.rs @@ -0,0 +1,942 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Bitswap service. + +use super::{ + is_cid_supported, BitswapCommand, BitswapHandle, Cid, FetchItem, LOG_TARGET, MAX_WANTED_BLOCKS, +}; +use crate::{ + handle::BitswapError, + metrics::{ + errors as metric_errors, outbound_events, outcomes as metric_outcomes, BitswapMetrics, + }, +}; + +use async_trait::async_trait; +use futures::{Stream, StreamExt}; +use litep2p::protocol::libp2p::bitswap::{ + BitswapEvent, BitswapHandle as LitepBitswapHandle, BlockPresenceType, ResponseType, WantType, +}; +use prometheus_endpoint::Registry; +use rand::{seq::IteratorRandom, Rng}; +use sc_client_api::BlockBackend; +use sc_network_common::role::Roles; +use sc_network_sync::{SyncEvent, SyncEventStream}; +use slotmap::{new_key_type, SlotMap}; +use smallvec::SmallVec; +use sp_core::H256; +use sp_runtime::traits::Block as BlockT; +use std::{ + collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, + future::Future, + pin::Pin, + sync::Arc, + time::Duration, +}; +use tokio::{ + sync::{mpsc, OwnedSemaphorePermit, Semaphore}, + time::Instant, +}; + +/// Transport boundary used by the Bitswap actor. +/// Implementations must verify block bytes before emitting `VerifiedBlock`. +#[async_trait] +trait BitswapTransport: Send { + async fn next_event(&mut self) -> Option; + async fn send_request(&self, peer: litep2p::PeerId, cids: Vec<(Cid, WantType)>); + async fn send_response(&self, peer: litep2p::PeerId, responses: Vec); +} + +enum TransportEvent { + Request { peer: litep2p::PeerId, cids: Vec<(Cid, WantType)> }, + Response { peer: litep2p::PeerId, responses: Vec }, +} + +enum TransportResponse { + /// A block whose CID is guaranteed by the transport to match `bytes`. litep2p derives + /// the CID from the received bytes (the wire format only carries a CID prefix), so + /// the pairing needs no further verification; any other transport must uphold this. + VerifiedBlock { + cid: Cid, + bytes: Vec, + }, + Presence { + cid: Cid, + presence: BlockPresenceType, + }, +} + +#[async_trait] +impl BitswapTransport for LitepBitswapHandle { + async fn next_event(&mut self) -> Option { + StreamExt::next(self).await.map(|event| match event { + BitswapEvent::Request { peer, cids } => TransportEvent::Request { peer, cids }, + BitswapEvent::Response { peer, responses } => TransportEvent::Response { + peer, + responses: responses + .into_iter() + .map(|response| match response { + ResponseType::Block { cid, block } => { + TransportResponse::VerifiedBlock { cid, bytes: block } + }, + ResponseType::Presence { cid, presence } => { + TransportResponse::Presence { cid, presence } + }, + }) + .collect(), + }, + }) + } + + async fn send_request(&self, peer: litep2p::PeerId, cids: Vec<(Cid, WantType)>) { + LitepBitswapHandle::send_request(self, peer, cids).await + } + + async fn send_response(&self, peer: litep2p::PeerId, responses: Vec) { + LitepBitswapHandle::send_response(self, peer, responses).await + } +} + +const MAX_LIVE_CIDS: usize = 1024; +const MAX_WAITERS_PER_CID: usize = 64; +const MAX_CONCURRENT_INBOUND_LOOKUPS: usize = 8; +const MAX_QUEUED_INBOUND_ENTRIES_PER_PEER: usize = MAX_LIVE_CIDS; +const CMD_CHANNEL_CAPACITY: usize = 256; +const LOOKUP_CHANNEL_CAPACITY: usize = MAX_CONCURRENT_INBOUND_LOOKUPS; +const PER_PEER_TIMEOUT: Duration = Duration::from_secs(5); +const ROUND_RETRY_DELAY: Duration = Duration::from_secs(5); +const SWEEP_INTERVAL: Duration = Duration::from_secs(1); + +new_key_type! { struct WaiterId; } + +#[derive(Clone, Copy, Default)] +enum CidPhase { + #[default] + Ready, + Queued { + retry_at: Option, + }, + InFlight { + peer: litep2p::PeerId, + deadline: Instant, + }, + RetryAt(Instant), +} + +#[derive(Default)] +struct CidState { + tried_peers: HashSet, + have_peers: SmallVec<[litep2p::PeerId; 1]>, + waiters: SmallVec<[WaiterId; 2]>, + phase: CidPhase, +} + +impl CidState { + fn has_waiters(&self) -> bool { + !self.waiters.is_empty() + } + + fn is_idle(&self) -> bool { + self.waiters.is_empty() && !matches!(self.phase, CidPhase::InFlight { .. }) + } + + fn is_queued(&self) -> bool { + matches!(self.phase, CidPhase::Queued { .. }) + } + + /// Moves a ready CID into the pending queue while preserving its retry deadline. + /// Returns `false` when the CID is already queued or in flight. + fn queue(&mut self) -> bool { + let retry_at = match self.phase { + CidPhase::Ready => None, + CidPhase::RetryAt(at) => Some(at), + CidPhase::Queued { .. } | CidPhase::InFlight { .. } => return false, + }; + self.phase = CidPhase::Queued { retry_at }; + true + } + + /// Removes a CID from the pending queue and restores its prior scheduling phase. + /// Returns `false` when the state was not queued. + fn dequeue(&mut self) -> bool { + let CidPhase::Queued { retry_at } = self.phase else { return false }; + self.phase = retry_at.map_or(CidPhase::Ready, CidPhase::RetryAt); + true + } + + /// Finishes the active request only when `peer` currently owns it. + /// The return value indicates whether an in-flight slot was released. + fn finish_peer(&mut self, peer: litep2p::PeerId) -> bool { + let CidPhase::InFlight { peer: active, .. } = self.phase else { return false }; + if active != peer { + return false; + } + self.phase = CidPhase::Ready; + true + } + + /// Parks an exhausted CID until `at`, including while it waits in the queue. + /// Returns `false` when a retry is already scheduled or work remains in flight. + fn schedule_retry(&mut self, at: Instant) -> bool { + match &mut self.phase { + CidPhase::Ready => self.phase = CidPhase::RetryAt(at), + CidPhase::Queued { retry_at: slot @ None } => *slot = Some(at), + CidPhase::Queued { retry_at: Some(_) } | + CidPhase::InFlight { .. } | + CidPhase::RetryAt(_) => return false, + } + true + } + + /// Starts a new peer-selection round once the retry deadline has elapsed. + /// Peer history is cleared so every connected peer becomes eligible again. + fn restart_round(&mut self, now: Instant) -> bool { + self.phase = match self.phase { + CidPhase::RetryAt(at) if at <= now => CidPhase::Ready, + CidPhase::Queued { retry_at: Some(at) } if at <= now => { + CidPhase::Queued { retry_at: None } + }, + _ => return false, + }; + self.tried_peers.clear(); + self.have_peers.clear(); + true + } +} + +struct WantSet { + inner: HashMap, + /// FIFO of CIDs waiting for a free dispatch-window slot. May contain stale entries + /// (resolved, abandoned or already-dispatched CIDs); those are skipped on pop, guarded + /// by [`CidState::is_queued`]. + pending: VecDeque, + /// Number of CIDs with at least one in-flight peer request. + live: usize, + /// Number of queued [`CidState`] entries, maintained incrementally: the + /// metrics path reads it after every actor event, so recomputing it by scanning + /// `inner` would make large requests quadratic. + queued: usize, + /// Dispatch-window size. + max_live: usize, +} + +impl WantSet { + fn new(max_live: usize) -> Self { + Self { inner: HashMap::new(), pending: VecDeque::new(), live: 0, queued: 0, max_live } + } + + fn contains(&self, cid: &Cid) -> bool { + self.inner.contains_key(cid) + } + + fn waiter_count(&self, cid: &Cid) -> usize { + self.inner.get(cid).map_or(0, |state| state.waiters.len()) + } + + fn add_waiter(&mut self, cid: Cid, waiter: WaiterId) { + self.inner.entry(cid).or_default().waiters.push(waiter); + } + + /// Detaches a user request from `cid` and removes newly idle state. + /// In-flight state remains until its response, timeout, or disconnect arrives. + fn remove_waiter(&mut self, cid: Cid, waiter: WaiterId) { + if let Some(state) = self.inner.get_mut(&cid) { + state.waiters.retain(|w| *w != waiter); + } + self.remove_if_idle(cid); + } + + fn all_cids(&self) -> Vec { + self.inner.keys().copied().collect() + } + + /// Removes a delivered CID and returns every waiter that should receive it. + /// Scheduler counters are released according to the CID's previous phase. + fn take_waiters_for_delivered_cid(&mut self, cid: Cid) -> Option> { + self.inner.remove(&cid).map(|state| { + if matches!(state.phase, CidPhase::InFlight { .. }) { + self.live -= 1; + } + if state.is_queued() { + self.queued -= 1; + } + state.waiters + }) + } + + fn has_window_capacity(&self) -> bool { + self.live < self.max_live + } + + /// Pops the next still-valid queued CID and updates its queue accounting. + /// Stale FIFO entries are skipped until a queued state is found. + fn pop_pending(&mut self) -> Option { + while let Some(cid) = self.pending.pop_front() { + if let Some(state) = self.inner.get_mut(&cid) { + if state.dequeue() { + self.queued -= 1; + return Some(cid); + } + } + } + None + } + + /// Selects an eligible peer for `cid`, queueing it when the dispatch window is full. + /// A selected peer owns one live slot until completion, timeout, or disconnect. + fn next_peer_to_request( + &mut self, + cid: Cid, + connected_peers: &HashSet, + now: Instant, + rng: &mut R, + ) -> Option { + let has_window_capacity = self.has_window_capacity(); + let state = self.inner.get_mut(&cid)?; + if !state.has_waiters() || matches!(state.phase, CidPhase::InFlight { .. }) { + return None; + } + + if !has_window_capacity { + // Preserve the CID for promotion when a dispatch slot opens. + if state.queue() { + self.pending.push_back(cid); + self.queued += 1; + } + return None; + } + + // Prefer peers that advertised HAVE before falling back to any untried peer. + let eligible = |peer: &litep2p::PeerId| !state.tried_peers.contains(peer); + let Some(peer) = state + .have_peers + .iter() + .filter(|peer| connected_peers.contains(*peer) && eligible(peer)) + .choose(&mut *rng) + .or_else(|| connected_peers.iter().filter(|peer| eligible(peer)).choose(&mut *rng)) + .copied() + else { + if !connected_peers.is_empty() { + // Retry later once every connected peer has been tried. + if state.schedule_retry(now + ROUND_RETRY_DELAY) { + log::trace!( + target: LOG_TARGET, + "all peers tried for {cid}, scheduling new round", + ); + } + } + return None; + }; + + if state.is_queued() { + self.queued -= 1; + } + self.live += 1; + state.phase = CidPhase::InFlight { peer, deadline: now + PER_PEER_TIMEOUT }; + + Some(peer) + } + + /// Records a terminal response from `peer` and releases its active slot. + /// Late responses cannot finish a newer request owned by a different peer. + fn mark_peer_done_for_cid(&mut self, peer: litep2p::PeerId, cid: Cid) { + if let Some(state) = self.inner.get_mut(&cid) { + if state.finish_peer(peer) { + self.live -= 1; + } + state.tried_peers.insert(peer); + } + self.remove_if_idle(cid); + } + + /// Records that `peer` has a CID and releases its active request. + /// A repeated `HAVE` marks that peer tried so another peer is selected next. + fn note_peer_have_for_cid(&mut self, peer: litep2p::PeerId, cid: Cid) { + if let Some(state) = self.inner.get_mut(&cid) { + if state.have_peers.contains(&peer) { + state.tried_peers.insert(peer); + } else { + state.have_peers.push(peer); + } + if state.finish_peer(peer) { + self.live -= 1; + } + } + self.remove_if_idle(cid); + } + + /// Releases every active request owned by a disconnected peer. + /// Returned CIDs are still wanted and ready for immediate failover. + fn remove_in_flight_peer(&mut self, peer: litep2p::PeerId) -> Vec { + let mut affected = Vec::new(); + for (cid, state) in self.inner.iter_mut() { + if state.finish_peer(peer) { + self.live -= 1; + affected.push(*cid); + } + } + + self.remove_idle_and_filter_existing(affected) + } + + /// Expire in-flight requests whose per-peer deadline passed. Returns the affected CIDs + /// still wanted (for re-dispatch) and the total number of timed-out peer requests. + fn expire_peer_timeouts(&mut self, now: Instant) -> (Vec, usize) { + let mut timed_out = Vec::new(); + for (cid, state) in self.inner.iter_mut() { + if let CidPhase::InFlight { peer, deadline } = state.phase { + if deadline <= now { + state.phase = CidPhase::Ready; + state.tried_peers.insert(peer); + timed_out.push(*cid); + } + } + } + let timed_out_count = timed_out.len(); + self.live -= timed_out_count; + + (self.remove_idle_and_filter_existing(timed_out), timed_out_count) + } + + /// Restarts every exhausted CID whose retry delay has elapsed. + /// Returned CIDs should be reconsidered for immediate dispatch. + fn restart_exhausted_rounds(&mut self, now: Instant) -> Vec { + let mut cids = Vec::new(); + for (cid, state) in self.inner.iter_mut() { + if state.has_waiters() && state.restart_round(now) { + cids.push(*cid); + } + } + cids + } + + fn clear(&mut self) { + self.inner.clear(); + self.pending.clear(); + self.live = 0; + self.queued = 0; + } + + /// Removes idle states from `cids` and returns those still tracked. + /// The filtered result is safe to feed back into peer selection. + fn remove_idle_and_filter_existing(&mut self, cids: Vec) -> Vec { + for cid in &cids { + self.remove_if_idle(*cid); + } + cids.into_iter().filter(|cid| self.inner.contains_key(cid)).collect() + } + + fn remove_if_idle(&mut self, cid: Cid) { + let Entry::Occupied(entry) = self.inner.entry(cid) else { return }; + if !entry.get().is_idle() { + return; + } + let state = entry.remove(); + if state.is_queued() { + self.queued -= 1; + } + } +} + +struct Waiter { + cids_remaining: HashSet, + sink: mpsc::Sender, +} + +/// A served batch, carrying the worker permit: the slot is released only once the actor +/// has forwarded the responses, so downstream backpressure reaches the dispatch path and +/// at most [`MAX_CONCURRENT_INBOUND_LOOKUPS`] responses are retained at any time. +type InboundLookupResult = (litep2p::PeerId, Vec, OwnedSemaphorePermit); + +struct InboundLookupPool { + client: Arc + Send + Sync>, + result_tx: mpsc::Sender, + semaphore: Arc, + metrics: BitswapMetrics, +} + +impl InboundLookupPool { + fn new( + client: Arc + Send + Sync>, + max_lookups: usize, + metrics: BitswapMetrics, + ) -> (Self, mpsc::Receiver) { + let (result_tx, result_rx) = mpsc::channel(LOOKUP_CHANNEL_CAPACITY); + ( + Self { client, result_tx, semaphore: Arc::new(Semaphore::new(max_lookups)), metrics }, + result_rx, + ) + } + + /// Reserves a worker slot, or returns `None` if all workers are busy. + /// Dropping an unused permit immediately returns the slot. + fn try_acquire_worker(&self) -> Option { + self.semaphore.clone().try_acquire_owned().ok() + } + + /// Serves a wantlist on a blocking worker occupying `permit`'s slot. + /// The permit travels with the result so backpressure retains the slot. + fn submit( + &self, + permit: OwnedSemaphorePermit, + peer: litep2p::PeerId, + cids: Vec<(Cid, WantType)>, + ) { + let client = self.client.clone(); + let result_tx = self.result_tx.clone(); + let metrics = self.metrics.clone(); + tokio::task::spawn_blocking(move || { + let responses = serve_inbound(&*client, cids, &metrics); + let _ = result_tx.blocking_send((peer, responses, permit)); + }); + } +} + +/// Fair per-peer queues. Overflow is dropped rather than reported as `DONT_HAVE`. +struct InboundQueue { + per_peer: HashMap>, + rotation: VecDeque, + max_entries_per_peer: usize, +} + +impl InboundQueue { + fn new(max_entries_per_peer: usize) -> Self { + Self { per_peer: HashMap::new(), rotation: VecDeque::new(), max_entries_per_peer } + } + + /// Queue `entries` for `peer`. Returns the number of entries dropped because the + /// peer's queue is full. + fn enqueue(&mut self, peer: litep2p::PeerId, mut entries: Vec<(Cid, WantType)>) -> usize { + if entries.is_empty() { + return 0; + } + let queue = self.per_peer.entry(peer).or_default(); + let free = self.max_entries_per_peer.saturating_sub(queue.len()); + let dropped = entries.len().saturating_sub(free); + entries.truncate(free); + // Enter the rotation only on the empty-to-nonempty transition. + if queue.is_empty() && !entries.is_empty() { + self.rotation.push_back(peer); + } + queue.extend(entries); + if queue.is_empty() { + self.per_peer.remove(&peer); + } + dropped + } + + /// Take the next batch of up to [`MAX_WANTED_BLOCKS`] entries, rotating the serviced + /// peer to the back of the queue. + fn next_batch(&mut self) -> Option<(litep2p::PeerId, Vec<(Cid, WantType)>)> { + while let Some(peer) = self.rotation.pop_front() { + let Some(queue) = self.per_peer.get_mut(&peer) else { + log::error!(target: LOG_TARGET, "stale peer in inbound queue rotation: {peer}"); + continue; + }; + if queue.is_empty() { + log::error!(target: LOG_TARGET, "empty peer queue in inbound rotation: {peer}"); + self.per_peer.remove(&peer); + continue; + } + + let batch: Vec<_> = queue.drain(..queue.len().min(MAX_WANTED_BLOCKS)).collect(); + if queue.is_empty() { + self.per_peer.remove(&peer); + } else { + self.rotation.push_back(peer); + } + return Some((peer, batch)); + } + None + } +} + +pub(crate) struct BitswapService { + handle: Box, + + cmd_rx: mpsc::Receiver, + cmd_channel_closed: bool, + sync_event_stream: Pin + Send>>, + inbound_lookup_pool: InboundLookupPool, + inbound_lookup_rx: mpsc::Receiver, + inbound_queue: InboundQueue, + + connected_peers: HashSet, + wants: WantSet, + waiters: SlotMap, + metrics: BitswapMetrics, +} + +/// Build the Bitswap service, returning the service future and the user-facing handle. +pub fn start( + client: Arc + Send + Sync>, + sync: &S, + litep2p_handle: LitepBitswapHandle, + metrics_registry: Option<&Registry>, +) -> (Pin + Send>>, BitswapHandle) +where + S: SyncEventStream + ?Sized, +{ + let (cmd_tx, cmd_rx) = mpsc::channel(CMD_CHANNEL_CAPACITY); + let metrics = BitswapMetrics::new(metrics_registry).unwrap_or_else(|err| { + log::debug!(target: LOG_TARGET, "failed to register bitswap metrics: {err}"); + BitswapMetrics::default() + }); + let (inbound_lookup_pool, inbound_lookup_rx) = + InboundLookupPool::new(client, MAX_CONCURRENT_INBOUND_LOOKUPS, metrics.clone()); + + let user_handle = BitswapHandle::new(cmd_tx); + let sync_event_stream = sync.event_stream("bitswap"); + + let service = BitswapService { + handle: Box::new(litep2p_handle), + cmd_rx, + cmd_channel_closed: false, + sync_event_stream, + inbound_lookup_pool, + inbound_lookup_rx, + inbound_queue: InboundQueue::new(MAX_QUEUED_INBOUND_ENTRIES_PER_PEER), + connected_peers: HashSet::new(), + wants: WantSet::new(MAX_LIVE_CIDS), + waiters: SlotMap::with_key(), + metrics, + }; + + let future = Box::pin(async move { service.run().await }); + + (future, user_handle) +} + +impl BitswapService { + /// Runs the service actor until a required input stream closes. + /// Each event is handled serially before metrics are refreshed. + async fn run(mut self) { + log::debug!(target: LOG_TARGET, "BitswapService starting"); + let mut sweep_ticker = tokio::time::interval(SWEEP_INTERVAL); + sweep_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + sweep_ticker.tick().await; + self.update_metrics(); + + loop { + tokio::select! { + event = self.handle.next_event() => match event { + Some(TransportEvent::Request { peer, cids }) => + self.on_inbound_request(peer, cids), + Some(TransportEvent::Response { peer, responses }) => + self.on_inbound_response(peer, responses).await, + None => { + log::debug!(target: LOG_TARGET, "litep2p bitswap stream ended; shutting down"); + self.shutdown_waiters(); + return; + }, + }, + + cmd = self.cmd_rx.recv(), if !self.cmd_channel_closed => match cmd { + Some(BitswapCommand::RequestStream { cids, sink }) => + self.on_request_stream(cids, sink).await, + None => { + log::debug!( + target: LOG_TARGET, + "all bitswap handles dropped; serving inbound requests only", + ); + self.cmd_channel_closed = true; + }, + }, + + sync_ev = self.sync_event_stream.next() => match sync_ev { + Some(SyncEvent::PeerConnected { peer_id, roles }) => + self.on_peer_connected(peer_id.into(), roles).await, + Some(SyncEvent::PeerDisconnected(peer)) => self.on_peer_disconnected(peer.into()).await, + None => { + log::debug!(target: LOG_TARGET, "sync event stream ended; shutting down"); + self.shutdown_waiters(); + return; + }, + }, + + Some((peer, responses, permit)) = self.inbound_lookup_rx.recv() => { + self.handle.send_response(peer, responses).await; + drop(permit); + self.dispatch_inbound_lookups(); + }, + + _ = sweep_ticker.tick() => { + self.on_sweep().await; + }, + } + self.update_metrics(); + } + } + + /// Admits a user wantlist and attaches one waiter to all requested CIDs. + /// Requests exceeding the per-CID waiter limit are rejected as overloaded. + async fn on_request_stream(&mut self, cids: Vec, sink: mpsc::Sender) { + for cid in &cids { + if self.wants.waiter_count(cid) >= MAX_WAITERS_PER_CID { + self.metrics.record_outbound(outbound_events::OVERLOADED, 1); + let _ = sink.try_send(Err(BitswapError::Overloaded)); + return; + } + } + + let cids_remaining: HashSet = cids.iter().copied().collect(); + let waiter_id = self.waiters.insert(Waiter { cids_remaining, sink }); + + for cid in &cids { + self.wants.add_waiter(*cid, waiter_id); + } + + self.top_up_in_flight(cids).await; + } + + /// Schedules the supplied CIDs, then promotes queued CIDs until the dispatch window is full. + /// Eligible wants are grouped by peer and sent in protocol-sized batches. + async fn top_up_in_flight(&mut self, cids: impl IntoIterator) { + let now = Instant::now(); + let by_peer = { + let mut rng = rand::thread_rng(); + let mut by_peer: HashMap> = HashMap::new(); + for cid in cids { + if let Some(peer) = + self.wants.next_peer_to_request(cid, &self.connected_peers, now, &mut rng) + { + log::trace!(target: LOG_TARGET, "WANT-BLOCK {cid} -> {peer:?}"); + by_peer.entry(peer).or_default().push((cid, WantType::Block)); + } + } + + // Fill remaining dispatch slots from the pending FIFO. + while self.wants.has_window_capacity() { + let Some(cid) = self.wants.pop_pending() else { break }; + if let Some(peer) = + self.wants.next_peer_to_request(cid, &self.connected_peers, now, &mut rng) + { + log::trace!(target: LOG_TARGET, "WANT-BLOCK {cid} -> {peer:?} (promoted)"); + by_peer.entry(peer).or_default().push((cid, WantType::Block)); + } + } + by_peer + }; + + for (peer, wants) in by_peer { + self.metrics.record_outbound(outbound_events::REQUESTED, wants.len()); + for chunk in wants.chunks(MAX_WANTED_BLOCKS) { + self.handle.send_request(peer, chunk.to_vec()).await; + } + } + } + + /// Applies verified blocks and presence updates received from a peer. + /// Unresolved presence responses are immediately reconsidered for dispatch. + async fn on_inbound_response( + &mut self, + peer: litep2p::PeerId, + responses: Vec, + ) { + let mut cids_to_top_up: HashSet = HashSet::new(); + + for response in responses { + match response { + TransportResponse::VerifiedBlock { cid, bytes } => { + // A late verified block may satisfy a want reassigned to another peer. + self.wants.mark_peer_done_for_cid(peer, cid); + + if self.wants.contains(&cid) { + self.deliver_block(cid, bytes); + } else { + log::debug!( + target: LOG_TARGET, + "{peer:?} sent unsolicited or unwanted block for {cid}", + ); + } + }, + TransportResponse::Presence { cid, presence } => { + match presence { + BlockPresenceType::DontHave => { + log::trace!(target: LOG_TARGET, "{peer:?} DONT_HAVE {cid}"); + self.wants.mark_peer_done_for_cid(peer, cid); + }, + BlockPresenceType::Have => { + log::trace!(target: LOG_TARGET, "{peer:?} HAVE {cid}"); + self.wants.note_peer_have_for_cid(peer, cid); + }, + } + cids_to_top_up.insert(cid); + }, + } + } + + self.top_up_in_flight(cids_to_top_up).await; + } + + /// Delivers a resolved block to every waiter sharing its CID. + /// The CID is removed once, releasing any replacement in-flight request. + fn deliver_block(&mut self, cid: Cid, bytes: Vec) { + let Some(waiter_ids) = self.wants.take_waiters_for_delivered_cid(cid) else { return }; + self.metrics.record_outbound(outbound_events::DELIVERED, 1); + + for waiter_id in waiter_ids { + let Some(waiter) = self.waiters.get_mut(waiter_id) else { continue }; + if !waiter.cids_remaining.remove(&cid) { + continue; + } + if waiter.sink.try_send(Ok((cid, bytes.clone()))).is_err() { + log::trace!(target: LOG_TARGET, "waiter sink full/closed for {cid}"); + self.drop_waiter(waiter_id); + continue; + } + if waiter.cids_remaining.is_empty() { + self.drop_waiter(waiter_id); + } + } + } + + /// Removes a waiter and detaches it from every unresolved CID. + /// CID state is retained only while another waiter or request needs it. + fn drop_waiter(&mut self, id: WaiterId) { + let Some(waiter) = self.waiters.remove(id) else { return }; + + for cid in &waiter.cids_remaining { + self.wants.remove_waiter(*cid, id); + } + } + + /// Adds a connected non-light peer and reconsiders every outstanding CID. + /// Light peers are ignored because they do not store indexed transactions. + async fn on_peer_connected(&mut self, peer: litep2p::PeerId, roles: Roles) { + if roles.is_light() { + return; + } + self.connected_peers.insert(peer); + let cids = self.wants.all_cids(); + self.top_up_in_flight(cids).await; + } + + /// Removes a peer and immediately fails over requests it owned. + /// Late responses remain safe because peer ownership is checked on completion. + async fn on_peer_disconnected(&mut self, peer: litep2p::PeerId) { + self.connected_peers.remove(&peer); + let cids_to_top_up = self.wants.remove_in_flight_peer(peer); + self.top_up_in_flight(cids_to_top_up).await; + } + + /// Periodic housekeeping: drop waiters whose receiver was dropped, expire per-peer request + /// timeouts, and start new rounds for CIDs whose retry delay has passed. + async fn on_sweep(&mut self) { + let abandoned: Vec = self + .waiters + .iter() + .filter_map(|(id, waiter)| waiter.sink.is_closed().then_some(id)) + .collect(); + self.metrics.record_outbound(outbound_events::ABANDONED, abandoned.len()); + for id in abandoned { + log::trace!(target: LOG_TARGET, "dropping abandoned waiter {id:?}"); + self.drop_waiter(id); + } + + let now = Instant::now(); + let (mut cids, timed_out) = self.wants.expire_peer_timeouts(now); + self.metrics.record_outbound(outbound_events::TIMED_OUT, timed_out); + let restarted = self.wants.restart_exhausted_rounds(now); + self.metrics.record_outbound(outbound_events::ROUND_RESTARTED, restarted.len()); + cids.extend(restarted); + self.top_up_in_flight(cids).await; + + // Recover capacity after a lookup task exits without a result. + self.dispatch_inbound_lookups(); + } + + /// Queues an inbound peer wantlist and records entries dropped by backpressure. + /// Available lookup workers are dispatched immediately after admission. + fn on_inbound_request(&mut self, peer: litep2p::PeerId, cids: Vec<(Cid, WantType)>) { + let dropped = self.inbound_queue.enqueue(peer, cids); + if dropped > 0 { + self.metrics.record_entries(metric_outcomes::DROPPED_OVERFLOW, dropped); + log::debug!( + target: LOG_TARGET, + "inbound queue for {peer:?} full; dropped {dropped} wantlist entries", + ); + } + self.dispatch_inbound_lookups(); + } + + /// Starts queued inbound lookups while worker permits remain available. + /// Work stays queued when either the pool is full or no batch is ready. + fn dispatch_inbound_lookups(&mut self) { + while let Some(permit) = self.inbound_lookup_pool.try_acquire_worker() { + let Some((peer, batch)) = self.inbound_queue.next_batch() else { break }; + self.inbound_lookup_pool.submit(permit, peer, batch); + } + } + + fn update_metrics(&self) { + self.metrics.set_state(self.wants.live, self.wants.queued, self.waiters.len()); + } + + /// Notifies every waiter that the service closed and clears scheduler state. + /// Send failures are ignored because the corresponding receiver already vanished. + fn shutdown_waiters(&mut self) { + for (_, waiter) in self.waiters.drain() { + let _ = waiter.sink.try_send(Err(BitswapError::ServiceClosed)); + } + self.wants.clear(); + self.update_metrics(); + } +} + +/// Resolves an inbound wantlist against locally indexed transactions. +/// Unsupported CIDs and missing transactions become `DONT_HAVE` responses. +fn serve_inbound( + client: &(dyn BlockBackend + Send + Sync), + cids: Vec<(Cid, WantType)>, + metrics: &BitswapMetrics, +) -> Vec { + let started = std::time::Instant::now(); + let responses = cids + .into_iter() + .map(|(cid, want_type)| { + if !is_cid_supported(&cid) { + metrics.record_entry(metric_outcomes::UNSUPPORTED_CID); + return ResponseType::Presence { cid, presence: BlockPresenceType::DontHave }; + } + // Supported CIDs always carry a 32-byte digest. + let hash = H256::from_slice(&cid.hash().digest()[0..32]); + let transaction = match client.indexed_transaction(hash) { + Ok(t) => t, + Err(e) => { + metrics.record_error(metric_errors::CLIENT); + log::error!(target: LOG_TARGET, "indexed_transaction({hash}) failed: {e}"); + None + }, + }; + match (transaction, want_type) { + (Some(transaction), WantType::Block) => { + ResponseType::Block { cid, block: transaction } + }, + (Some(_), _) => ResponseType::Presence { cid, presence: BlockPresenceType::Have }, + (None, _) => ResponseType::Presence { cid, presence: BlockPresenceType::DontHave }, + } + }) + .collect::>(); + metrics.record_responses(&responses); + metrics.record_duration(started.elapsed()); + responses +} + +#[cfg(test)] +mod tests; diff --git a/substrate/client/network/bitswap/src/service/tests.rs b/substrate/client/network/bitswap/src/service/tests.rs new file mode 100644 index 000000000000..395e1cc3bc4f --- /dev/null +++ b/substrate/client/network/bitswap/src/service/tests.rs @@ -0,0 +1,1204 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Bitswap service tests. + +use super::*; +use crate::{ + BLAKE2B_256_MULTIHASH_CODE, KECCAK_256_MULTIHASH_CODE, RAW_CODEC, SHA2_256_MULTIHASH_CODE, +}; +use cid::multihash::Multihash as CidMultihash; +use rand::{rngs::StdRng, SeedableRng}; +use rstest::rstest; +use sc_block_builder::BlockBuilderBuilder; +use sc_network_sync::SyncEvent; +use sc_network_types::PeerId as TypesPeerId; +use sp_consensus::BlockOrigin; +use sp_runtime::codec::Encode; +use substrate_test_runtime::ExtrinsicBuilder; +use substrate_test_runtime_client::{prelude::*, TestClientBuilder}; +use tokio::{ + sync::Mutex as AsyncMutex, + time::{sleep, timeout}, +}; + +struct MockTransport { + inbound: AsyncMutex>, + outbound_req_tx: mpsc::Sender<(litep2p::PeerId, Vec<(Cid, WantType)>)>, + outbound_resp_tx: mpsc::Sender<(litep2p::PeerId, Vec)>, +} + +#[async_trait] +impl BitswapTransport for MockTransport { + async fn next_event(&mut self) -> Option { + self.inbound.get_mut().recv().await + } + + async fn send_request(&self, peer: litep2p::PeerId, cids: Vec<(Cid, WantType)>) { + let _ = self.outbound_req_tx.send((peer, cids)).await; + } + + async fn send_response(&self, peer: litep2p::PeerId, responses: Vec) { + let _ = self.outbound_resp_tx.send((peer, responses)).await; + } +} + +struct TestRig { + user_handle: BitswapHandle, + sync_event_tx: mpsc::Sender, + inbound_tx: mpsc::Sender, + outbound_req_rx: mpsc::Receiver<(litep2p::PeerId, Vec<(Cid, WantType)>)>, + outbound_resp_rx: mpsc::Receiver<(litep2p::PeerId, Vec)>, + _handle: tokio::task::JoinHandle<()>, +} + +fn build_rig_with( + client: Arc + Send + Sync>, + max_live_cids: usize, +) -> TestRig { + build_rig_with_inbound_limits( + client, + max_live_cids, + MAX_CONCURRENT_INBOUND_LOOKUPS, + MAX_QUEUED_INBOUND_ENTRIES_PER_PEER, + ) +} + +fn build_rig_with_inbound_limits( + client: Arc + Send + Sync>, + max_live_cids: usize, + max_lookups: usize, + queued_entries_per_peer: usize, +) -> TestRig { + let (inbound_tx, inbound_rx) = mpsc::channel(64); + let (outbound_req_tx, outbound_req_rx) = mpsc::channel(64); + let (outbound_resp_tx, outbound_resp_rx) = mpsc::channel(64); + let (cmd_tx, cmd_rx) = mpsc::channel(CMD_CHANNEL_CAPACITY); + let (sync_event_tx, sync_event_rx) = mpsc::channel::(64); + let metrics = BitswapMetrics::default(); + let (inbound_lookup_pool, inbound_lookup_rx) = + InboundLookupPool::new(client, max_lookups, metrics.clone()); + + let transport = + MockTransport { inbound: AsyncMutex::new(inbound_rx), outbound_req_tx, outbound_resp_tx }; + + let sync_event_stream: Pin + Send>> = + Box::pin(tokio_stream::wrappers::ReceiverStream::new(sync_event_rx)); + + let service: BitswapService = BitswapService { + handle: Box::new(transport), + cmd_rx, + cmd_channel_closed: false, + sync_event_stream, + inbound_lookup_pool, + inbound_lookup_rx, + inbound_queue: InboundQueue::new(queued_entries_per_peer), + connected_peers: HashSet::new(), + wants: WantSet::new(max_live_cids), + waiters: SlotMap::with_key(), + metrics, + }; + + let user_handle = BitswapHandle::new(cmd_tx); + let _handle = tokio::spawn(async move { service.run().await }); + + TestRig { user_handle, sync_event_tx, inbound_tx, outbound_req_rx, outbound_resp_rx, _handle } +} + +fn empty_rig() -> TestRig { + small_window_rig(MAX_LIVE_CIDS) +} + +fn small_window_rig(max_live_cids: usize) -> TestRig { + let client = Arc::new(substrate_test_runtime_client::new()); + build_rig_with(client, max_live_cids) +} + +fn cid_for_data(mh_code: u64, data: &[u8]) -> Cid { + let digest = match mh_code { + BLAKE2B_256_MULTIHASH_CODE => sp_crypto_hashing::blake2_256(data), + SHA2_256_MULTIHASH_CODE => sp_crypto_hashing::sha2_256(data), + KECCAK_256_MULTIHASH_CODE => sp_crypto_hashing::keccak_256(data), + _ => panic!("unsupported multihash code"), + }; + let mh = CidMultihash::<64>::wrap(mh_code, &digest).unwrap(); + Cid::new_v1(RAW_CODEC, mh) +} + +fn cid_for_digest(mh_code: u64, digest: [u8; 32]) -> Cid { + let mh = CidMultihash::<64>::wrap(mh_code, &digest).unwrap(); + Cid::new_v1(RAW_CODEC, mh) +} + +async fn drain_next(rx: &mut mpsc::Receiver) -> Option { + timeout(Duration::from_secs(2), rx.recv()).await.ok().flatten() +} + +fn to_types_peer(peer: litep2p::PeerId) -> TypesPeerId { + TypesPeerId::from_bytes(&peer.to_bytes()).expect("peer ID bytes are valid") +} + +fn sync_connected(peer: litep2p::PeerId) -> SyncEvent { + SyncEvent::PeerConnected { peer_id: to_types_peer(peer), roles: Roles::FULL } +} + +fn sync_connected_light(peer: litep2p::PeerId) -> SyncEvent { + SyncEvent::PeerConnected { peer_id: to_types_peer(peer), roles: Roles::LIGHT } +} + +fn sync_disconnected(peer: litep2p::PeerId) -> SyncEvent { + SyncEvent::PeerDisconnected(to_types_peer(peer)) +} + +impl TestRig { + async fn connect(&self, peer: litep2p::PeerId) { + self.sync_event_tx.send(sync_connected(peer)).await.unwrap(); + } + + async fn send_response(&self, peer: litep2p::PeerId, responses: Vec) { + self.inbound_tx + .send(TransportEvent::Response { peer, responses }) + .await + .unwrap(); + } + + async fn send_block(&self, peer: litep2p::PeerId, cid: Cid, data: &[u8]) { + self.send_response( + peer, + vec![TransportResponse::VerifiedBlock { cid, bytes: data.to_vec() }], + ) + .await; + } + + async fn send_presence(&self, peer: litep2p::PeerId, cid: Cid, presence: BlockPresenceType) { + self.send_response(peer, vec![TransportResponse::Presence { cid, presence }]) + .await; + } + + async fn send_wantlist(&self, peer: litep2p::PeerId, cids: Vec<(Cid, WantType)>) { + self.inbound_tx.send(TransportEvent::Request { peer, cids }).await.unwrap(); + } +} + +async fn expect_block(rx: &mut mpsc::Receiver, cid: Cid, data: &[u8]) { + match drain_next(rx).await.expect("stream item") { + Ok((got_cid, bytes)) => { + assert_eq!(got_cid, cid); + assert_eq!(bytes, data); + }, + other => panic!("expected block, got {other:?}"), + } +} + +fn assert_single_dont_have(responses: &[ResponseType], cid: Cid) { + assert!(matches!( + responses, + [ResponseType::Presence { cid: got, presence: BlockPresenceType::DontHave }] + if *got == cid + )); +} + +#[test] +fn want_set_removes_cid_after_last_waiter_and_peer_complete() { + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0xaa; 32]); + let peer = litep2p::PeerId::random(); + let mut waiter_ids = SlotMap::with_key(); + let waiter_id = waiter_ids.insert(()); + let mut wants = WantSet::new(MAX_LIVE_CIDS); + let mut rng = StdRng::seed_from_u64(0); + + wants.add_waiter(cid, waiter_id); + let selected = wants + .next_peer_to_request(cid, &HashSet::from([peer]), Instant::now(), &mut rng) + .unwrap(); + assert_eq!(selected, peer); + + wants.remove_waiter(cid, waiter_id); + assert!(wants.contains(&cid)); + + wants.mark_peer_done_for_cid(peer, cid); + assert!(!wants.contains(&cid)); +} + +#[test] +fn want_set_window_queues_and_promotes() { + let cid_a = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0x01; 32]); + let cid_b = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0x02; 32]); + let peer = litep2p::PeerId::random(); + let peers = HashSet::from([peer]); + let mut waiter_ids = SlotMap::with_key(); + let waiter_id = waiter_ids.insert(()); + let mut wants = WantSet::new(1); + let mut rng = StdRng::seed_from_u64(0); + + wants.add_waiter(cid_a, waiter_id); + wants.add_waiter(cid_b, waiter_id); + + assert_eq!(wants.next_peer_to_request(cid_a, &peers, Instant::now(), &mut rng), Some(peer)); + assert_eq!(wants.next_peer_to_request(cid_b, &peers, Instant::now(), &mut rng), None); + assert!(!wants.has_window_capacity()); + + wants.take_waiters_for_delivered_cid(cid_a); + assert!(wants.has_window_capacity()); + assert_eq!(wants.pop_pending(), Some(cid_b)); + assert_eq!(wants.next_peer_to_request(cid_b, &peers, Instant::now(), &mut rng), Some(peer)); + assert_eq!(wants.pop_pending(), None); +} + +#[test] +fn peer_selection_varies_across_cids() { + let peers: HashSet<_> = (0..3).map(|_| litep2p::PeerId::random()).collect(); + let mut waiter_ids = SlotMap::with_key(); + let mut wants = WantSet::new(32); + let mut rng = StdRng::seed_from_u64(0); + let mut selected = HashSet::new(); + + for byte in 0..32 { + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [byte; 32]); + let waiter = waiter_ids.insert(()); + wants.add_waiter(cid, waiter); + selected.insert( + wants + .next_peer_to_request(cid, &peers, Instant::now(), &mut rng) + .expect("a connected peer is eligible"), + ); + } + + assert!(selected.len() > 1, "fresh CIDs should not all select the same peer"); +} + +#[test] +fn inbound_queue_rotates_peers_between_batches() { + let peer_a = litep2p::PeerId::random(); + let peer_b = litep2p::PeerId::random(); + let entry = |i: u8| (cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [i; 32]), WantType::Block); + let mut queue = InboundQueue::new(MAX_QUEUED_INBOUND_ENTRIES_PER_PEER); + + assert_eq!(queue.enqueue(peer_a, (0..(MAX_WANTED_BLOCKS + 4) as u8).map(entry).collect()), 0); + assert_eq!(queue.enqueue(peer_b, vec![entry(0xff)]), 0); + + let (peer, batch) = queue.next_batch().unwrap(); + assert_eq!((peer, batch.len()), (peer_a, MAX_WANTED_BLOCKS)); + let (peer, batch) = queue.next_batch().unwrap(); + assert_eq!((peer, batch.len()), (peer_b, 1)); + let (peer, batch) = queue.next_batch().unwrap(); + assert_eq!((peer, batch.len()), (peer_a, 4)); + assert!(queue.next_batch().is_none()); +} + +#[test] +fn inbound_queue_skips_inconsistent_rotation_entries() { + let stale_peer = litep2p::PeerId::random(); + let empty_peer = litep2p::PeerId::random(); + let ready_peer = litep2p::PeerId::random(); + let entry = (cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0xaa; 32]), WantType::Block); + let mut queue = InboundQueue::new(MAX_QUEUED_INBOUND_ENTRIES_PER_PEER); + + queue.rotation.extend([stale_peer, empty_peer]); + queue.per_peer.insert(empty_peer, VecDeque::new()); + queue.enqueue(ready_peer, vec![entry]); + + let (peer, batch) = queue.next_batch().expect("ready peer remains serviceable"); + assert_eq!(peer, ready_peer); + assert_eq!(batch, vec![entry]); + assert!(!queue.per_peer.contains_key(&empty_peer)); + assert!(queue.next_batch().is_none()); +} + +#[test] +fn inbound_queue_drops_overflow_and_counts_it() { + let peer = litep2p::PeerId::random(); + let entry = |i: u8| (cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [i; 32]), WantType::Block); + let mut queue = InboundQueue::new(4); + + assert_eq!(queue.enqueue(peer, (0..3).map(entry).collect()), 0); + assert_eq!(queue.enqueue(peer, (3..6).map(entry).collect()), 2); + + let (_, batch) = queue.next_batch().unwrap(); + let kept: Vec = batch.into_iter().map(|(cid, _)| cid).collect(); + assert_eq!( + kept, + (0..4) + .map(|i| cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [i; 32])) + .collect::>() + ); + assert!(queue.next_batch().is_none()); +} + +#[tokio::test(start_paused = true)] +async fn single_cid_single_peer_block_response() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let data = b"payload-a".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + + let (out_peer, out_cids) = drain_next(&mut rig.outbound_req_rx).await.expect("outbound WANT"); + assert_eq!(out_peer, peer); + assert_eq!(out_cids, vec![(cid, WantType::Block)]); + + rig.send_block(peer, cid, &data).await; + expect_block(&mut rx, cid, &data).await; +} + +#[tokio::test(start_paused = true)] +async fn have_response_reasks_same_peer_then_delivers() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let data = b"payload-have".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + + let (out_peer, _) = drain_next(&mut rig.outbound_req_rx).await.expect("first WANT"); + assert_eq!(out_peer, peer); + + rig.send_presence(peer, cid, BlockPresenceType::Have).await; + + let (out_peer, out_cids) = drain_next(&mut rig.outbound_req_rx).await.expect("re-ask"); + assert_eq!(out_peer, peer); + assert_eq!(out_cids, vec![(cid, WantType::Block)]); + + rig.send_block(peer, cid, &data).await; + expect_block(&mut rx, cid, &data).await; +} + +#[tokio::test(start_paused = true)] +async fn second_have_without_block_moves_to_other_peer() { + let mut rig = empty_rig(); + let peer_a = litep2p::PeerId::random(); + let peer_b = litep2p::PeerId::random(); + let data = b"payload-have-2".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer_a).await; + rig.connect(peer_b).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + + let (first, _) = drain_next(&mut rig.outbound_req_rx).await.expect("first WANT"); + let other = if first == peer_a { peer_b } else { peer_a }; + + rig.send_presence(first, cid, BlockPresenceType::Have).await; + let (out_peer, _) = drain_next(&mut rig.outbound_req_rx).await.expect("re-ask"); + assert_eq!(out_peer, first); + + rig.send_presence(first, cid, BlockPresenceType::Have).await; + let (out_peer, out_cids) = drain_next(&mut rig.outbound_req_rx).await.expect("failover WANT"); + assert_eq!(out_peer, other); + assert_eq!(out_cids, vec![(cid, WantType::Block)]); + + rig.send_block(other, cid, &data).await; + expect_block(&mut rx, cid, &data).await; +} + +#[rstest] +#[case::after_dont_have(true)] +#[case::after_timeout(false)] +#[tokio::test(start_paused = true)] +async fn exhausted_round_reasks_peer_after_delay(#[case] answer_dont_have: bool) { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let data = b"payload-round".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + let _ = drain_next(&mut rig.outbound_req_rx).await.expect("first WANT"); + + if answer_dont_have { + rig.send_presence(peer, cid, BlockPresenceType::DontHave).await; + let no_req = + timeout(ROUND_RETRY_DELAY - Duration::from_secs(1), rig.outbound_req_rx.recv()).await; + assert!(no_req.is_err()); + } + + let (out_peer, out_cids) = + timeout(PER_PEER_TIMEOUT + ROUND_RETRY_DELAY * 2, rig.outbound_req_rx.recv()) + .await + .expect("new-round WANT") + .expect("transport channel open"); + assert_eq!(out_peer, peer); + assert_eq!(out_cids, vec![(cid, WantType::Block)]); + + rig.send_block(peer, cid, &data).await; + expect_block(&mut rx, cid, &data).await; +} + +#[tokio::test(start_paused = true)] +async fn new_peer_is_asked_immediately_while_round_is_parked() { + let mut rig = empty_rig(); + let peer_a = litep2p::PeerId::random(); + let peer_b = litep2p::PeerId::random(); + let data = b"payload-round-3".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer_a).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + let _ = drain_next(&mut rig.outbound_req_rx).await.expect("first WANT"); + + rig.send_presence(peer_a, cid, BlockPresenceType::DontHave).await; + + rig.connect(peer_b).await; + let (out_peer, _) = drain_next(&mut rig.outbound_req_rx).await.expect("WANT to new peer"); + assert_eq!(out_peer, peer_b); + + rig.send_block(peer_b, cid, &data).await; + expect_block(&mut rx, cid, &data).await; + + tokio::time::advance(ROUND_RETRY_DELAY * 2).await; + assert!(rig.outbound_req_rx.try_recv().is_err()); +} + +#[tokio::test(start_paused = true)] +async fn light_client_peers_are_not_tracked() { + let mut rig = empty_rig(); + let light_peer = litep2p::PeerId::random(); + let full_peer = litep2p::PeerId::random(); + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [9u8; 32]); + + rig.sync_event_tx.send(sync_connected_light(light_peer)).await.unwrap(); + let _rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + + assert!(drain_next(&mut rig.outbound_req_rx).await.is_none()); + + rig.connect(full_peer).await; + let (out_peer, out_cids) = drain_next(&mut rig.outbound_req_rx).await.expect("outbound WANT"); + assert_eq!(out_peer, full_peer); + assert_eq!(out_cids, vec![(cid, WantType::Block)]); +} + +#[tokio::test(start_paused = true)] +async fn dont_have_from_only_peer_leaves_stream_open() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, b"the-real-payload"); + + rig.connect(peer).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + + let _ = drain_next(&mut rig.outbound_req_rx).await.expect("outbound WANT"); + + rig.send_presence(peer, cid, BlockPresenceType::DontHave).await; + + tokio::time::advance(Duration::from_secs(60)).await; + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); +} + +enum FailoverTrigger { + DontHave, + Timeout, + Disconnect, +} + +/// However the first peer fails the request — DONT_HAVE, an unanswered request timing +/// out, a block failing CID verification, or disconnecting — the want fails over to +/// the other connected peer. +#[rstest] +#[case::dont_have(FailoverTrigger::DontHave)] +#[case::timeout(FailoverTrigger::Timeout)] +#[case::disconnect(FailoverTrigger::Disconnect)] +#[tokio::test(start_paused = true)] +async fn first_peer_failure_triggers_failover(#[case] trigger: FailoverTrigger) { + let mut rig = empty_rig(); + let peer_a = litep2p::PeerId::random(); + let peer_b = litep2p::PeerId::random(); + let data = b"failover-payload".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer_a).await; + rig.connect(peer_b).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + let (first_peer, _) = drain_next(&mut rig.outbound_req_rx).await.expect("first WANT"); + + match trigger { + FailoverTrigger::DontHave => { + rig.send_presence(first_peer, cid, BlockPresenceType::DontHave).await + }, + FailoverTrigger::Timeout => { + tokio::time::advance(PER_PEER_TIMEOUT + Duration::from_secs(2)).await + }, + FailoverTrigger::Disconnect => { + rig.sync_event_tx.send(sync_disconnected(first_peer)).await.unwrap() + }, + } + + let (second_peer, _) = drain_next(&mut rig.outbound_req_rx).await.expect("failover WANT"); + assert_ne!(first_peer, second_peer); + + rig.send_block(second_peer, cid, &data).await; + expect_block(&mut rx, cid, &data).await; +} + +#[tokio::test(start_paused = true)] +async fn receiver_drop_cancels_wants() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [1u8; 32]); + + let rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + + // No peers connected: nothing is dispatched, the want just sits there. + assert!(drain_next(&mut rig.outbound_req_rx).await.is_none()); + + // The caller gives up; the sweep drops the abandoned waiter. + drop(rx); + tokio::time::advance(Duration::from_secs(2)).await; + + // A peer connecting afterwards must not trigger a WANT for the cancelled request. + rig.connect(peer).await; + assert!(drain_next(&mut rig.outbound_req_rx).await.is_none()); +} + +#[tokio::test(start_paused = true)] +async fn two_waiters_overlapping_cid_both_get_block() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let data = b"shared".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer).await; + + let mut rx_a = rig.user_handle.request_stream(vec![cid]).unwrap(); + let mut rx_b = rig.user_handle.request_stream(vec![cid]).unwrap(); + + let _ = drain_next(&mut rig.outbound_req_rx).await.expect("outbound"); + + rig.send_block(peer, cid, &data).await; + expect_block(&mut rx_a, cid, &data).await; + expect_block(&mut rx_b, cid, &data).await; +} + +#[tokio::test(start_paused = true)] +async fn waiter_drop_does_not_break_other_waiter() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let data = b"survivor".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer).await; + + let rx_a = rig.user_handle.request_stream(vec![cid]).unwrap(); + let mut rx_b = rig.user_handle.request_stream(vec![cid]).unwrap(); + + drop(rx_a); + sleep(Duration::from_millis(1)).await; + + let _ = drain_next(&mut rig.outbound_req_rx).await.expect("outbound"); + + rig.send_block(peer, cid, &data).await; + expect_block(&mut rx_b, cid, &data).await; +} + +#[tokio::test(start_paused = true)] +async fn dispatch_window_queues_excess_cids_and_promotes_on_delivery() { + let mut rig = small_window_rig(4); + let peer = litep2p::PeerId::random(); + rig.connect(peer).await; + + let payloads: Vec> = (0..5u8).map(|i| vec![i; 8]).collect(); + let cids: Vec = + payloads.iter().map(|p| cid_for_data(BLAKE2B_256_MULTIHASH_CODE, p)).collect(); + + let mut rx = rig.user_handle.request_stream(cids.clone()).unwrap(); + + // Only the window (4 CIDs) is dispatched; the fifth is queued. + let (_, entries) = drain_next(&mut rig.outbound_req_rx).await.expect("first bundle"); + assert_eq!(entries.len(), 4); + let dispatched: HashSet = entries.iter().map(|(cid, _)| *cid).collect(); + let queued_idx = cids.iter().position(|cid| !dispatched.contains(cid)).expect("one CID queued"); + + // Answering one dispatched CID frees a slot and promotes the queued CID. + let answered_idx = cids.iter().position(|cid| dispatched.contains(cid)).unwrap(); + rig.send_block(peer, cids[answered_idx], &payloads[answered_idx]).await; + expect_block(&mut rx, cids[answered_idx], &payloads[answered_idx]).await; + + let (_, promoted) = drain_next(&mut rig.outbound_req_rx).await.expect("promoted WANT"); + assert_eq!(promoted, vec![(cids[queued_idx], WantType::Block)]); +} + +#[tokio::test(start_paused = true)] +async fn inbound_request_with_known_block_serves_it() { + let client = TestClientBuilder::with_tx_storage(u32::MAX).build(); + let mut block_builder = BlockBuilderBuilder::new(&client) + .on_parent_block(client.chain_info().genesis_hash) + .with_parent_block_number(0) + .build() + .unwrap(); + + let ext = ExtrinsicBuilder::new_indexed_call(vec![0x42, 0x42, 0x42, 0x42]).build(); + let pattern_index = ext.encoded_size() - 4; + let data_hash = sp_crypto_hashing::blake2_256(&ext.encode()[pattern_index..]); + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, data_hash); + + block_builder.push(ext.clone()).unwrap(); + let block = block_builder.build().unwrap().block; + client.import(BlockOrigin::File, block).await.unwrap(); + + let mut rig = build_rig_with(Arc::new(client), MAX_LIVE_CIDS); + + let peer = litep2p::PeerId::random(); + rig.send_wantlist(peer, vec![(cid, WantType::Block)]).await; + + let (resp_peer, responses) = drain_next(&mut rig.outbound_resp_rx).await.expect("response"); + assert_eq!(resp_peer, peer); + assert_eq!(responses.len(), 1); + match &responses[0] { + ResponseType::Block { cid: got_cid, block } => { + assert_eq!(*got_cid, cid); + assert_eq!(*block, vec![0x42, 0x42, 0x42, 0x42]); + }, + other => panic!("expected Block, got {other:?}"), + } +} + +type BlockchainResult = sc_client_api::blockchain::Result; + +/// A backend whose `indexed_transaction` parks until the paired sender fires, keeping +/// an inbound lookup worker occupied for as long as the test needs. +struct GatedBackend { + gate: std::sync::Mutex>, +} + +impl BlockBackend for GatedBackend { + fn block_body( + &self, + _hash: H256, + ) -> BlockchainResult>> { + unimplemented!() + } + + fn block_indexed_body(&self, _hash: H256) -> BlockchainResult>>> { + unimplemented!() + } + + fn block_indexed_hashes(&self, _hash: H256) -> BlockchainResult>> { + unimplemented!() + } + + fn block( + &self, + _hash: H256, + ) -> BlockchainResult>> + { + unimplemented!() + } + + fn block_status(&self, _hash: H256) -> BlockchainResult { + unimplemented!() + } + + fn justifications(&self, _hash: H256) -> BlockchainResult> { + unimplemented!() + } + + fn block_hash(&self, _number: u64) -> BlockchainResult> { + unimplemented!() + } + + fn indexed_transaction(&self, _hash: H256) -> BlockchainResult>> { + let _ = self.gate.lock().unwrap().recv(); + Ok(None) + } + + fn requires_full_sync(&self) -> bool { + false + } +} + +#[tokio::test] +async fn busy_pool_queues_wantlists_until_worker_frees() { + let (gate_tx, gate_rx) = std::sync::mpsc::channel(); + let backend = Arc::new(GatedBackend { gate: std::sync::Mutex::new(gate_rx) }); + let mut rig = build_rig_with_inbound_limits( + backend, + MAX_LIVE_CIDS, + 1, + MAX_QUEUED_INBOUND_ENTRIES_PER_PEER, + ); + + let peer = litep2p::PeerId::random(); + let first_cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0x0a; 32]); + let second_cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0x0b; 32]); + + // The first wantlist occupies the only lookup worker (parked on the gate); the + // second waits in the peer's queue instead of being refused. + for cid in [first_cid, second_cid] { + rig.send_wantlist(peer, vec![(cid, WantType::Block)]).await; + } + let no_response = timeout(Duration::from_millis(300), rig.outbound_resp_rx.recv()).await; + assert!(no_response.is_err(), "nothing must be answered while the worker is parked"); + + // Releasing the worker serves both wantlists, in arrival order. + gate_tx.send(()).unwrap(); + let (resp_peer, responses) = + drain_next(&mut rig.outbound_resp_rx).await.expect("first response"); + assert_eq!(resp_peer, peer); + assert_single_dont_have(&responses, first_cid); + + gate_tx.send(()).unwrap(); + let (_, responses) = drain_next(&mut rig.outbound_resp_rx).await.expect("second response"); + assert_single_dont_have(&responses, second_cid); +} + +#[tokio::test] +async fn inbound_queue_round_robins_between_peers() { + let (gate_tx, gate_rx) = std::sync::mpsc::channel(); + let backend = Arc::new(GatedBackend { gate: std::sync::Mutex::new(gate_rx) }); + let mut rig = build_rig_with_inbound_limits( + backend, + MAX_LIVE_CIDS, + 1, + MAX_QUEUED_INBOUND_ENTRIES_PER_PEER, + ); + + let peer_a = litep2p::PeerId::random(); + let peer_b = litep2p::PeerId::random(); + let wantlist = |tag: u8, len: usize| -> Vec<(Cid, WantType)> { + (0..len as u8) + .map(|i| { + let mut digest = [tag; 32]; + digest[1] = i; + (cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, digest), WantType::Block) + }) + .collect() + }; + + // Peer A backlogs three batches: the first is dispatched immediately, two queue. + rig.send_wantlist(peer_a, wantlist(0xaa, 3 * MAX_WANTED_BLOCKS)).await; + // Peer B queues one batch behind A's backlog. + rig.send_wantlist(peer_b, wantlist(0xbb, MAX_WANTED_BLOCKS)).await; + + // Release one batch worth of lookups at a time and record who gets answered. + let mut served_order = Vec::new(); + for _ in 0..4 { + for _ in 0..MAX_WANTED_BLOCKS { + gate_tx.send(()).unwrap(); + } + let (resp_peer, responses) = + drain_next(&mut rig.outbound_resp_rx).await.expect("batch response"); + assert_eq!(responses.len(), MAX_WANTED_BLOCKS); + served_order.push(resp_peer); + } + + // Round-robin: B's batch is interleaved into A's backlog instead of waiting for + // all of it. (A goes twice first: its second batch re-entered the rotation + // before B arrived.) + assert_eq!(served_order, vec![peer_a, peer_a, peer_b, peer_a]); +} + +#[tokio::test] +async fn per_peer_queue_overflow_drops_newest_entries() { + let (gate_tx, gate_rx) = std::sync::mpsc::channel(); + let backend = Arc::new(GatedBackend { gate: std::sync::Mutex::new(gate_rx) }); + // Single gated worker, queue capped at 4 entries per peer. + let mut rig = build_rig_with_inbound_limits(backend, MAX_LIVE_CIDS, 1, 4); + + let peer = litep2p::PeerId::random(); + let cid = |i: u8| cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [i; 32]); + + // The first wantlist occupies the worker; the next four fill the queue; the + // sixth overflows the per-peer cap and is dropped silently. + for i in 0..6u8 { + rig.send_wantlist(peer, vec![(cid(i), WantType::Block)]).await; + } + + // Serve everything: the five accepted entries are answered, the sixth never is. + for _ in 0..5 { + gate_tx.send(()).unwrap(); + } + let mut answered = HashSet::new(); + while answered.len() < 5 { + let (_, responses) = drain_next(&mut rig.outbound_resp_rx).await.expect("response"); + for response in responses { + match response { + ResponseType::Presence { cid, presence: BlockPresenceType::DontHave } => { + assert!(answered.insert(cid), "duplicate reply for {cid}"); + }, + other => panic!("expected DONT_HAVE, got {other:?}"), + } + } + } + assert_eq!(answered, (0..5u8).map(cid).collect()); + let no_more = timeout(Duration::from_millis(300), rig.outbound_resp_rx.recv()).await; + assert!(no_more.is_err(), "the overflowed entry must not be answered"); +} + +#[tokio::test] +async fn large_wantlist_is_served_in_batches_covering_every_entry() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + + let cids: Vec<(Cid, WantType)> = (0..(MAX_WANTED_BLOCKS + 2) as u8) + .map(|i| { + let mut digest = [0u8; 32]; + digest[0] = i; + (cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, digest), WantType::Block) + }) + .collect(); + + rig.send_wantlist(peer, cids.clone()).await; + + // A wantlist larger than `MAX_WANTED_BLOCKS` is accepted whole and served in + // batches. Every entry gets a reply (DONT_HAVE: the test client holds no + // indexed data). + let mut answered = HashSet::new(); + while answered.len() < cids.len() { + let (resp_peer, responses) = + drain_next(&mut rig.outbound_resp_rx).await.expect("reply for every entry"); + assert_eq!(resp_peer, peer); + for response in responses { + match response { + ResponseType::Presence { cid, presence: BlockPresenceType::DontHave } => { + assert!(answered.insert(cid), "duplicate reply for {cid}"); + }, + other => panic!("expected DONT_HAVE, got {other:?}"), + } + } + } + assert_eq!(answered, cids.into_iter().map(|(cid, _)| cid).collect()); +} + +#[tokio::test] +async fn unsupported_cid_in_wantlist_gets_dont_have() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let unsupported = Cid::new_v1( + RAW_CODEC, + CidMultihash::<64>::wrap(0x99 /* unsupported */, &[0u8; 32]).unwrap(), + ); + + rig.send_wantlist(peer, vec![(unsupported, WantType::Block)]).await; + + let (resp_peer, responses) = drain_next(&mut rig.outbound_resp_rx).await.expect("reply"); + assert_eq!(resp_peer, peer); + assert_single_dont_have(&responses, unsupported); +} + +#[tokio::test(start_paused = true)] +async fn inbound_serving_continues_after_all_handles_dropped() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [3u8; 32]); + + drop(rig.user_handle); + sleep(Duration::from_millis(1)).await; + + rig.inbound_tx + .send(TransportEvent::Request { peer, cids: vec![(cid, WantType::Block)] }) + .await + .unwrap(); + + let (resp_peer, responses) = drain_next(&mut rig.outbound_resp_rx) + .await + .expect("service must keep serving inbound after all handles are dropped"); + assert_eq!(resp_peer, peer); + assert_single_dont_have(&responses, cid); +} + +#[tokio::test(start_paused = true)] +async fn outbound_wants_bundled_and_split_at_message_cap() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + rig.connect(peer).await; + + let cids: Vec = (0..=MAX_WANTED_BLOCKS as u8) + .map(|i| { + let mut d = [0u8; 32]; + d[0] = i; + cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, d) + }) + .collect(); + + let _rx = rig.user_handle.request_stream(cids.clone()).unwrap(); + + let (peer_a, first) = drain_next(&mut rig.outbound_req_rx).await.expect("first bundle"); + let (peer_b, second) = drain_next(&mut rig.outbound_req_rx).await.expect("second bundle"); + assert_eq!(peer_a, peer); + assert_eq!(peer_b, peer); + + let mut sizes = [first.len(), second.len()]; + sizes.sort(); + assert_eq!(sizes, [1, MAX_WANTED_BLOCKS]); + + let sent: HashSet = first.into_iter().chain(second).map(|(cid, _)| cid).collect(); + assert_eq!(sent, cids.into_iter().collect::>()); +} + +#[tokio::test] +async fn admission_invalid_cid_rejected() { + let rig = empty_rig(); + let bad = Cid::new_v1( + RAW_CODEC, + CidMultihash::<64>::wrap(0x99 /* unsupported */, &[0u8; 32]).unwrap(), + ); + + let err = rig.user_handle.request_stream(vec![bad]).err().expect("err"); + assert!(matches!(err, BitswapError::InvalidCid { .. })); +} + +#[tokio::test] +async fn admission_empty_returns_closed_receiver() { + let rig = empty_rig(); + let mut rx = rig.user_handle.request_stream(vec![]).unwrap(); + assert!(rx.recv().await.is_none()); +} + +#[tokio::test(start_paused = true)] +async fn service_shutdown_emits_service_closed() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0xee; 32]); + + rig.connect(peer).await; + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + let _ = drain_next(&mut rig.outbound_req_rx).await; + + drop(rig.inbound_tx); + + let item = drain_next(&mut rx).await.expect("expect Err"); + assert!(matches!(item, Err(BitswapError::ServiceClosed))); +} + +#[tokio::test(start_paused = true)] +async fn late_response_after_receiver_drop_is_ignored_and_cid_refetchable() { + let mut rig = empty_rig(); + let peer = litep2p::PeerId::random(); + let data = b"too-late".to_vec(); + let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); + + rig.connect(peer).await; + let rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + let _ = drain_next(&mut rig.outbound_req_rx).await.expect("outbound"); + + // Caller gives up; the sweep drops the waiter while the peer request is still + // in flight. + drop(rx); + tokio::time::advance(Duration::from_secs(2)).await; + + // The late response hits no waiter and is dropped. + rig.send_block(peer, cid, &data).await; + // Let the actor process the late response before the fresh request goes in. + sleep(Duration::from_millis(1)).await; + + // A fresh request for the same CID starts from a clean slate: the peer is asked + // again and the block is delivered. + let mut rx = rig.user_handle.request_stream(vec![cid]).unwrap(); + let _ = drain_next(&mut rig.outbound_req_rx).await.expect("fresh WANT"); + rig.send_block(peer, cid, &data).await; + expect_block(&mut rx, cid, &data).await; +} + +#[tokio::test(start_paused = true)] +async fn too_many_waiters_per_cid_yields_overloaded() { + let rig = empty_rig(); + let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [0x77; 32]); + + let mut receivers = Vec::new(); + for _ in 0..MAX_WAITERS_PER_CID { + receivers.push(rig.user_handle.request_stream(vec![cid]).unwrap()); + } + // Give the actor a chance to admit all waiters before the one-too-many request. + sleep(Duration::from_millis(10)).await; + + let mut rejected = rig.user_handle.request_stream(vec![cid]).unwrap(); + let item = drain_next(&mut rejected).await.expect("item"); + assert!(matches!(item, Err(BitswapError::Overloaded))); +} + +mod proptests { + use super::*; + use crate::{BLAKE2B_256_MULTIHASH_CODE, RAW_CODEC}; + use cid::multihash::Multihash as CidMultihash; + use proptest::prelude::*; + use rand::{rngs::StdRng, SeedableRng}; + + const NUM_CIDS: u8 = 8; + const NUM_PEERS: usize = 4; + const SMALL_WINDOW: usize = 3; + + #[derive(Debug, Clone)] + enum Op { + AddWaiter(u8), + RemoveWaiter(usize), + Deliver(u8), + MarkPeerDone(usize, u8), + NoteHave(usize, u8), + Connect(usize), + Disconnect(usize), + Sweep(u64), + } + + fn op_strategy() -> impl Strategy { + prop_oneof![ + (0..NUM_CIDS).prop_map(Op::AddWaiter), + any::().prop_map(Op::RemoveWaiter), + (0..NUM_CIDS).prop_map(Op::Deliver), + (0..NUM_PEERS, 0..NUM_CIDS).prop_map(|(p, c)| Op::MarkPeerDone(p, c)), + (0..NUM_PEERS, 0..NUM_CIDS).prop_map(|(p, c)| Op::NoteHave(p, c)), + (0..NUM_PEERS).prop_map(Op::Connect), + (0..NUM_PEERS).prop_map(Op::Disconnect), + (1u64..8).prop_map(Op::Sweep), + ] + } + + struct Harness { + wants: WantSet, + waiters: SlotMap, + peers: Vec, + cids: Vec, + connected: HashSet, + now: Instant, + rng: StdRng, + } + + impl Harness { + fn new() -> Self { + let cids = (0..NUM_CIDS) + .map(|i| { + let mh = + CidMultihash::<64>::wrap(BLAKE2B_256_MULTIHASH_CODE, &[i; 32]).unwrap(); + Cid::new_v1(RAW_CODEC, mh) + }) + .collect(); + Self { + wants: WantSet::new(SMALL_WINDOW), + waiters: SlotMap::with_key(), + peers: (0..NUM_PEERS).map(|_| litep2p::PeerId::random()).collect(), + cids, + connected: HashSet::new(), + now: Instant::now(), + rng: StdRng::seed_from_u64(0), + } + } + + fn apply(&mut self, op: Op) { + match op { + Op::AddWaiter(c) => { + let cid = self.cids[c as usize]; + let id = self.waiters.insert(cid); + self.wants.add_waiter(cid, id); + }, + Op::RemoveWaiter(seed) => { + let Some(id) = self.waiters.keys().nth(seed % self.waiters.len().max(1)) else { + return; + }; + let cid = self.waiters.remove(id).expect("listed waiter exists"); + self.wants.remove_waiter(cid, id); + }, + Op::Deliver(c) => { + let waiter_ids = self + .wants + .take_waiters_for_delivered_cid(self.cids[c as usize]) + .unwrap_or_default(); + for id in waiter_ids { + self.waiters.remove(id); + } + }, + Op::MarkPeerDone(p, c) => { + self.wants.mark_peer_done_for_cid(self.peers[p], self.cids[c as usize]) + }, + Op::NoteHave(p, c) => { + self.wants.note_peer_have_for_cid(self.peers[p], self.cids[c as usize]) + }, + Op::Connect(p) => { + self.connected.insert(self.peers[p]); + }, + Op::Disconnect(p) => { + self.connected.remove(&self.peers[p]); + let _ = self.wants.remove_in_flight_peer(self.peers[p]); + }, + Op::Sweep(secs) => { + self.now += Duration::from_secs(secs); + let _ = self.wants.expire_peer_timeouts(self.now); + let _ = self.wants.restart_exhausted_rounds(self.now); + }, + } + } + + fn top_up(&mut self) { + for cid in self.wants.all_cids() { + let _ = + self.wants.next_peer_to_request(cid, &self.connected, self.now, &mut self.rng); + } + while self.wants.has_window_capacity() { + let Some(cid) = self.wants.pop_pending() else { break }; + let _ = + self.wants.next_peer_to_request(cid, &self.connected, self.now, &mut self.rng); + } + } + + fn check_invariants(&self) { + let live = self + .wants + .inner + .values() + .filter(|state| matches!(state.phase, CidPhase::InFlight { .. })) + .count(); + assert_eq!(self.wants.live, live, "live counter drifted"); + assert!(self.wants.live <= self.wants.max_live, "dispatch window overrun"); + let queued = self.wants.inner.values().filter(|state| state.is_queued()).count(); + assert_eq!(self.wants.queued, queued, "queued counter drifted"); + + for (cid, state) in &self.wants.inner { + assert!(!state.is_idle(), "idle entry retained for {cid}"); + if let CidPhase::InFlight { peer, .. } = state.phase { + assert!(!state.tried_peers.contains(&peer), "peer both tried and in flight"); + } + if state.is_queued() { + assert!( + self.wants.pending.contains(cid), + "queued CID without queue entry for {cid}", + ); + } + let retry_at = match state.phase { + CidPhase::Queued { retry_at } => retry_at, + CidPhase::RetryAt(at) => Some(at), + CidPhase::Ready | CidPhase::InFlight { .. } => None, + }; + if let Some(at) = retry_at { + assert!(at > self.now, "overdue round never restarted for {cid}"); + } + if state.has_waiters() && !self.connected.is_empty() { + assert!( + matches!( + state.phase, + CidPhase::Queued { .. } | + CidPhase::InFlight { .. } | + CidPhase::RetryAt(_) + ), + "stranded CID {cid}", + ); + } + } + } + } + + proptest! { + #[test] + fn want_set_invariants_hold(ops in prop::collection::vec(op_strategy(), 1..256)) { + let mut harness = Harness::new(); + for op in ops { + harness.apply(op); + harness.top_up(); + harness.check_invariants(); + } + } + } +} diff --git a/substrate/client/network/build.rs b/substrate/client/network/build.rs deleted file mode 100644 index 2495a9c25653..000000000000 --- a/substrate/client/network/build.rs +++ /dev/null @@ -1,23 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -const PROTOS: &[&str] = &["src/schema/bitswap.v1.2.0.proto"]; - -fn main() { - prost_build::compile_protos(PROTOS, &["src/schema"]).unwrap(); -} diff --git a/substrate/client/network/src/bitswap/client.rs b/substrate/client/network/src/bitswap/client.rs deleted file mode 100644 index 60e5240dae76..000000000000 --- a/substrate/client/network/src/bitswap/client.rs +++ /dev/null @@ -1,906 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Substrate. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use crate::{IfDisconnected, NetworkRequest, ProtocolName}; - -use cid::{multihash::Multihash as CidMultihash, Cid, Version as CidVersion}; -use log::{debug, trace, warn}; -use prost::Message; -use sc_network_types::PeerId; -use std::collections::{HashMap, HashSet}; - -use super::{ - is_cid_supported, - schema::bitswap::{ - message::{ - wantlist::{Entry, WantType as ProtoWantType}, - BlockPresence, BlockPresenceType, Wantlist, - }, - Message as BitswapMessage, - }, - Prefix, LOG_TARGET, MAX_WANTED_BLOCKS, PROTOCOL_NAME, -}; - -/// Const from -/// Multihash code for BLAKE2b-256. -pub const BLAKE2B_256_MULTIHASH_CODE: u64 = 0xb220; -/// Multihash code for SHA2-256. -pub const SHA2_256_MULTIHASH_CODE: u64 = 0x12; -/// Multihash code for Keccak-256. -pub const KECCAK_256_MULTIHASH_CODE: u64 = 0x1b; - -/// Per-CID outcome from a Bitswap block request. -/// -/// The public contract is intentionally narrow: either the peer delivered the bytes for the CID -/// or it did not. A peer signalling `DONT_HAVE` and a peer staying silent for a CID are both -/// surfaced as [`FetchOutcome::Missing`]; callers needing a different policy must implement it -/// over [`FetchOutcome`]. -#[derive(Debug)] -pub enum FetchOutcome { - /// Peer returned bytes for the requested CID. - Block(Vec), - /// Peer did not deliver bytes for this CID. - /// - /// Covers the peer explicitly answering `DONT_HAVE`, the peer answering `HAVE` without bytes, - /// and the peer not acknowledging the CID at all. From the caller's perspective these are - /// equivalent: no block was delivered. - Missing, -} - -/// Multihash type with a 64-byte digest capacity. -type Multihash = CidMultihash<64>; - -/// Validate the wantlist length is within bounds. -fn validate_wantlist_size(len: usize) -> Result<(), BitswapError> { - if len == 0 { - return Err(BitswapError::DecodeError("empty wantlist".into())); - } - if len > MAX_WANTED_BLOCKS { - return Err(BitswapError::DecodeError(format!( - "wantlist too large: {len} > {MAX_WANTED_BLOCKS}", - ))); - } - Ok(()) -} - -/// Validate CIDs: enforce length, CID support, and CID uniqueness. -fn validate_cids(cids: &[Cid]) -> Result<(), BitswapError> { - validate_wantlist_size(cids.len())?; - - let mut seen: HashSet = HashSet::with_capacity(cids.len()); - for cid in cids { - if !is_cid_supported(cid) { - return Err(BitswapError::UnsupportedHashing { multihash_code: cid.hash().code() }); - } - if !seen.insert(*cid) { - return Err(BitswapError::DecodeError(format!("duplicate CID in wantlist: {cid}"))); - } - } - - Ok(()) -} - -/// Send one `WANT-BLOCK` request for `cids` to `peer` and classify the response. -/// -/// Returned blocks are verified by recomputing the CID from the response prefix and bytes. -/// Blocks whose recomputed CID was not requested are ignored. -/// -/// Errors if `cids` is empty, larger than [`MAX_WANTED_BLOCKS`], contains an unsupported CID, -/// or contains a duplicate CID. -/// -/// Note: This is a temporary API that shall be superseeded by a better abstraction such as -/// -pub async fn request_bitswap_blocks( - network: &N, - peer: PeerId, - cids: &[Cid], -) -> Result, BitswapError> -where - N: NetworkRequest + ?Sized, -{ - validate_cids(cids)?; - - let wanted: HashSet = cids.iter().copied().collect(); - let response = send_request(network, peer, cids).await?; - Ok(classify_response(response, &wanted, peer)) -} - -/// Like [`request_bitswap_blocks`], but does not recompute or verify the hash of received bytes. -/// -/// Use this when the requester must fetch by CID-shaped identifiers before it can verify the -/// returned bytes through an external authority. The response is matched by request order and -/// CID prefix only; integrity verification is delegated to the caller. -/// -/// Note: This is a temporary API that shall be superseeded by a better abstraction such as -/// -pub async fn request_bitswap_blocks_unverified( - network: &N, - peer: PeerId, - cids: &[Cid], -) -> Result, BitswapError> -where - N: NetworkRequest + ?Sized, -{ - validate_cids(cids)?; - - let response = send_request(network, peer, cids).await?; - Ok(classify_response_unverified(response, cids, peer)) -} - -/// Dispatch a bitswap WANT request to `peer` and decode the response. -async fn send_request( - network: &N, - peer: PeerId, - cids: &[Cid], -) -> Result -where - N: NetworkRequest + ?Sized, -{ - let entries: Vec = cids - .iter() - .copied() - .map(|cid| Entry { - block: cid.to_bytes(), - want_type: ProtoWantType::Block as i32, - send_dont_have: true, - ..Default::default() - }) - .collect(); - let request = - BitswapMessage { wantlist: Some(Wantlist { entries, full: false }), ..Default::default() }; - - trace!( - target: LOG_TARGET, - "client: sending Bitswap wantlist for {} CIDs to {peer}, protocol {PROTOCOL_NAME}", - cids.len(), - ); - - let payload = match network - .request( - peer, - ProtocolName::from(PROTOCOL_NAME), - request.encode_to_vec(), - None, - IfDisconnected::TryConnect, - ) - .await - { - Ok((payload, _)) => payload, - Err(err) => { - debug!(target: LOG_TARGET, "client: batch request to {peer} rejected by network: {err:?}"); - return Err(BitswapError::RequestFailed(err.to_string())); - }, - }; - - BitswapMessage::decode(&payload[..]).map_err(|err| { - debug!(target: LOG_TARGET, "client: failed to decode batch response from {peer}: {err}"); - BitswapError::DecodeError(err.to_string()) - }) -} - -/// Classify the response by verifying each block's CID against the wanted set. -/// -/// Every wanted CID is recorded exactly once: as [`FetchOutcome::Block`] if the peer delivered -/// bytes whose recomputed CID is in `wanted`, otherwise as [`FetchOutcome::Missing`]. Presence -/// frames (`HAVE` / `DONT_HAVE`) are logged for diagnostics but do not change the outcome. -fn classify_response( - response: BitswapMessage, - wanted: &HashSet, - peer: PeerId, -) -> HashMap { - let mut result: HashMap = HashMap::with_capacity(wanted.len()); - - for block in response.payload { - let Ok(cid) = cid_from_block_prefix(&block.prefix, &block.data).inspect_err(|err| { - debug!(target: LOG_TARGET, "client: malformed block prefix from {peer}: {err:?}"); - }) else { - continue; - }; - if !wanted.contains(&cid) { - debug!(target: LOG_TARGET, "client: {peer} returned unsolicited block for CID {cid}"); - continue; - } - debug!(target: LOG_TARGET, "client: {peer} returned {} bytes for CID {cid}", block.data.len()); - result.insert(cid, FetchOutcome::Block(block.data)); - } - - log_presences(response.block_presences, wanted, peer); - - for cid in wanted { - result.entry(*cid).or_insert(FetchOutcome::Missing); - } - - result -} - -/// Classify an unverified response via order-based correlation. -/// -/// Every wanted CID is recorded exactly once: as [`FetchOutcome::Block`] if the peer delivered -/// bytes whose declared prefix matches a requested CID at the corresponding position in the -/// wantlist, otherwise as [`FetchOutcome::Missing`]. -fn classify_response_unverified( - response: BitswapMessage, - cids: &[Cid], - peer: PeerId, -) -> HashMap { - let mut result: HashMap = HashMap::with_capacity(cids.len()); - let wanted_set: HashSet = cids.iter().copied().collect(); - let mut dont_have_cids: HashSet = HashSet::with_capacity(cids.len()); - - for presence in response.block_presences { - let Ok(cid) = Cid::read_bytes(presence.cid.as_slice()).inspect_err(|err| { - debug!(target: LOG_TARGET, "client: malformed presence CID from {peer}: {err}"); - }) else { - continue; - }; - if !wanted_set.contains(&cid) { - debug!(target: LOG_TARGET, "client: {peer} returned unsolicited presence for CID {cid}"); - continue; - } - if presence.r#type == BlockPresenceType::DontHave as i32 { - debug!(target: LOG_TARGET, "client: {peer} DONT_HAVE for CID {cid}"); - dont_have_cids.insert(cid); - } else if presence.r#type == BlockPresenceType::Have as i32 { - debug!(target: LOG_TARGET, "client: {peer} HAVE for CID {cid}"); - } else { - warn!( - target: LOG_TARGET, - "client: {peer} unexpected presence type {} for CID {cid}", - presence.r#type, - ); - } - } - - // Unverified payloads cannot be matched by recomputing their CID from bytes, so attribute - // each block to the next requested CID (skipping any the peer already said it doesn't have) - // whose CID metadata matches the payload prefix. - let mut expected_payload_order = - cids.iter().copied().filter(|cid| !dont_have_cids.contains(cid)); - - for block in response.payload { - let Some(expected_cid) = expected_payload_order.next() else { - debug!(target: LOG_TARGET, "client: {peer} returned more payload blocks than expected; dropping extras"); - break; - }; - let Ok(prefix) = decode_prefix(&block.prefix).inspect_err(|err| { - debug!(target: LOG_TARGET, "client: malformed block prefix from {peer}: {err:?}"); - }) else { - break; - }; - if !prefix_matches_cid(&prefix, &expected_cid) { - debug!( - target: LOG_TARGET, - "client: {peer} returned block with prefix {:?} but expected CID {expected_cid}; \ - stopping payload attribution", - prefix, - ); - break; - } - debug!( - target: LOG_TARGET, - "client: {peer} returned {} unverified bytes for CID {expected_cid}", - block.data.len(), - ); - result.insert(expected_cid, FetchOutcome::Block(block.data.clone())); - } - - for cid in cids { - result.entry(*cid).or_insert(FetchOutcome::Missing); - } - - result -} - -/// Log per-CID presence frames for diagnostics. Presence does not influence the public outcome. -fn log_presences(presences: Vec, wanted: &HashSet, peer: PeerId) { - for presence in presences { - let Ok(cid) = Cid::read_bytes(presence.cid.as_slice()).inspect_err(|err| { - debug!(target: LOG_TARGET, "client: malformed presence CID from {peer}: {err}"); - }) else { - continue; - }; - if !wanted.contains(&cid) { - debug!(target: LOG_TARGET, "client: {peer} returned unsolicited presence for CID {cid}"); - continue; - } - if presence.r#type == BlockPresenceType::DontHave as i32 { - debug!(target: LOG_TARGET, "client: {peer} DONT_HAVE for CID {cid}"); - } else if presence.r#type == BlockPresenceType::Have as i32 { - debug!(target: LOG_TARGET, "client: {peer} HAVE for CID {cid}"); - } else { - debug!( - target: LOG_TARGET, - "client: {peer} unexpected presence type {} for CID {cid}", - presence.r#type, - ); - } - } -} - -/// Check that a decoded prefix matches a CID's version, codec, and multihash metadata. -fn prefix_matches_cid(prefix: &Prefix, cid: &Cid) -> bool { - prefix.version == cid.version() && - prefix.codec == cid.codec() && - prefix.mh_type == cid.hash().code() && - prefix.mh_len == cid.hash().size() -} - -/// Reconstruct a CID from a block's prefix bytes and payload data. -fn cid_from_block_prefix(prefix: &[u8], data: &[u8]) -> Result { - let prefix = decode_prefix(prefix)?; - if prefix.version != CidVersion::V1 { - return Err(BitswapError::UnsupportedCidVersion { version: prefix.version.into() }); - } - - let hash = hash_for_multihash_code(prefix.mh_type, data) - .ok_or(BitswapError::UnsupportedHashing { multihash_code: prefix.mh_type })?; - let multihash = Multihash::wrap(prefix.mh_type, &hash) - .map_err(|err| BitswapError::DecodeError(err.to_string()))?; - Ok(Cid::new_v1(prefix.codec, multihash)) -} - -/// Compute a 32-byte hash for the given multihash code. -fn hash_for_multihash_code(multihash_code: u64, data: &[u8]) -> Option<[u8; 32]> { - match multihash_code { - BLAKE2B_256_MULTIHASH_CODE => Some(sp_crypto_hashing::blake2_256(data)), - SHA2_256_MULTIHASH_CODE => Some(sp_crypto_hashing::sha2_256(data)), - KECCAK_256_MULTIHASH_CODE => Some(sp_crypto_hashing::keccak_256(data)), - _ => None, - } -} - -/// Decode varint-encoded CID prefix bytes. -fn decode_prefix(mut bytes: &[u8]) -> Result { - let mut read_varint = || -> Result { - let (v, rest) = unsigned_varint::decode::u64(bytes) - .map_err(|err| BitswapError::DecodeError(err.to_string()))?; - bytes = rest; - Ok(v) - }; - - let version = read_varint()?; - let codec = read_varint()?; - let mh_type = read_varint()?; - let mh_len = read_varint()?; - - if !bytes.is_empty() { - return Err(BitswapError::DecodeError("bitswap block prefix had trailing bytes".into())); - } - - let version = CidVersion::try_from(version) - .map_err(|_| BitswapError::UnsupportedCidVersion { version })?; - let mh_len = u8::try_from(mh_len).map_err(|_| { - BitswapError::DecodeError(format!("multihash length {mh_len} does not fit into u8")) - })?; - - Ok(Prefix { version, codec, mh_type, mh_len }) -} - -/// Bitswap client errors. -#[derive(Debug)] -pub enum BitswapError { - /// Failed to decode or validate a bitswap payload. - DecodeError(String), - /// Request/response exchange failed. - RequestFailed(String), - /// Block prefix declared an unsupported multihash code. - UnsupportedHashing { - /// The unrecognised IPFS multihash code. - multihash_code: u64, - }, - /// CID version is unsupported for this bitswap client. - UnsupportedCidVersion { - /// The unsupported CID version number. - version: u64, - }, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{OutboundFailure, RequestFailure}; - use futures::channel::oneshot; - use sc_network_types::PeerId; - use std::{collections::VecDeque, sync::Mutex}; - - use super::super::{ - is_supported_multihash_code, - schema::bitswap::message::{Block as MessageBlock, BlockPresence, BlockPresenceType}, - RAW_CODEC, - }; - - /// Build a raw-codec CID from a 32-byte digest and supported multihash code. - fn raw_cid_from_digest(multihash_code: u64, digest: [u8; 32]) -> Result { - if !is_supported_multihash_code(multihash_code) { - return Err(BitswapError::UnsupportedHashing { multihash_code }); - } - let multihash = CidMultihash::wrap(multihash_code, &digest) - .map_err(|e| BitswapError::DecodeError(e.to_string()))?; - Ok(Cid::new_v1(RAW_CODEC, multihash)) - } - - struct StubSender { - responses: Mutex, RequestFailure>>>, - requests: Mutex>>, - } - - impl StubSender { - fn new(responses: impl IntoIterator, RequestFailure>>) -> Self { - Self { - responses: Mutex::new(responses.into_iter().collect()), - requests: Mutex::new(Vec::new()), - } - } - - fn pop_request(&self) -> BitswapMessage { - let bytes = self.requests.lock().unwrap().pop().expect("request should be recorded"); - BitswapMessage::decode(bytes.as_slice()).expect("request should decode") - } - } - - #[async_trait::async_trait] - impl NetworkRequest for StubSender { - async fn request( - &self, - _target: PeerId, - _protocol: ProtocolName, - request: Vec, - _fallback_request: Option<(Vec, ProtocolName)>, - _connect: IfDisconnected, - ) -> Result<(Vec, ProtocolName), RequestFailure> { - self.requests.lock().unwrap().push(request); - self.responses - .lock() - .unwrap() - .pop_front() - .expect("StubSender: no canned response queued") - .map(|bytes| (bytes, ProtocolName::from(PROTOCOL_NAME))) - } - - fn start_request( - &self, - _peer: PeerId, - _protocol: ProtocolName, - payload: Vec, - _fallback_request: Option<(Vec, ProtocolName)>, - tx: oneshot::Sender, ProtocolName), RequestFailure>>, - _connect: IfDisconnected, - ) { - self.requests.lock().unwrap().push(payload); - let resp = self - .responses - .lock() - .unwrap() - .pop_front() - .expect("StubSender: no canned response queued"); - let _ = tx.send(resp.map(|bytes| (bytes, ProtocolName::from(PROTOCOL_NAME)))); - } - } - - fn prefix_for(multihash_code: u64) -> Vec { - Prefix { version: CidVersion::V1, codec: RAW_CODEC, mh_type: multihash_code, mh_len: 32 } - .to_bytes() - } - - fn cid_for_data(multihash_code: u64, data: &[u8]) -> Cid { - raw_cid_from_digest(multihash_code, hash_for_multihash_code(multihash_code, data).unwrap()) - .unwrap() - } - - fn cid_for_digest(multihash_code: u64, digest: [u8; 32]) -> Cid { - raw_cid_from_digest(multihash_code, digest).unwrap() - } - - fn encode_response(blocks: &[(u64, Vec)], presences: &[(Cid, i32)]) -> Vec { - let payload = blocks - .iter() - .map(|(multihash_code, data)| MessageBlock { - prefix: prefix_for(*multihash_code), - data: data.clone(), - }) - .collect(); - let block_presences = presences - .iter() - .map(|(cid, ptype)| BlockPresence { cid: cid.to_bytes(), r#type: *ptype }) - .collect(); - BitswapMessage { payload, block_presences, ..Default::default() }.encode_to_vec() - } - - #[tokio::test] - async fn request_bitswap_blocks_returns_blocks_for_all_wanted() { - let data_a = b"hash-a-payload".to_vec(); - let data_b = b"hash-b-payload".to_vec(); - let data_c = b"hash-c-payload".to_vec(); - let cid_a = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data_a); - let cid_b = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data_b); - let cid_c = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data_c); - - let response = encode_response( - &[ - (BLAKE2B_256_MULTIHASH_CODE, data_a.clone()), - (BLAKE2B_256_MULTIHASH_CODE, data_b.clone()), - (BLAKE2B_256_MULTIHASH_CODE, data_c.clone()), - ], - &[], - ); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[cid_a, cid_b, cid_c]) - .await - .expect("request_bitswap_blocks should succeed"); - - assert_eq!(result.len(), 3); - assert!(matches!(result.get(&cid_a), Some(FetchOutcome::Block(d)) if *d == data_a)); - assert!(matches!(result.get(&cid_b), Some(FetchOutcome::Block(d)) if *d == data_b)); - assert!(matches!(result.get(&cid_c), Some(FetchOutcome::Block(d)) if *d == data_c)); - } - - #[tokio::test] - async fn request_bitswap_blocks_dont_have_is_surfaced_as_missing() { - let data_a = b"a".to_vec(); - let data_b = b"b".to_vec(); - let cid_a = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data_a); - let cid_b = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data_b); - let cid_c = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, b"c-not-served"); - - let response = encode_response( - &[ - (BLAKE2B_256_MULTIHASH_CODE, data_a.clone()), - (BLAKE2B_256_MULTIHASH_CODE, data_b.clone()), - ], - &[(cid_c, BlockPresenceType::DontHave as i32)], - ); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[cid_a, cid_b, cid_c]) - .await - .unwrap(); - - assert_eq!(result.len(), 3); - assert!(matches!(result.get(&cid_a), Some(FetchOutcome::Block(_)))); - assert!(matches!(result.get(&cid_b), Some(FetchOutcome::Block(_)))); - assert!(matches!(result.get(&cid_c), Some(FetchOutcome::Missing))); - } - - #[tokio::test] - async fn request_bitswap_blocks_corrupted_data_dropped_as_unsolicited() { - let real_data = b"real-payload".to_vec(); - let wanted_cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &real_data); - let corrupted_data = b"i-am-not-the-real-payload".to_vec(); - let response = encode_response(&[(BLAKE2B_256_MULTIHASH_CODE, corrupted_data)], &[]); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[wanted_cid]).await.unwrap(); - - assert_eq!(result.len(), 1); - assert!(matches!(result.get(&wanted_cid), Some(FetchOutcome::Missing))); - } - - #[tokio::test] - async fn request_bitswap_blocks_encodes_only_want_block_entries() { - let cid_a = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [1u8; 32]); - let cid_b = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [2u8; 32]); - let stub = StubSender::new([Ok(BitswapMessage::default().encode_to_vec())]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[cid_a, cid_b]) - .await - .expect("block-only request must encode"); - - assert!(matches!(result.get(&cid_a), Some(FetchOutcome::Missing))); - assert!(matches!(result.get(&cid_b), Some(FetchOutcome::Missing))); - - let request = stub.pop_request(); - let entries = request.wantlist.expect("wantlist should be present").entries; - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].want_type, ProtoWantType::Block as i32); - assert_eq!(entries[1].want_type, ProtoWantType::Block as i32); - } - - #[tokio::test] - async fn request_bitswap_blocks_have_presence_alone_is_missing() { - let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [3u8; 32]); - let response = encode_response(&[], &[(cid, BlockPresenceType::Have as i32)]); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[cid]) - .await - .expect("HAVE-only response should classify successfully"); - - assert_eq!(result.len(), 1); - assert!(matches!(result.get(&cid), Some(FetchOutcome::Missing))); - } - - #[tokio::test] - async fn request_bitswap_blocks_unverified_accepts_bytes_without_hash_recompute() { - let data = b"sha2-digest-but-blake2b-request-prefix".to_vec(); - let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, sp_crypto_hashing::sha2_256(&data)); - let response = encode_response(&[(BLAKE2B_256_MULTIHASH_CODE, data.clone())], &[]); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks_unverified(&stub, PeerId::random(), &[cid]) - .await - .expect("unverified fetch should not recompute hashes"); - - assert_eq!(result.len(), 1); - assert!(matches!(result.get(&cid), Some(FetchOutcome::Block(d)) if *d == data)); - } - - #[tokio::test] - async fn request_bitswap_blocks_unverified_dont_have_returned_as_missing() { - let cid = cid_for_digest( - BLAKE2B_256_MULTIHASH_CODE, - sp_crypto_hashing::sha2_256(b"pruned-unverified-payload"), - ); - let response = encode_response(&[], &[(cid, BlockPresenceType::DontHave as i32)]); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks_unverified(&stub, PeerId::random(), &[cid]) - .await - .expect("unverified DONT_HAVE should classify successfully"); - - assert_eq!(result.len(), 1); - assert!(matches!(result.get(&cid), Some(FetchOutcome::Missing))); - } - - #[tokio::test] - async fn request_bitswap_blocks_unverified_empty_wants_errors() { - let stub = StubSender::new(std::iter::empty()); - - let err = request_bitswap_blocks_unverified(&stub, PeerId::random(), &[]) - .await - .expect_err("empty wantlist must error"); - assert!(matches!(err, BitswapError::DecodeError(msg) if msg == "empty wantlist")); - } - - #[tokio::test] - async fn request_bitswap_blocks_duplicate_cids_error() { - let cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [9u8; 32]); - let stub = StubSender::new(std::iter::empty()); - - let err = request_bitswap_blocks(&stub, PeerId::random(), &[cid, cid]) - .await - .expect_err("two wants for the same CID are ambiguous"); - assert!(matches!(err, BitswapError::DecodeError(msg) if msg.starts_with("duplicate CID"))); - } - - #[tokio::test] - async fn request_bitswap_blocks_unverified_multi_want_all_served_in_request_order() { - let data_a = b"first-unverified-payload".to_vec(); - let data_b = b"second-unverified-payload".to_vec(); - let data_c = b"third-unverified-payload".to_vec(); - let cid_a = - cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, sp_crypto_hashing::sha2_256(&data_a)); - let cid_b = - cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, sp_crypto_hashing::keccak_256(&data_b)); - let cid_c = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data_c); - - let response = encode_response( - &[ - (BLAKE2B_256_MULTIHASH_CODE, data_a.clone()), - (BLAKE2B_256_MULTIHASH_CODE, data_b.clone()), - (BLAKE2B_256_MULTIHASH_CODE, data_c.clone()), - ], - &[], - ); - let stub = StubSender::new([Ok(response)]); - - let result = - request_bitswap_blocks_unverified(&stub, PeerId::random(), &[cid_a, cid_b, cid_c]) - .await - .expect("multi-want unverified must succeed via positional correlation"); - - assert_eq!(result.len(), 3); - assert!(matches!(result.get(&cid_a), Some(FetchOutcome::Block(d)) if *d == data_a)); - assert!(matches!(result.get(&cid_b), Some(FetchOutcome::Block(d)) if *d == data_b)); - assert!(matches!(result.get(&cid_c), Some(FetchOutcome::Block(d)) if *d == data_c)); - } - - #[tokio::test] - async fn request_bitswap_blocks_unverified_dont_have_skips_position_in_payload_order() { - let data = b"second-payload-after-dont-have".to_vec(); - let dont_have_cid = cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, [4u8; 32]); - let block_cid = - cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, sp_crypto_hashing::sha2_256(&data)); - let response = encode_response( - &[(BLAKE2B_256_MULTIHASH_CODE, data.clone())], - &[(dont_have_cid, BlockPresenceType::DontHave as i32)], - ); - let stub = StubSender::new([Ok(response)]); - - let result = - request_bitswap_blocks_unverified(&stub, PeerId::random(), &[dont_have_cid, block_cid]) - .await - .expect("unverified mixed presence/payload should classify successfully"); - - assert_eq!(result.len(), 2); - assert!(matches!(result.get(&dont_have_cid), Some(FetchOutcome::Missing))); - assert!(matches!(result.get(&block_cid), Some(FetchOutcome::Block(d)) if *d == data)); - } - - #[tokio::test] - async fn request_bitswap_blocks_dispatches_per_entry_multihash() { - let data_b2 = b"blake2b-payload".to_vec(); - let data_sha = b"sha2-256-payload".to_vec(); - let data_kec = b"keccak-256-payload".to_vec(); - let cid_b2 = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data_b2); - let cid_sha = cid_for_data(SHA2_256_MULTIHASH_CODE, &data_sha); - let cid_kec = cid_for_data(KECCAK_256_MULTIHASH_CODE, &data_kec); - - let response = encode_response( - &[ - (BLAKE2B_256_MULTIHASH_CODE, data_b2.clone()), - (SHA2_256_MULTIHASH_CODE, data_sha.clone()), - (KECCAK_256_MULTIHASH_CODE, data_kec.clone()), - ], - &[], - ); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[cid_b2, cid_sha, cid_kec]) - .await - .unwrap(); - - assert_eq!(result.len(), 3); - assert!(matches!(result.get(&cid_b2), Some(FetchOutcome::Block(d)) if *d == data_b2)); - assert!(matches!(result.get(&cid_sha), Some(FetchOutcome::Block(d)) if *d == data_sha)); - assert!(matches!(result.get(&cid_kec), Some(FetchOutcome::Block(d)) if *d == data_kec)); - } - - #[tokio::test] - async fn request_bitswap_blocks_over_cap_errors() { - let wants: Vec<_> = (0..(MAX_WANTED_BLOCKS + 1) as u8) - .map(|i| { - let mut h = [0u8; 32]; - h[0] = i; - cid_for_digest(BLAKE2B_256_MULTIHASH_CODE, h) - }) - .collect(); - let stub = StubSender::new(std::iter::empty()); - - let err = request_bitswap_blocks(&stub, PeerId::random(), &wants) - .await - .expect_err("over-cap wantlist must error"); - assert!(matches!(err, BitswapError::DecodeError(_))); - } - - #[tokio::test] - async fn request_bitswap_blocks_at_exactly_max_wanted_blocks_succeeds() { - let mut wants = Vec::with_capacity(MAX_WANTED_BLOCKS); - let mut blocks = Vec::with_capacity(MAX_WANTED_BLOCKS); - for i in 0..MAX_WANTED_BLOCKS { - let data = format!("payload-{i}").into_bytes(); - wants.push(cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data)); - blocks.push((BLAKE2B_256_MULTIHASH_CODE, data)); - } - - let response = encode_response(&blocks, &[]); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &wants) - .await - .expect("exactly MAX_WANTED_BLOCKS must succeed"); - - assert_eq!(result.len(), MAX_WANTED_BLOCKS); - for cid in &wants { - assert!(matches!(result.get(cid), Some(FetchOutcome::Block(_)))); - } - } - - #[tokio::test] - async fn request_bitswap_blocks_block_beats_presence_for_same_cid() { - let data = b"both-block-and-presence".to_vec(); - let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &data); - - let response = encode_response( - &[(BLAKE2B_256_MULTIHASH_CODE, data.clone())], - &[(cid, BlockPresenceType::DontHave as i32)], - ); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[cid]).await.unwrap(); - - assert_eq!(result.len(), 1); - assert!(matches!(result.get(&cid), Some(FetchOutcome::Block(d)) if *d == data)); - } - - #[tokio::test] - async fn request_bitswap_blocks_response_decode_failure() { - let stub = StubSender::new([Ok(vec![0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])]); - let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, b"any"); - - let err = request_bitswap_blocks(&stub, PeerId::random(), &[cid]) - .await - .expect_err("malformed response bytes must surface as DecodeError"); - assert!(matches!(err, BitswapError::DecodeError(_))); - } - - #[tokio::test] - async fn request_bitswap_blocks_request_failure_propagates() { - struct FailingSender; - #[async_trait::async_trait] - impl NetworkRequest for FailingSender { - async fn request( - &self, - _target: PeerId, - _protocol: ProtocolName, - _request: Vec, - _fallback_request: Option<(Vec, ProtocolName)>, - _connect: IfDisconnected, - ) -> Result<(Vec, ProtocolName), RequestFailure> { - Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)) - } - - fn start_request( - &self, - _peer: PeerId, - _protocol: ProtocolName, - _payload: Vec, - _fallback_request: Option<(Vec, ProtocolName)>, - tx: oneshot::Sender, ProtocolName), RequestFailure>>, - _connect: IfDisconnected, - ) { - drop(tx); - } - } - - let cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, b"any"); - let err = request_bitswap_blocks(&FailingSender, PeerId::random(), &[cid]) - .await - .expect_err("request failure must surface as RequestFailed"); - assert!(matches!(err, BitswapError::RequestFailed(_))); - } - - #[tokio::test] - async fn request_bitswap_blocks_unsupported_multihash_in_block_dropped() { - let wanted_data = b"wanted".to_vec(); - let wanted_cid = cid_for_data(BLAKE2B_256_MULTIHASH_CODE, &wanted_data); - const UNSUPPORTED_MH_CODE: u64 = 0x99; - let bad_prefix = Prefix { - version: CidVersion::V1, - codec: RAW_CODEC, - mh_type: UNSUPPORTED_MH_CODE, - mh_len: 32, - } - .to_bytes(); - - let mut payload_msg = BitswapMessage::default(); - payload_msg.payload = - vec![MessageBlock { prefix: bad_prefix, data: b"some-bytes".to_vec() }]; - let response = payload_msg.encode_to_vec(); - let stub = StubSender::new([Ok(response)]); - - let result = request_bitswap_blocks(&stub, PeerId::random(), &[wanted_cid]).await.unwrap(); - - assert_eq!(result.len(), 1); - assert!(matches!(result.get(&wanted_cid), Some(FetchOutcome::Missing))); - } - - #[test] - fn cid_from_block_prefix_rejects_cid_v0_as_unsupported() { - let prefix = Prefix { - version: CidVersion::V0, - codec: RAW_CODEC, - mh_type: BLAKE2B_256_MULTIHASH_CODE, - mh_len: 32, - } - .to_bytes(); - - let err = - cid_from_block_prefix(&prefix, b"payload").expect_err("CIDv0 must be unsupported"); - assert!(matches!(err, BitswapError::UnsupportedCidVersion { version: 0 })); - } -} diff --git a/substrate/client/network/src/bitswap/mod.rs b/substrate/client/network/src/bitswap/mod.rs deleted file mode 100644 index 3d37fc568e45..000000000000 --- a/substrate/client/network/src/bitswap/mod.rs +++ /dev/null @@ -1,667 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Substrate. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Bitswap server for Substrate. -//! -//! Supports querying indexed transactions by hash over the standard bitswap protocol (v1.2.0). -//! CIDs must reference a supported 256-bit transaction hash. - -use crate::{ - request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig}, - types::ProtocolName, - MAX_RESPONSE_SIZE, -}; - -use cid::{Error as CidError, Version as CidVersion}; -use futures::StreamExt; -use log::{debug, error, trace}; -use prost::Message; -use sc_client_api::BlockBackend; -use sc_network_types::PeerId; -use schema::bitswap::{ - message::{wantlist::WantType, Block as MessageBlock, BlockPresence, BlockPresenceType}, - Message as BitswapMessage, -}; -use sp_core::H256; -use sp_runtime::traits::Block as BlockT; -use std::{io, sync::Arc, time::Duration}; -use unsigned_varint::encode as varint_encode; - -/// Bitswap client. -mod client; -/// Bitswap protobuf schema, generated from the protocol definitions. -pub mod schema; - -pub use cid::Cid; - -pub use client::{ - request_bitswap_blocks, request_bitswap_blocks_unverified, BitswapError, FetchOutcome, - BLAKE2B_256_MULTIHASH_CODE, KECCAK_256_MULTIHASH_CODE, SHA2_256_MULTIHASH_CODE, -}; - -pub(crate) use schema::bitswap::Message as BitswapProtoMessage; - -pub(crate) const LOG_TARGET: &str = "sub-libp2p::bitswap"; - -// Use the network-wide response cap for Bitswap messages. -const MAX_PACKET_SIZE: u64 = MAX_RESPONSE_SIZE; - -/// Max number of queued responses before denying requests. -const MAX_REQUEST_QUEUE: usize = 20; - -/// Max number of blocks per wantlist. -pub const MAX_WANTED_BLOCKS: usize = 16; - -/// Bitswap protocol name. -pub(crate) const PROTOCOL_NAME: &str = "/ipfs/bitswap/1.2.0"; - -/// IPFS raw multicodec used for indexed transaction payload bytes. -pub const RAW_CODEC: u64 = 0x55; - -/// Check if a CID is supported by the bitswap protocol — CIDv1, 32-byte digest, with a -/// supported multihash code (Blake2b-256, SHA2-256, or Keccak-256). -pub fn is_cid_supported(cid: &Cid) -> bool { - cid.version() != CidVersion::V0 && - cid.hash().size() == 32 && - is_supported_multihash_code(cid.hash().code()) -} - -/// Return `true` if `code` is a supported multihash code. -pub(crate) fn is_supported_multihash_code(code: u64) -> bool { - matches!(code, BLAKE2B_256_MULTIHASH_CODE | SHA2_256_MULTIHASH_CODE | KECCAK_256_MULTIHASH_CODE) -} - -/// CID metadata without the actual content bytes. -#[derive(PartialEq, Eq, Clone, Debug)] -pub struct Prefix { - /// The version of CID. - pub version: CidVersion, - /// The codec of CID. - pub codec: u64, - /// The multihash type of CID. - pub mh_type: u64, - /// The multihash length of CID. - pub mh_len: u8, -} - -impl From<&Cid> for Prefix { - fn from(cid: &Cid) -> Self { - Self { - version: cid.version(), - codec: cid.codec(), - mh_type: cid.hash().code(), - mh_len: cid.hash().size(), - } - } -} - -impl Prefix { - /// Convert the prefix to encoded bytes. - pub fn to_bytes(&self) -> Vec { - let mut res = Vec::with_capacity(4); - let mut buf = varint_encode::u64_buffer(); - let version = varint_encode::u64(self.version.into(), &mut buf); - res.extend_from_slice(version); - let mut buf = varint_encode::u64_buffer(); - let codec = varint_encode::u64(self.codec, &mut buf); - res.extend_from_slice(codec); - let mut buf = varint_encode::u64_buffer(); - let mh_type = varint_encode::u64(self.mh_type, &mut buf); - res.extend_from_slice(mh_type); - let mut buf = varint_encode::u64_buffer(); - let mh_len = varint_encode::u64(self.mh_len as u64, &mut buf); - res.extend_from_slice(mh_len); - res - } -} - -/// Bitswap request handler. -pub(crate) struct BitswapRequestHandler { - client: Arc + Send + Sync>, - request_receiver: async_channel::Receiver, -} - -impl BitswapRequestHandler { - /// Create a new [`BitswapRequestHandler`]. - pub(crate) fn new(client: Arc + Send + Sync>) -> (Self, ProtocolConfig) { - let (tx, request_receiver) = async_channel::bounded(MAX_REQUEST_QUEUE); - - let config = ProtocolConfig { - name: ProtocolName::from(PROTOCOL_NAME), - fallback_names: vec![], - max_request_size: MAX_PACKET_SIZE, - max_response_size: MAX_PACKET_SIZE, - request_timeout: Duration::from_secs(15), - inbound_queue: Some(tx), - }; - - (Self { client, request_receiver }, config) - } - - /// Run [`BitswapRequestHandler`]. - pub(crate) async fn run(mut self) { - while let Some(request) = self.request_receiver.next().await { - let IncomingRequest { peer, payload, pending_response } = request; - - match self.handle_message(&peer, &payload) { - Ok(response) => { - let response = OutgoingResponse { - result: Ok(response), - reputation_changes: Vec::new(), - sent_feedback: None, - }; - - match pending_response.send(response) { - Ok(()) => { - trace!(target: LOG_TARGET, "Handled bitswap request from {peer}.",) - }, - Err(_) => debug!( - target: LOG_TARGET, - "Failed to handle bitswap request from {peer}: {}", - RequestHandlerError::SendResponse, - ), - } - }, - Err(err) => { - error!(target: LOG_TARGET, "Failed to process request from {peer}: {err}"); - - // TODO: adjust reputation? - - let response = OutgoingResponse { - result: Err(()), - reputation_changes: vec![], - sent_feedback: None, - }; - - if pending_response.send(response).is_err() { - debug!( - target: LOG_TARGET, - "Failed to handle bitswap request from {peer}: {}", - RequestHandlerError::SendResponse, - ); - } - }, - } - } - } - - /// Handle received Bitswap request - fn handle_message( - &mut self, - peer: &PeerId, - payload: &[u8], - ) -> Result, RequestHandlerError> { - let request = schema::bitswap::Message::decode(payload)?; - - trace!(target: LOG_TARGET, "Received request: {:?} from {}", request, peer); - - let mut response = BitswapMessage::default(); - - let wantlist = match request.wantlist { - Some(wantlist) => wantlist, - None => { - debug!(target: LOG_TARGET, "Unexpected bitswap message from {}", peer); - return Err(RequestHandlerError::InvalidWantList); - }, - }; - - if wantlist.entries.len() > MAX_WANTED_BLOCKS { - trace!(target: LOG_TARGET, "Ignored request: too many entries"); - return Err(RequestHandlerError::TooManyEntries); - } - - for entry in wantlist.entries { - let cid = match Cid::read_bytes(entry.block.as_slice()) { - Ok(cid) => cid, - Err(e) => { - trace!(target: LOG_TARGET, "Bad CID {:?}: {:?}", entry.block, e); - continue; - }, - }; - - if !is_cid_supported(&cid) { - trace!(target: LOG_TARGET, "Ignoring unsupported CID {}: {}", peer, cid); - continue; - } - - let mut hash = H256::default(); - hash.as_mut().copy_from_slice(&cid.hash().digest()[0..32]); - let transaction = match self.client.indexed_transaction(hash) { - Ok(ex) => ex, - Err(e) => { - error!(target: LOG_TARGET, "Error retrieving transaction {}: {}", hash, e); - None - }, - }; - - match transaction { - Some(transaction) => { - trace!(target: LOG_TARGET, "Found CID {:?}, hash {:?}", cid, hash); - - if entry.want_type == WantType::Block as i32 { - let prefix: Prefix = (&cid).into(); - response - .payload - .push(MessageBlock { prefix: prefix.to_bytes(), data: transaction }); - } else { - response.block_presences.push(BlockPresence { - r#type: BlockPresenceType::Have as i32, - cid: cid.to_bytes(), - }); - } - }, - None => { - trace!(target: LOG_TARGET, "Missing CID {:?}, hash {:?}", cid, hash); - - if entry.send_dont_have { - response.block_presences.push(BlockPresence { - r#type: BlockPresenceType::DontHave as i32, - cid: cid.to_bytes(), - }); - } - }, - } - } - - Ok(response.encode_to_vec()) - } -} - -/// Bitswap protocol error. -#[derive(Debug, thiserror::Error)] -enum RequestHandlerError { - /// Protobuf decoding error. - #[error("Failed to decode request: {0}.")] - DecodeProto(#[from] prost::DecodeError), - - /// Protobuf encoding error. - #[error("Failed to encode response: {0}.")] - EncodeProto(#[from] prost::EncodeError), - - /// Client backend error. - #[error(transparent)] - Client(#[from] sp_blockchain::Error), - - /// Error parsing CID - #[error(transparent)] - BadCid(#[from] CidError), - - /// Packet read error. - #[error(transparent)] - Read(#[from] io::Error), - - /// Error sending response. - #[error("Failed to send response.")] - SendResponse, - - /// Message doesn't have a WANT list. - #[error("Invalid WANT list.")] - InvalidWantList, - - /// Too many blocks requested. - #[error("Too many block entries in the request.")] - TooManyEntries, -} - -#[cfg(test)] -mod tests { - use super::*; - use futures::channel::oneshot; - use litep2p::types::multihash::Code as LiteP2pCode; - use sc_block_builder::BlockBuilderBuilder; - use schema::bitswap::{ - message::{wantlist::Entry, Wantlist}, - Message as BitswapMessage, - }; - use sp_consensus::BlockOrigin; - use sp_runtime::codec::Encode; - use substrate_test_runtime::ExtrinsicBuilder; - use substrate_test_runtime_client::{self, prelude::*, TestClientBuilder}; - - #[tokio::test] - async fn undecodable_message() { - let client = substrate_test_runtime_client::new(); - let (bitswap, config) = BitswapRequestHandler::new(Arc::new(client)); - - tokio::spawn(async move { bitswap.run().await }); - - let (tx, rx) = oneshot::channel(); - config - .inbound_queue - .unwrap() - .send(IncomingRequest { - peer: PeerId::random(), - payload: vec![0x13, 0x37, 0x13, 0x38], - pending_response: tx, - }) - .await - .unwrap(); - - if let Ok(OutgoingResponse { result, reputation_changes, sent_feedback }) = rx.await { - assert_eq!(result, Err(())); - assert_eq!(reputation_changes, Vec::new()); - assert!(sent_feedback.is_none()); - } else { - panic!("invalid event received"); - } - } - - #[tokio::test] - async fn empty_want_list() { - let client = substrate_test_runtime_client::new(); - let (bitswap, mut config) = BitswapRequestHandler::new(Arc::new(client)); - - tokio::spawn(async move { bitswap.run().await }); - - let (tx, rx) = oneshot::channel(); - config - .inbound_queue - .as_mut() - .unwrap() - .send(IncomingRequest { - peer: PeerId::random(), - payload: BitswapMessage { wantlist: None, ..Default::default() }.encode_to_vec(), - pending_response: tx, - }) - .await - .unwrap(); - - if let Ok(OutgoingResponse { result, reputation_changes, sent_feedback }) = rx.await { - assert_eq!(result, Err(())); - assert_eq!(reputation_changes, Vec::new()); - assert!(sent_feedback.is_none()); - } else { - panic!("invalid event received"); - } - - // Empty WANT list should not cause an error - let (tx, rx) = oneshot::channel(); - config - .inbound_queue - .unwrap() - .send(IncomingRequest { - peer: PeerId::random(), - payload: BitswapMessage { - wantlist: Some(Default::default()), - ..Default::default() - } - .encode_to_vec(), - pending_response: tx, - }) - .await - .unwrap(); - - if let Ok(OutgoingResponse { result, reputation_changes, sent_feedback }) = rx.await { - assert_eq!(result, Ok(BitswapMessage::default().encode_to_vec())); - assert_eq!(reputation_changes, Vec::new()); - assert!(sent_feedback.is_none()); - } else { - panic!("invalid event received"); - } - } - - #[tokio::test] - async fn too_long_want_list() { - let client = substrate_test_runtime_client::new(); - let (bitswap, config) = BitswapRequestHandler::new(Arc::new(client)); - - tokio::spawn(async move { bitswap.run().await }); - - let (tx, rx) = oneshot::channel(); - config - .inbound_queue - .unwrap() - .send(IncomingRequest { - peer: PeerId::random(), - payload: BitswapMessage { - wantlist: Some(Wantlist { - entries: (0..MAX_WANTED_BLOCKS + 1) - .map(|_| Entry::default()) - .collect::>(), - full: false, - }), - ..Default::default() - } - .encode_to_vec(), - pending_response: tx, - }) - .await - .unwrap(); - - if let Ok(OutgoingResponse { result, reputation_changes, sent_feedback }) = rx.await { - assert_eq!(result, Err(())); - assert_eq!(reputation_changes, Vec::new()); - assert!(sent_feedback.is_none()); - } else { - panic!("invalid event received"); - } - } - - #[tokio::test] - async fn transaction_not_found() { - let client = TestClientBuilder::with_tx_storage(u32::MAX).build(); - - let (bitswap, config) = BitswapRequestHandler::new(Arc::new(client)); - tokio::spawn(async move { bitswap.run().await }); - - let (tx, rx) = oneshot::channel(); - config - .inbound_queue - .unwrap() - .send(IncomingRequest { - peer: PeerId::random(), - payload: BitswapMessage { - wantlist: Some(Wantlist { - entries: vec![Entry { - block: cid::Cid::new_v1( - 0x70, - cid::multihash::Multihash::wrap( - u64::from(LiteP2pCode::Blake2b256), - &[0u8; 32], - ) - .unwrap(), - ) - .to_bytes(), - ..Default::default() - }], - full: false, - }), - ..Default::default() - } - .encode_to_vec(), - pending_response: tx, - }) - .await - .unwrap(); - - if let Ok(OutgoingResponse { result, reputation_changes, sent_feedback }) = rx.await { - assert_eq!(result, Ok(vec![])); - assert_eq!(reputation_changes, Vec::new()); - assert!(sent_feedback.is_none()); - } else { - panic!("invalid event received"); - } - } - - #[tokio::test] - async fn transaction_found() { - let client = TestClientBuilder::with_tx_storage(u32::MAX).build(); - let mut block_builder = BlockBuilderBuilder::new(&client) - .on_parent_block(client.chain_info().genesis_hash) - .with_parent_block_number(0) - .build() - .unwrap(); - - // encoded extrinsic: [161, .. , 2, 6, 16, 19, 55, 19, 56] - let ext = ExtrinsicBuilder::new_indexed_call(vec![0x13, 0x37, 0x13, 0x38]).build(); - let pattern_index = ext.encoded_size() - 4; - - block_builder.push(ext.clone()).unwrap(); - let block = block_builder.build().unwrap().block; - - client.import(BlockOrigin::File, block).await.unwrap(); - - let (bitswap, config) = BitswapRequestHandler::new(Arc::new(client)); - - tokio::spawn(async move { bitswap.run().await }); - - let (tx, rx) = oneshot::channel(); - config - .inbound_queue - .unwrap() - .send(IncomingRequest { - peer: PeerId::random(), - payload: BitswapMessage { - wantlist: Some(Wantlist { - entries: vec![Entry { - block: cid::Cid::new_v1( - 0x70, - cid::multihash::Multihash::wrap( - u64::from(LiteP2pCode::Blake2b256), - &sp_crypto_hashing::blake2_256(&ext.encode()[pattern_index..]), - ) - .unwrap(), - ) - .to_bytes(), - ..Default::default() - }], - full: false, - }), - ..Default::default() - } - .encode_to_vec(), - pending_response: tx, - }) - .await - .unwrap(); - - if let Ok(OutgoingResponse { result, reputation_changes, sent_feedback }) = rx.await { - assert_eq!(reputation_changes, Vec::new()); - assert!(sent_feedback.is_none()); - - let response = - schema::bitswap::Message::decode(&result.expect("fetch to succeed")[..]).unwrap(); - assert_eq!(response.payload[0].data, vec![0x13, 0x37, 0x13, 0x38]); - } else { - panic!("invalid event received"); - } - } - - #[tokio::test] - async fn transaction_not_found_sends_dont_have_when_requested() { - let client = TestClientBuilder::with_tx_storage(u32::MAX).build(); - let (mut bitswap, _config) = BitswapRequestHandler::new(Arc::new(client)); - let cid = cid::Cid::new_v1( - 0x70, - cid::multihash::Multihash::wrap(u64::from(LiteP2pCode::Blake2b256), &[0u8; 32]) - .unwrap(), - ); - let request = BitswapMessage { - wantlist: Some(Wantlist { - entries: vec![Entry { - block: cid.to_bytes(), - send_dont_have: true, - ..Default::default() - }], - full: false, - }), - ..Default::default() - } - .encode_to_vec(); - - let response = BitswapMessage::decode( - bitswap.handle_message(&PeerId::random(), &request).unwrap().as_slice(), - ) - .unwrap(); - - assert!(response.payload.is_empty()); - assert_eq!(response.block_presences.len(), 1); - assert_eq!(response.block_presences[0].cid, cid.to_bytes()); - assert_eq!(response.block_presences[0].r#type, BlockPresenceType::DontHave as i32); - } - - #[tokio::test] - async fn transaction_found_sends_have_for_want_have() { - let client = TestClientBuilder::with_tx_storage(u32::MAX).build(); - let mut block_builder = BlockBuilderBuilder::new(&client) - .on_parent_block(client.chain_info().genesis_hash) - .with_parent_block_number(0) - .build() - .unwrap(); - - let ext = ExtrinsicBuilder::new_indexed_call(vec![0x13, 0x37, 0x13, 0x38]).build(); - let pattern_index = ext.encoded_size() - 4; - let cid = cid::Cid::new_v1( - 0x70, - cid::multihash::Multihash::wrap( - u64::from(LiteP2pCode::Blake2b256), - &sp_crypto_hashing::blake2_256(&ext.encode()[pattern_index..]), - ) - .unwrap(), - ); - - block_builder.push(ext).unwrap(); - let block = block_builder.build().unwrap().block; - client.import(BlockOrigin::File, block).await.unwrap(); - - let (mut bitswap, _config) = BitswapRequestHandler::new(Arc::new(client)); - let request = BitswapMessage { - wantlist: Some(Wantlist { - entries: vec![Entry { - block: cid.to_bytes(), - want_type: WantType::Have as i32, - ..Default::default() - }], - full: false, - }), - ..Default::default() - } - .encode_to_vec(); - - let response = BitswapMessage::decode( - bitswap.handle_message(&PeerId::random(), &request).unwrap().as_slice(), - ) - .unwrap(); - - assert!(response.payload.is_empty()); - assert_eq!(response.block_presences.len(), 1); - assert_eq!(response.block_presences[0].cid, cid.to_bytes()); - assert_eq!(response.block_presences[0].r#type, BlockPresenceType::Have as i32); - } - - #[test] - fn is_cid_supported_accepts_all_three_supported_hashings() { - use cid::multihash::Multihash; - for multihash_code in - [BLAKE2B_256_MULTIHASH_CODE, SHA2_256_MULTIHASH_CODE, KECCAK_256_MULTIHASH_CODE] - { - let digest = [9u8; 32]; - let mh = Multihash::<64>::wrap(multihash_code, &digest).unwrap(); - let cid = Cid::new_v1(RAW_CODEC, mh); - assert!(is_cid_supported(&cid), "{multihash_code} CID should be supported"); - } - } - - #[test] - fn is_cid_supported_rejects_unknown_multihash_code() { - use cid::multihash::Multihash; - let digest = [9u8; 32]; - let mh = Multihash::<64>::wrap(0x99, &digest).unwrap(); - let cid = Cid::new_v1(RAW_CODEC, mh); - assert!(!is_cid_supported(&cid)); - } -} diff --git a/substrate/client/network/src/bitswap/schema.rs b/substrate/client/network/src/bitswap/schema.rs deleted file mode 100644 index 07a770540f1a..000000000000 --- a/substrate/client/network/src/bitswap/schema.rs +++ /dev/null @@ -1,24 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Include sources generated from protobuf definitions. - -#[allow(missing_docs)] -pub mod bitswap { - include!(concat!(env!("OUT_DIR"), "/bitswap.message.rs")); -} diff --git a/substrate/client/network/src/config.rs b/substrate/client/network/src/config.rs index 53ed03a51d36..c360a29bba8e 100644 --- a/substrate/client/network/src/config.rs +++ b/substrate/client/network/src/config.rs @@ -36,6 +36,10 @@ pub use crate::{ }; pub use sc_network_types::{build_multiaddr, ed25519}; + +/// Litep2p transport-side Bitswap handle, created together with [`IpfsConfig`] via +/// [`IpfsConfig::new`]. Re-exported so callers need not depend on `litep2p` directly. +pub use litep2p::protocol::libp2p::bitswap::BitswapHandle as LitepBitswapHandle; use sc_network_types::{ multiaddr::{self, Multiaddr}, PeerId, @@ -766,15 +770,27 @@ impl NetworkConfiguration { } /// IPFS server configuration. -pub struct IpfsConfig> { - /// Network-backend-specific Bitswap configuration. - pub bitswap_config: N::BitswapConfig, +pub struct IpfsConfig { + /// Litep2p Bitswap protocol config, consumed by the litep2p network backend. + pub litep2p_bitswap_config: litep2p::protocol::libp2p::bitswap::Config, /// Indexed transactions provider. pub block_provider: Box, /// IPFS bootstrap nodes. pub bootnodes: Vec, } +impl IpfsConfig { + /// Construct an [`IpfsConfig`] together with the litep2p transport-side Bitswap handle. + pub fn new( + block_provider: Box, + bootnodes: Vec, + ) -> (Self, LitepBitswapHandle) { + let (litep2p_bitswap_config, litep2p_handle) = + litep2p::protocol::libp2p::bitswap::Config::new(); + (Self { litep2p_bitswap_config, block_provider, bootnodes }, litep2p_handle) + } +} + /// Network initialization parameters. pub struct Params> { /// Assigned role for our node (full, light, ...). @@ -803,7 +819,7 @@ pub struct Params> { pub block_announce_config: N::NotificationProtocolConfig, /// Bitswap configuration, if the server has been enabled. - pub ipfs_config: Option>, + pub ipfs_config: Option, /// Notification metrics. pub notification_metrics: NotificationMetrics, diff --git a/substrate/client/network/src/lib.rs b/substrate/client/network/src/lib.rs index dcf97fd14135..d029123960bc 100644 --- a/substrate/client/network/src/lib.rs +++ b/substrate/client/network/src/lib.rs @@ -243,7 +243,6 @@ //! More precise usage details are still being worked on and will likely change in the future. mod behaviour; -pub mod bitswap; mod ipfs_block_provider; mod litep2p; mod protocol; diff --git a/substrate/client/network/src/litep2p/bitswap.rs b/substrate/client/network/src/litep2p/bitswap.rs deleted file mode 100644 index 7cc0a95b41ce..000000000000 --- a/substrate/client/network/src/litep2p/bitswap.rs +++ /dev/null @@ -1,703 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Bidirectional bitswap shim for litep2p. -//! -//! Wraps litep2p's native [`BitswapHandle`] to provide both server-side (inbound WANT handling) -//! and client-side (outbound WANT dispatch + response correlation) functionality. - -use crate::{ - bitswap::{ - is_cid_supported, - schema::bitswap::message::{ - Block as MessageBlock, BlockPresence, BlockPresenceType as ProtoPresenceType, - }, - BitswapProtoMessage, Cid, Prefix, LOG_TARGET, MAX_WANTED_BLOCKS, PROTOCOL_NAME, - }, - litep2p::bitswap_metrics::{errors, outcomes, BitswapMetrics}, - request_responses::RequestFailure, - OutboundFailure, ProtocolName, MAX_RESPONSE_SIZE, -}; -use futures::{channel::oneshot, StreamExt}; -use litep2p::protocol::libp2p::bitswap::{ - BitswapEvent, BitswapHandle, BlockPresenceType, Config, ResponseType, WantType, -}; -use prometheus_endpoint::Registry; -use prost::Message as ProstMessage; -use sc_client_api::BlockBackend; -use sp_core::H256; -use sp_runtime::traits::Block as BlockT; -use std::{ - collections::HashMap, - future::Future, - pin::Pin, - sync::Arc, - time::{Duration, Instant}, -}; -use tokio::sync::mpsc; - -/// Command channel capacity. -const CMD_CHANNEL_CAPACITY: usize = 256; -/// Timeout for pending bitswap requests. -const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); -/// Interval for reaping expired pending batches. -const EXPIRY_TICK_INTERVAL: Duration = Duration::from_secs(10); - -pub(crate) type ResponseSender = oneshot::Sender, ProtocolName), RequestFailure>>; - -/// Outbound bitswap command sent from [`super::service::Litep2pNetworkService`]. -pub(crate) struct BitswapOutboundCmd { - pub(crate) peer: litep2p::PeerId, - pub(crate) wants: Vec<(Cid, WantType)>, - pub(crate) response_tx: ResponseSender, -} - -/// Pending outbound WANT batch. -struct PendingBatch { - cids: Vec, - responses: HashMap, - response_bytes: usize, - response_tx: Option, - inserted: Instant, -} - -impl PendingBatch { - fn new(cids: Vec, response_tx: ResponseSender, inserted: Instant) -> Self { - Self { - cids, - responses: HashMap::new(), - response_bytes: 0, - response_tx: Some(response_tx), - inserted, - } - } - - fn record_responses(&mut self, responses: &HashMap) { - for cid in &self.cids { - if self.responses.contains_key(cid) { - continue; - } - let Some(resp) = responses.get(cid) else { continue }; - self.response_bytes = self.response_bytes.saturating_add(response_retained_bytes(resp)); - self.responses.insert(*cid, resp.clone()); - } - } - - fn is_complete(&self) -> bool { - self.cids.len() == self.responses.len() - } - - fn is_over_limit(&self, max_response_bytes: usize) -> bool { - self.response_bytes > max_response_bytes - } - - fn is_expired(&self, timeout: Duration, now: Instant) -> bool { - now.saturating_duration_since(self.inserted) >= timeout - } - - fn send_success(&mut self) { - let responses: Vec = - self.cids.iter().filter_map(|cid| self.responses.get(cid).cloned()).collect(); - let encoded = encode_responses_as_bitswap_message(&responses); - if let Some(response_tx) = self.response_tx.take() { - let _ = response_tx.send(Ok((encoded, ProtocolName::from(PROTOCOL_NAME)))); - } - } - - fn send_failure(&mut self, failure: RequestFailure) { - if let Some(response_tx) = self.response_tx.take() { - let _ = response_tx.send(Err(failure)); - } - } -} - -/// Litep2p-specific bitswap configuration returned by [`BitswapService::new`]. -/// -/// Carries the native litep2p [`Config`] and the sender half of the command -/// channel so that [`super::service::Litep2pNetworkService`] can forward -/// client-side bitswap requests. -pub struct BitswapConfig { - pub(crate) litep2p_config: Config, - pub(crate) cmd_tx: mpsc::Sender, -} - -/// Pending outbound WANT batches, indexed by peer. -#[derive(Default)] -struct PendingBatches { - by_peer: HashMap>, -} - -impl PendingBatches { - fn insert(&mut self, peer: litep2p::PeerId, batch: PendingBatch) { - self.by_peer.entry(peer).or_default().push(batch); - } - - fn handle_response(&mut self, peer: litep2p::PeerId, responses: Vec) { - self.handle_response_with_limit(peer, responses, MAX_RESPONSE_SIZE as usize); - } - - fn handle_response_with_limit( - &mut self, - peer: litep2p::PeerId, - responses: Vec, - max_response_bytes: usize, - ) { - log::debug!( - target: LOG_TARGET, - "bitswap: received response from {peer:?} with {} entries", - responses.len() - ); - - let Some(peer_batches) = self.by_peer.get_mut(&peer) else { return }; - let best = select_best_response_per_cid(responses); - - peer_batches.retain_mut(|batch| { - batch.record_responses(&best); - - if batch.is_over_limit(max_response_bytes) { - log::warn!( - target: LOG_TARGET, - "bitswap: response from {peer:?} exceeded pending batch byte limit: {} > {}", - batch.response_bytes, - max_response_bytes, - ); - batch.send_failure(RequestFailure::Network(OutboundFailure::ConnectionClosed)); - false - } else if batch.is_complete() { - batch.send_success(); - false - } else { - true - } - }); - - if peer_batches.is_empty() { - self.by_peer.remove(&peer); - } - } - - fn expire(&mut self, timeout: Duration, now: Instant) { - self.by_peer.retain(|peer, peer_batches| { - peer_batches.retain_mut(|batch| { - if batch.is_expired(timeout, now) { - log::debug!( - target: LOG_TARGET, - "bitswap: expired pending batch for {} CIDs from {:?}", - batch.cids.len(), - peer, - ); - batch.send_failure(RequestFailure::Network(OutboundFailure::Timeout)); - false - } else { - true - } - }); - - !peer_batches.is_empty() - }); - } - - #[cfg(test)] - fn is_empty(&self) -> bool { - self.by_peer.is_empty() - } - - #[cfg(test)] - fn len(&self) -> usize { - self.by_peer.len() - } - - #[cfg(test)] - fn contains_key(&self, peer: &litep2p::PeerId) -> bool { - self.by_peer.contains_key(peer) - } -} - -/// Bidirectional bitswap service for litep2p. -pub(crate) struct BitswapService { - handle: BitswapHandle, - client: Arc + Send + Sync>, - cmd_rx: mpsc::Receiver, - pending: PendingBatches, - metrics: BitswapMetrics, -} - -impl BitswapService { - /// Create a new bidirectional bitswap service. - /// - /// Returns the boxed task future (to be spawned on the executor) and the - /// [`BitswapConfig`] to be passed into the litep2p config builder. - /// - /// If `metrics_registry` is `Some`, Prometheus metrics are registered with - /// it. A registration failure is logged and falls back to disabled metrics - /// (the service still runs). - pub(crate) fn new( - client: Arc + Send + Sync>, - metrics_registry: Option<&Registry>, - ) -> (Pin + Send>>, BitswapConfig) { - let metrics = BitswapMetrics::new(metrics_registry).unwrap_or_else(|err| { - log::debug!(target: LOG_TARGET, "failed to register bitswap metrics: {err}"); - BitswapMetrics::new(None).expect("registering with None registry never fails; qed") - }); - let (litep2p_config, handle) = Config::new(); - let (cmd_tx, cmd_rx) = mpsc::channel(CMD_CHANNEL_CAPACITY); - let service = Self { handle, client, cmd_rx, pending: PendingBatches::default(), metrics }; - let future = Box::pin(async move { service.run().await }); - let config = BitswapConfig { litep2p_config, cmd_tx }; - (future, config) - } - - /// Run the bitswap event loop. - async fn run(mut self) { - log::debug!(target: LOG_TARGET, "starting bidirectional bitswap service"); - let mut expiry_ticker = tokio::time::interval(EXPIRY_TICK_INTERVAL); - expiry_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - expiry_ticker.tick().await; - - loop { - tokio::select! { - event = self.handle.next() => match event { - Some(BitswapEvent::Request { peer, cids }) => - self.handle_inbound_request(peer, cids).await, - Some(BitswapEvent::Response { peer, responses }) => - self.pending.handle_response(peer, responses), - None => { - log::debug!(target: LOG_TARGET, "bitswap handle stream ended"); - return; - }, - }, - cmd = self.cmd_rx.recv() => match cmd { - Some(BitswapOutboundCmd { peer, wants, response_tx }) => - self.handle_outbound_cmd(peer, wants, response_tx).await, - None => { - log::debug!(target: LOG_TARGET, "bitswap cmd channel closed"); - return; - }, - }, - _ = expiry_ticker.tick() => { - self.pending.expire(REQUEST_TIMEOUT, Instant::now()); - }, - } - } - } - - /// Handle an inbound bitswap WANT request from `peer`. - async fn handle_inbound_request(&mut self, peer: litep2p::PeerId, cids: Vec<(Cid, WantType)>) { - let started = Instant::now(); - let want_count = cids.len(); - if inbound_wantlist_exceeds_limit(want_count) { - self.metrics.record_error(errors::TOO_MANY_ENTRIES); - log::trace!(target: LOG_TARGET, "bitswap: ignored inbound request with {want_count} entries"); - return; - } - - log::debug!(target: LOG_TARGET, "bitswap: handle inbound request from {peer:?} for {cids:?}"); - - let metrics = &self.metrics; - let response: Vec = cids - .into_iter() - .filter(|(cid, _)| { - let supported = is_cid_supported(cid); - if !supported { - metrics.record_entry(outcomes::UNSUPPORTED_CID); - } - supported - }) - .map(|(cid, want_type)| { - let hash = H256::from_slice(&cid.hash().digest()[0..32]); - let transaction = match self.client.indexed_transaction(hash) { - Ok(ex) => ex, - Err(error) => { - metrics.record_error(errors::CLIENT); - log::error!(target: LOG_TARGET, "error retrieving transaction {hash}: {error}"); - None - }, - }; - let response = match transaction { - Some(transaction) => match want_type { - WantType::Block => ResponseType::Block { cid, block: transaction }, - _ => ResponseType::Presence { cid, presence: BlockPresenceType::Have }, - }, - None => ResponseType::Presence { cid, presence: BlockPresenceType::DontHave }, - }; - metrics.record_response(&response); - response - }) - .collect(); - - // note: we assume the duplicate encode (litep2p re-serialises internally inside - // `send_response`) is cheap. - let response_bytes = encode_responses_as_bitswap_message(&response).len(); - self.metrics.add_response_bytes(response_bytes as u64); - - self.handle.send_response(peer, response).await; - self.metrics.record_duration(started.elapsed()); - } - - /// Handle an outbound bitswap command from the network service. - async fn handle_outbound_cmd( - &mut self, - peer: litep2p::PeerId, - wants: Vec<(Cid, WantType)>, - response_tx: ResponseSender, - ) { - log::debug!( - target: LOG_TARGET, - "bitswap: outbound WANT for {} CIDs to {peer:?}", - wants.len(), - ); - let cids: Vec<_> = wants.iter().map(|(cid, _)| *cid).collect(); - self.pending.insert(peer, PendingBatch::new(cids, response_tx, Instant::now())); - self.handle.send_request(peer, wants).await; - } -} - -fn inbound_wantlist_exceeds_limit(len: usize) -> bool { - len > MAX_WANTED_BLOCKS -} - -/// Collapse a response list into at most one entry per CID, preferring `Block` -/// over `Presence` when both arrive for the same CID. -fn select_best_response_per_cid(responses: Vec) -> HashMap { - let mut best: HashMap = HashMap::new(); - for resp in responses { - let cid = match &resp { - ResponseType::Block { cid, .. } => *cid, - ResponseType::Presence { cid, .. } => *cid, - }; - match best.entry(cid) { - std::collections::hash_map::Entry::Vacant(e) => { - e.insert(resp); - }, - std::collections::hash_map::Entry::Occupied(mut e) => { - if matches!(resp, ResponseType::Block { .. }) && - matches!(*e.get(), ResponseType::Presence { .. }) - { - e.insert(resp); - } - }, - } - } - best -} - -/// Return the byte size of a response that counts toward the pending batch cap. -fn response_retained_bytes(response: &ResponseType) -> usize { - match response { - ResponseType::Block { block, .. } => block.len(), - ResponseType::Presence { .. } => 0, - } -} - -/// Encode litep2p [`ResponseType`] values into a [`BitswapProtoMessage`] byte vector. -fn encode_responses_as_bitswap_message(responses: &[ResponseType]) -> Vec { - let mut msg = BitswapProtoMessage::default(); - - for resp in responses { - match resp { - ResponseType::Block { cid, block } => { - let prefix: Prefix = cid.into(); - msg.payload - .push(MessageBlock { prefix: prefix.to_bytes(), data: block.clone() }); - }, - ResponseType::Presence { cid, presence } => { - msg.block_presences.push(BlockPresence { - cid: cid.to_bytes(), - r#type: match presence { - BlockPresenceType::Have => ProtoPresenceType::Have as i32, - BlockPresenceType::DontHave => ProtoPresenceType::DontHave as i32, - }, - }); - }, - } - } - - msg.encode_to_vec() -} - -#[cfg(test)] -mod tests { - use super::*; - use cid::multihash::Multihash as CidMultihash; - use prometheus_endpoint::Registry; - use substrate_test_runtime_client; - - fn make_peer() -> litep2p::PeerId { - litep2p::PeerId::random() - } - - fn make_cid(byte: u8) -> Cid { - let digest = [byte; 32]; - let mh = CidMultihash::<64>::wrap(0xb220, &digest).unwrap(); - Cid::new_v1(0x55, mh) - } - - #[test] - fn inbound_wantlist_limit_rejects_only_over_cap_requests() { - assert!(!inbound_wantlist_exceeds_limit(MAX_WANTED_BLOCKS)); - assert!(inbound_wantlist_exceeds_limit(MAX_WANTED_BLOCKS + 1)); - } - - #[test] - fn bitswap_service_constructs_without_registry() { - let client = Arc::new(substrate_test_runtime_client::new()); - let (_future, _config) = BitswapService::new(client, None); - } - - #[test] - fn bitswap_service_constructs_with_registry() { - let registry = Registry::new(); - let client = Arc::new(substrate_test_runtime_client::new()); - let (_future, _config) = BitswapService::new(client, Some(®istry)); - - // Sanity check: registering the same metric names a second time on the same - // registry must fail — proves the first registration actually went through. - let second = crate::litep2p::bitswap_metrics::BitswapMetrics::new(Some(®istry)); - assert!(second.is_err(), "double registration should fail"); - } - - fn pending_batch( - cids: Vec, - response_tx: ResponseSender, - inserted: Instant, - ) -> PendingBatch { - PendingBatch::new(cids, response_tx, inserted) - } - - #[test] - fn encode_responses_are_decodable() { - let block_cid = make_cid(1); - let presence_cid = make_cid(2); - let data = b"block-data-payload".to_vec(); - let responses = vec![ - ResponseType::Block { cid: block_cid, block: data.clone() }, - ResponseType::Presence { cid: presence_cid, presence: BlockPresenceType::DontHave }, - ]; - - let bytes = encode_responses_as_bitswap_message(&responses); - let msg = BitswapProtoMessage::decode(bytes.as_slice()).unwrap(); - - assert_eq!(msg.payload.len(), 1); - assert_eq!(msg.payload[0].data, data); - assert_eq!(msg.block_presences.len(), 1); - assert_eq!(msg.block_presences[0].r#type, ProtoPresenceType::DontHave as i32); - } - - #[test] - fn select_best_prefers_block_over_presence() { - let cid = make_cid(3); - let data = b"data".to_vec(); - let responses = vec![ - ResponseType::Presence { cid, presence: BlockPresenceType::Have }, - ResponseType::Block { cid, block: data.clone() }, - ]; - let best = select_best_response_per_cid(responses); - assert_eq!(best.len(), 1); - match best.into_iter().next().unwrap().1 { - ResponseType::Block { block, .. } => assert_eq!(block, data), - _ => panic!("expected Block to win"), - } - } - - #[test] - fn select_best_prefers_block_over_presence_regardless_of_order() { - let cid = make_cid(4); - let data = b"data-reversed".to_vec(); - let responses = vec![ - ResponseType::Block { cid, block: data.clone() }, - ResponseType::Presence { cid, presence: BlockPresenceType::Have }, - ]; - let best = select_best_response_per_cid(responses); - assert_eq!(best.len(), 1); - match best.into_iter().next().unwrap().1 { - ResponseType::Block { block, .. } => assert_eq!(block, data), - _ => panic!("expected Block to win"), - } - } - - #[test] - fn select_best_keeps_distinct_cids() { - let cid_a = make_cid(5); - let cid_b = make_cid(6); - let responses = vec![ - ResponseType::Block { cid: cid_a, block: b"a".to_vec() }, - ResponseType::Presence { cid: cid_b, presence: BlockPresenceType::DontHave }, - ]; - let best = select_best_response_per_cid(responses); - assert_eq!(best.len(), 2); - assert!(best.contains_key(&cid_a)); - assert!(best.contains_key(&cid_b)); - } - - #[tokio::test] - async fn pending_batch_single_request_resolves() { - let peer = make_peer(); - let cid = make_cid(7); - let data = b"resolved-data".to_vec(); - - let (tx, rx) = oneshot::channel(); - let mut pending = PendingBatches::default(); - pending.insert(peer, pending_batch(vec![cid], tx, Instant::now())); - - pending.handle_response(peer, vec![ResponseType::Block { cid, block: data.clone() }]); - - let (payload, _) = rx.await.unwrap().unwrap(); - let msg = BitswapProtoMessage::decode(payload.as_slice()).unwrap(); - assert_eq!(msg.payload.len(), 1); - assert_eq!(msg.payload[0].data, data); - assert!(pending.is_empty()); - } - - #[tokio::test] - async fn pending_batch_duplicate_requests_both_resolve() { - let peer = make_peer(); - let cid = make_cid(8); - let data = b"shared-blob".to_vec(); - - let (tx_a, rx_a) = oneshot::channel(); - let (tx_b, rx_b) = oneshot::channel(); - let mut pending = PendingBatches::default(); - pending.insert(peer, pending_batch(vec![cid], tx_a, Instant::now())); - pending.insert(peer, pending_batch(vec![cid], tx_b, Instant::now())); - - pending.handle_response(peer, vec![ResponseType::Block { cid, block: data.clone() }]); - - let a = rx_a.await.unwrap().unwrap(); - let b = rx_b.await.unwrap().unwrap(); - let msg_a = BitswapProtoMessage::decode(a.0.as_slice()).unwrap(); - let msg_b = BitswapProtoMessage::decode(b.0.as_slice()).unwrap(); - assert_eq!(msg_a.payload[0].data, data); - assert_eq!(msg_b.payload[0].data, data); - assert!(pending.is_empty()); - } - - #[tokio::test] - async fn pending_batch_multi_want_waits_for_all_cids() { - let peer = make_peer(); - let cid_a = make_cid(11); - let cid_b = make_cid(12); - let data_a = b"first".to_vec(); - let data_b = b"second".to_vec(); - - let (tx, rx) = oneshot::channel(); - let mut pending = PendingBatches::default(); - pending.insert(peer, pending_batch(vec![cid_a, cid_b], tx, Instant::now())); - - pending - .handle_response(peer, vec![ResponseType::Block { cid: cid_a, block: data_a.clone() }]); - assert_eq!(pending.len(), 1); - - pending - .handle_response(peer, vec![ResponseType::Block { cid: cid_b, block: data_b.clone() }]); - - let (payload, _) = rx.await.unwrap().unwrap(); - let msg = BitswapProtoMessage::decode(payload.as_slice()).unwrap(); - assert_eq!(msg.payload.len(), 2); - assert_eq!(msg.payload[0].data, data_a); - assert_eq!(msg.payload[1].data, data_b); - assert!(pending.is_empty()); - } - - #[tokio::test] - async fn pending_batch_fails_when_partial_responses_exceed_byte_limit() { - let peer = make_peer(); - let cid_a = make_cid(13); - let cid_b = make_cid(14); - - let (tx, rx) = oneshot::channel(); - let mut pending = PendingBatches::default(); - pending.insert(peer, pending_batch(vec![cid_a, cid_b], tx, Instant::now())); - - pending.handle_response_with_limit( - peer, - vec![ResponseType::Block { cid: cid_a, block: vec![0u8; 8] }], - 4, - ); - - let result = rx.await.unwrap(); - assert!(matches!(result, Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)))); - assert!(pending.is_empty()); - } - - #[tokio::test] - async fn pending_batch_expiry_sends_failure() { - let peer = make_peer(); - let cid = make_cid(9); - - let (tx_stale, rx_stale) = oneshot::channel(); - let (tx_fresh, rx_fresh) = oneshot::channel(); - let past = Instant::now() - Duration::from_secs(60); - let fresh_time = Instant::now(); - - let mut pending = PendingBatches::default(); - pending.insert(peer, pending_batch(vec![cid], tx_stale, past)); - pending.insert(peer, pending_batch(vec![cid], tx_fresh, fresh_time)); - - pending.expire(Duration::from_secs(30), Instant::now()); - - let stale_result = rx_stale.await.unwrap(); - assert!(matches!(stale_result, Err(RequestFailure::Network(OutboundFailure::Timeout)))); - assert_eq!(pending.len(), 1); - drop(rx_fresh); - } - - #[tokio::test] - async fn pending_batch_mismatched_peer_does_not_resolve() { - let peer_a = make_peer(); - let peer_b = make_peer(); - let cid = make_cid(10); - - let (tx, mut rx) = oneshot::channel(); - let mut pending = PendingBatches::default(); - pending.insert(peer_a, pending_batch(vec![cid], tx, Instant::now())); - - pending.handle_response(peer_b, vec![ResponseType::Block { cid, block: b"data".to_vec() }]); - - assert_eq!(pending.len(), 1); - assert!(rx.try_recv().unwrap().is_none()); - } - - #[tokio::test] - async fn pending_batch_response_from_one_peer_does_not_affect_other_peer() { - let peer_a = make_peer(); - let peer_b = make_peer(); - let cid_a = make_cid(20); - let cid_b = make_cid(21); - let data_b = b"peer-b-data".to_vec(); - - let (tx_a, mut rx_a) = oneshot::channel(); - let (tx_b, rx_b) = oneshot::channel(); - let mut pending = PendingBatches::default(); - pending.insert(peer_a, pending_batch(vec![cid_a], tx_a, Instant::now())); - pending.insert(peer_b, pending_batch(vec![cid_b], tx_b, Instant::now())); - - pending.handle_response( - peer_b, - vec![ResponseType::Block { cid: cid_b, block: data_b.clone() }], - ); - - let (payload, _) = rx_b.await.unwrap().unwrap(); - let msg = BitswapProtoMessage::decode(payload.as_slice()).unwrap(); - assert_eq!(msg.payload.len(), 1); - assert_eq!(msg.payload[0].data, data_b); - - assert!(rx_a.try_recv().unwrap().is_none()); - assert_eq!(pending.len(), 1); - assert!(pending.contains_key(&peer_a)); - } -} diff --git a/substrate/client/network/src/litep2p/bitswap_metrics.rs b/substrate/client/network/src/litep2p/bitswap_metrics.rs deleted file mode 100644 index cc5c26274cac..000000000000 --- a/substrate/client/network/src/litep2p/bitswap_metrics.rs +++ /dev/null @@ -1,219 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Prometheus metrics for the litep2p bitswap server. - -use litep2p::protocol::libp2p::bitswap::{BlockPresenceType, ResponseType}; -use prometheus_endpoint::{ - exponential_buckets, register, Counter, CounterVec, Histogram, HistogramOpts, Opts, - PrometheusError, Registry, U64, -}; -use std::time::Duration; - -/// `outcome` label values for `substrate_sub_libp2p_bitswap_entries_total`. -pub mod outcomes { - pub const BLOCK_SERVED: &str = "block_served"; - pub const HAVE: &str = "have"; - pub const DONT_HAVE: &str = "dont_have"; - pub const UNSUPPORTED_CID: &str = "unsupported_cid"; -} - -/// `reason` label values for `substrate_sub_libp2p_bitswap_request_errors_total`. -pub mod errors { - pub const TOO_MANY_ENTRIES: &str = "too_many_entries"; - pub const CLIENT: &str = "client"; -} - -struct Inner { - entries_total: CounterVec, - request_errors_total: CounterVec, - inbound_request_duration_seconds: Histogram, - response_bytes_total: Counter, -} - -impl Inner { - fn register(registry: &Registry) -> Result { - Ok(Self { - entries_total: register( - CounterVec::new( - Opts::new( - "substrate_sub_libp2p_bitswap_entries_total", - "Total number of bitswap wantlist entries processed, by outcome", - ), - &["outcome"], - )?, - registry, - )?, - request_errors_total: register( - CounterVec::new( - Opts::new( - "substrate_sub_libp2p_bitswap_request_errors_total", - "Total number of bitswap inbound requests rejected, by reason", - ), - &["reason"], - )?, - registry, - )?, - inbound_request_duration_seconds: register( - Histogram::with_opts(HistogramOpts { - common_opts: Opts::new( - "substrate_sub_libp2p_bitswap_inbound_request_duration_seconds", - "Duration of handling an inbound bitswap wantlist, in seconds", - ), - buckets: exponential_buckets(0.001, 2.0, 16) - .expect("parameters are always valid values; qed"), - })?, - registry, - )?, - response_bytes_total: register( - Counter::new( - "substrate_sub_libp2p_bitswap_response_bytes_total", - "Total bytes sent in bitswap responses to inbound wantlists", - )?, - registry, - )?, - }) - } -} - -/// Helper wrapper around the bitswap server metrics. -/// -/// When constructed without a `Registry`, all recording methods become no-ops. -pub struct BitswapMetrics { - inner: Option, -} - -impl BitswapMetrics { - /// Register the metrics with the given Prometheus registry, if any. - pub fn new(registry: Option<&Registry>) -> Result { - Ok(Self { inner: registry.map(Inner::register).transpose()? }) - } - - /// Record one wantlist entry processed with the given outcome. - pub fn record_entry(&self, outcome: &str) { - if let Some(inner) = &self.inner { - inner.entries_total.with_label_values(&[outcome]).inc(); - } - } - - /// Record one outbound response variant under the matching outcome label. - pub fn record_response(&self, response: &ResponseType) { - let outcome = match response { - ResponseType::Block { .. } => outcomes::BLOCK_SERVED, - ResponseType::Presence { presence: BlockPresenceType::Have, .. } => outcomes::HAVE, - ResponseType::Presence { presence: BlockPresenceType::DontHave, .. } => { - outcomes::DONT_HAVE - }, - }; - self.record_entry(outcome); - } - - /// Record one request-level error with the given reason. - pub fn record_error(&self, reason: &str) { - if let Some(inner) = &self.inner { - inner.request_errors_total.with_label_values(&[reason]).inc(); - } - } - - /// Observe the duration of an inbound wantlist handling. - pub fn record_duration(&self, duration: Duration) { - if let Some(inner) = &self.inner { - inner.inbound_request_duration_seconds.observe(duration.as_secs_f64()); - } - } - - /// Add to the running total of bitswap response bytes sent. - pub fn add_response_bytes(&self, bytes: u64) { - if let Some(inner) = &self.inner { - inner.response_bytes_total.inc_by(bytes); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use cid::{multihash::Multihash as CidMultihash, Cid}; - - fn make_cid() -> Cid { - let mh = CidMultihash::<64>::wrap(0xb220, &[0u8; 32]).unwrap(); - Cid::new_v1(0x55, mh) - } - - #[test] - fn disabled_metrics_are_no_ops() { - let metrics = BitswapMetrics::new(None).unwrap(); - metrics.record_entry(outcomes::BLOCK_SERVED); - metrics.record_error(errors::CLIENT); - metrics.record_duration(Duration::from_millis(1)); - metrics.add_response_bytes(42); - } - - #[test] - fn record_response_maps_variants_to_outcomes() { - let registry = Registry::new(); - let metrics = BitswapMetrics::new(Some(®istry)).unwrap(); - let cid = make_cid(); - - metrics.record_response(&ResponseType::Block { cid, block: vec![1, 2, 3] }); - metrics.record_response(&ResponseType::Block { cid, block: vec![4] }); - metrics.record_response(&ResponseType::Presence { cid, presence: BlockPresenceType::Have }); - metrics.record_response(&ResponseType::Presence { - cid, - presence: BlockPresenceType::DontHave, - }); - metrics.record_response(&ResponseType::Presence { - cid, - presence: BlockPresenceType::DontHave, - }); - metrics.record_response(&ResponseType::Presence { - cid, - presence: BlockPresenceType::DontHave, - }); - - let inner = metrics.inner.as_ref().expect("inner should be present"); - assert_eq!(inner.entries_total.with_label_values(&[outcomes::BLOCK_SERVED]).get(), 2); - assert_eq!(inner.entries_total.with_label_values(&[outcomes::HAVE]).get(), 1); - assert_eq!(inner.entries_total.with_label_values(&[outcomes::DONT_HAVE]).get(), 3); - assert_eq!(inner.entries_total.with_label_values(&[outcomes::UNSUPPORTED_CID]).get(), 0); - } - - #[test] - fn enabled_metrics_register_and_increment() { - let registry = Registry::new(); - let metrics = BitswapMetrics::new(Some(®istry)).unwrap(); - - metrics.record_entry(outcomes::BLOCK_SERVED); - metrics.record_entry(outcomes::BLOCK_SERVED); - metrics.record_entry(outcomes::HAVE); - metrics.record_error(errors::TOO_MANY_ENTRIES); - metrics.record_duration(Duration::from_millis(5)); - metrics.add_response_bytes(1024); - - let inner = metrics.inner.as_ref().expect("inner should be present when registry given"); - assert_eq!(inner.entries_total.with_label_values(&[outcomes::BLOCK_SERVED]).get(), 2); - assert_eq!(inner.entries_total.with_label_values(&[outcomes::HAVE]).get(), 1); - assert_eq!(inner.entries_total.with_label_values(&[outcomes::DONT_HAVE]).get(), 0); - assert_eq!( - inner.request_errors_total.with_label_values(&[errors::TOO_MANY_ENTRIES]).get(), - 1 - ); - assert_eq!(inner.response_bytes_total.get(), 1024); - assert_eq!(inner.inbound_request_duration_seconds.get_sample_count(), 1); - } -} diff --git a/substrate/client/network/src/litep2p/mod.rs b/substrate/client/network/src/litep2p/mod.rs index 4923e83c6148..3c666a291ac3 100644 --- a/substrate/client/network/src/litep2p/mod.rs +++ b/substrate/client/network/src/litep2p/mod.rs @@ -26,7 +26,6 @@ use crate::{ error::Error, event::{DhtEvent, Event}, litep2p::{ - bitswap::BitswapService, discovery::{Discovery, DiscoveryEvent}, ipfs_dht::IpfsDht, peerstore::Peerstore, @@ -74,7 +73,6 @@ use litep2p::{ use prometheus_endpoint::Registry; use sc_network_types::kad::{Key as RecordKey, PeerRecord, Record as P2PRecord}; -use sc_client_api::BlockBackend; use sc_network_common::{role::Roles, ExHashT}; use sc_network_types::PeerId; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; @@ -94,8 +92,6 @@ use std::{ time::{Duration, Instant}, }; -mod bitswap; -mod bitswap_metrics; mod discovery; mod ipfs_dht; mod peerstore; @@ -386,11 +382,12 @@ impl Litep2pNetworkBackend { #[async_trait::async_trait] impl NetworkBackend for Litep2pNetworkBackend { + const SUPPORTS_IPFS: bool = true; + type NotificationProtocolConfig = NotificationProtocolConfig; type RequestResponseProtocolConfig = RequestResponseConfig; type NetworkService = Arc; type PeerStore = Peerstore; - type BitswapConfig = bitswap::BitswapConfig; fn new(mut params: Params) -> Result where @@ -568,15 +565,11 @@ impl NetworkBackend for Litep2pNetworkBac Arc::clone(&peer_store_handle), ); - let bitswap_cmd_tx = params.ipfs_config.as_ref().map(|c| c.bitswap_config.cmd_tx.clone()); - - // enable Bitswap & IPFS DHT - if let Some(config) = params.ipfs_config { - config_builder = - config_builder.with_libp2p_bitswap(config.bitswap_config.litep2p_config); + if let Some(ipfs) = params.ipfs_config { + config_builder = config_builder.with_libp2p_bitswap(ipfs.litep2p_bitswap_config); - if !config.bootnodes.is_empty() { - let (ipfs_dht, kad_config) = IpfsDht::new(config.bootnodes, config.block_provider); + if !ipfs.bootnodes.is_empty() { + let (ipfs_dht, kad_config) = IpfsDht::new(ipfs.bootnodes, ipfs.block_provider); config_builder = config_builder.with_libp2p_kademlia(kad_config); executor.run(Box::pin(ipfs_dht.run())); } else { @@ -635,7 +628,6 @@ impl NetworkBackend for Litep2pNetworkBac request_response_senders, Arc::clone(&listen_addresses), public_addresses, - bitswap_cmd_tx, )); // register rest of the metrics now that `Litep2p` has been created @@ -678,14 +670,6 @@ impl NetworkBackend for Litep2pNetworkBac NotificationMetrics::new(registry) } - /// Create Bitswap server. - fn bitswap_server( - client: Arc + Send + Sync>, - metrics_registry: Option, - ) -> (Pin + Send>>, Self::BitswapConfig) { - BitswapService::new(client, metrics_registry.as_ref()) - } - /// Create notification protocol configuration for `protocol`. fn notification_config( protocol_name: ProtocolName, diff --git a/substrate/client/network/src/litep2p/service.rs b/substrate/client/network/src/litep2p/service.rs index cb5ecfb84e4b..8638ffed967c 100644 --- a/substrate/client/network/src/litep2p/service.rs +++ b/substrate/client/network/src/litep2p/service.rs @@ -19,7 +19,6 @@ //! `NetworkService` implementation for `litep2p`. use crate::{ - bitswap::schema::bitswap::message::wantlist::WantType as ProtoBitswapWantType, config::MultiaddrWithPeerId, litep2p::shim::{ notification::{config::ProtocolControlHandle, peerset::PeersetCommand}, @@ -38,7 +37,6 @@ use futures::{channel::oneshot, stream::BoxStream}; use libp2p::identity::SigningError; use litep2p::{ addresses::PublicAddresses, crypto::ed25519::Keypair, - protocol::libp2p::bitswap::WantType as LitepBitswapWantType, types::multiaddr::Multiaddr as LiteP2pMultiaddr, }; use parking_lot::RwLock; @@ -217,9 +215,6 @@ pub struct Litep2pNetworkService { /// External addresses. external_addresses: PublicAddresses, - - /// Sender for outbound bitswap requests; `None` if IPFS/bitswap is not configured. - bitswap_cmd_tx: Option>, } impl Litep2pNetworkService { @@ -234,7 +229,6 @@ impl Litep2pNetworkService { request_response_protocols: HashMap>, listen_addresses: Arc>>, external_addresses: PublicAddresses, - bitswap_cmd_tx: Option>, ) -> Self { Self { local_peer_id, @@ -246,80 +240,6 @@ impl Litep2pNetworkService { request_response_protocols, listen_addresses, external_addresses, - bitswap_cmd_tx, - } - } - - /// Route an outbound request whose protocol name matches the bitswap protocol. - /// - /// Native bitswap is not registered as a generic request-response protocol, so this bridges - /// the `NetworkRequest` payload to the litep2p bitswap service. - fn route_bitswap_request( - &self, - peer: PeerId, - request_payload: Vec, - sender: oneshot::Sender, ProtocolName), RequestFailure>>, - ) { - use prost::Message as _; - - let Some(cmd_tx) = self.bitswap_cmd_tx.as_ref() else { - log::warn!( - target: LOG_TARGET, - "bitswap: received outbound request but BitswapService is not configured" - ); - let _ = sender.send(Err(RequestFailure::UnknownProtocol)); - return; - }; - - let msg = match crate::bitswap::BitswapProtoMessage::decode(request_payload.as_slice()) { - Ok(m) => m, - Err(e) => { - log::warn!(target: LOG_TARGET, "bitswap: failed to decode WANT payload: {e}"); - let _ = sender.send(Err(RequestFailure::InvalidRequest)); - return; - }, - }; - - let wantlist = match msg.wantlist { - Some(w) if !w.entries.is_empty() => w, - _ => { - log::warn!(target: LOG_TARGET, "bitswap: WANT message has no wantlist entries"); - let _ = sender.send(Err(RequestFailure::InvalidRequest)); - return; - }, - }; - - let mut cids = Vec::with_capacity(wantlist.entries.len()); - for entry in wantlist.entries { - let cid = match crate::bitswap::Cid::read_bytes(entry.block.as_slice()) { - Ok(c) => c, - Err(e) => { - log::warn!(target: LOG_TARGET, "bitswap: invalid CID in WANT entry: {e}"); - let _ = sender.send(Err(RequestFailure::InvalidRequest)); - return; - }, - }; - let want_type = if entry.want_type == ProtoBitswapWantType::Have as i32 { - LitepBitswapWantType::Have - } else { - LitepBitswapWantType::Block - }; - cids.push((cid, want_type)); - } - - let cmd = super::bitswap::BitswapOutboundCmd { - peer: peer.into(), - wants: cids, - response_tx: sender, - }; - - if let Err(e) = cmd_tx.try_send(cmd) { - log::warn!( - target: LOG_TARGET, - "bitswap cmd channel full or closed; dropping request for {peer:?}: {e}", - ); - let cmd = e.into_inner(); - let _ = cmd.response_tx.send(Err(RequestFailure::UnknownProtocol)); } } } @@ -645,11 +565,6 @@ impl NetworkRequest for Litep2pNetworkService { sender: oneshot::Sender, ProtocolName), RequestFailure>>, connect: IfDisconnected, ) { - if protocol.as_ref() == crate::bitswap::PROTOCOL_NAME { - self.route_bitswap_request(peer, request, sender); - return; - } - match self.request_response_protocols.get(&protocol) { Some(tx) => { let _ = tx.unbounded_send(OutboundRequest::new( diff --git a/substrate/client/network/src/schema/bitswap.v1.2.0.proto b/substrate/client/network/src/schema/bitswap.v1.2.0.proto deleted file mode 100644 index a4138b516d63..000000000000 --- a/substrate/client/network/src/schema/bitswap.v1.2.0.proto +++ /dev/null @@ -1,43 +0,0 @@ -syntax = "proto3"; - -package bitswap.message; - -message Message { - message Wantlist { - enum WantType { - Block = 0; - Have = 1; - } - - message Entry { - bytes block = 1; // the block cid (cidV0 in bitswap 1.0.0, cidV1 in bitswap 1.1.0) - int32 priority = 2; // the priority (normalized). default to 1 - bool cancel = 3; // whether this revokes an entry - WantType wantType = 4; // Note: defaults to enum 0, ie Block - bool sendDontHave = 5; // Note: defaults to false - } - - repeated Entry entries = 1; // a list of wantlist entries - bool full = 2; // whether this is the full wantlist. default to false - } - - message Block { - bytes prefix = 1; // CID prefix (cid version, multicodec and multihash prefix (type + length) - bytes data = 2; - } - - enum BlockPresenceType { - Have = 0; - DontHave = 1; - } - message BlockPresence { - bytes cid = 1; - BlockPresenceType type = 2; - } - - Wantlist wantlist = 1; - repeated bytes blocks = 2; // used to send Blocks in bitswap 1.0.0 - repeated Block payload = 3; // used to send Blocks in bitswap 1.1.0 - repeated BlockPresence blockPresences = 4; - int32 pendingBytes = 5; -} diff --git a/substrate/client/network/src/service.rs b/substrate/client/network/src/service.rs index a1ee4bfd817e..1bef52ff4c77 100644 --- a/substrate/client/network/src/service.rs +++ b/substrate/client/network/src/service.rs @@ -29,7 +29,6 @@ use crate::{ behaviour::{self, Behaviour, BehaviourOut}, - bitswap::BitswapRequestHandler, config::{ parse_addr, FullNetworkConfiguration, IncomingRequest, MultiaddrWithPeerId, NonDefaultSetConfig, NotificationHandshake, Params, SetConfig, TransportConfig, @@ -78,7 +77,6 @@ use parking_lot::Mutex; use prometheus_endpoint::Registry; use sc_network_types::kad::{Key as KademliaKey, Record}; -use sc_client_api::BlockBackend; use sc_network_common::{ role::{ObservedRole, Roles}, ExHashT, @@ -171,7 +169,6 @@ where type RequestResponseProtocolConfig = RequestResponseConfig; type NetworkService = Arc>; type PeerStore = PeerStore; - type BitswapConfig = RequestResponseConfig; fn new(params: Params) -> Result where @@ -197,15 +194,6 @@ where NotificationMetrics::new(registry) } - fn bitswap_server( - client: Arc + Send + Sync>, - _metrics_registry: Option, - ) -> (Pin + Send>>, Self::BitswapConfig) { - let (handler, protocol_config) = BitswapRequestHandler::new(client.clone()); - - (Box::pin(async move { handler.run().await }), protocol_config) - } - /// Create notification protocol configuration. fn notification_config( protocol_name: ProtocolName, diff --git a/substrate/client/network/src/service/traits.rs b/substrate/client/network/src/service/traits.rs index e0df3c012244..9b5d30818662 100644 --- a/substrate/client/network/src/service/traits.rs +++ b/substrate/client/network/src/service/traits.rs @@ -34,7 +34,6 @@ use crate::{ use futures::{channel::oneshot, Stream}; use prometheus_endpoint::Registry; -use sc_client_api::BlockBackend; use sc_network_common::{role::ObservedRole, ExHashT}; pub use sc_network_types::{ kad::{Key as KademliaKey, Record}, @@ -111,6 +110,9 @@ pub trait PeerStore { /// Networking backend. #[async_trait::async_trait] pub trait NetworkBackend: Send + 'static { + /// Whether this backend supports Bitswap ([`crate::config::Params::ipfs_config`]). + const SUPPORTS_IPFS: bool = false; + /// Type representing notification protocol-related configuration. type NotificationProtocolConfig: NotificationConfig; @@ -126,9 +128,6 @@ pub trait NetworkBackend: Send + 'static { /// Type implementing [`PeerStore`]. type PeerStore: PeerStore; - /// Bitswap config. - type BitswapConfig; - /// Create new `NetworkBackend`. fn new(params: Params) -> Result where @@ -143,12 +142,6 @@ pub trait NetworkBackend: Send + 'static { /// Register metrics that are used by the notification protocols. fn register_notification_metrics(registry: Option<&Registry>) -> NotificationMetrics; - /// Create Bitswap server. - fn bitswap_server( - client: Arc + Send + Sync>, - metrics_registry: Option, - ) -> (Pin + Send>>, Self::BitswapConfig); - /// Create notification protocol configuration and an associated `NotificationService` /// for the protocol. fn notification_config( diff --git a/substrate/client/service/Cargo.toml b/substrate/client/service/Cargo.toml index ce354be9d66c..4b07156a9dc0 100644 --- a/substrate/client/service/Cargo.toml +++ b/substrate/client/service/Cargo.toml @@ -36,6 +36,7 @@ sc-executor = { workspace = true, default-features = true } sc-informant = { workspace = true, default-features = true } sc-keystore = { workspace = true, default-features = true } sc-network = { workspace = true, default-features = true } +sc-network-bitswap = { workspace = true, default-features = true } sc-network-common = { workspace = true, default-features = true } sc-network-light = { workspace = true, default-features = true } sc-network-sync = { workspace = true, default-features = true } diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index 265091801be3..0f36fb88dd26 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -1018,6 +1018,7 @@ pub fn build_network( TracingUnboundedSender>, sc_network_transactions::TransactionsHandlerController<::Hash>, Arc>, + Option, ), Error, > @@ -1174,8 +1175,9 @@ where pub blocks_pruning: BlocksPruning, } -/// Build the network service, the network status sinks and an RPC sender, this is a lower-level -/// version of [`build_network`] for those needing more control. +/// Builds the lower-level network service. +/// +/// The final tuple element contains the Bitswap handle when IPFS is enabled. pub fn build_network_advanced( params: BuildNetworkAdvancedParams, ) -> Result< @@ -1184,6 +1186,7 @@ pub fn build_network_advanced( TracingUnboundedSender>, sc_network_transactions::TransactionsHandlerController<::Hash>, Arc>, + Option, ), Error, > @@ -1222,6 +1225,7 @@ where } = params; let genesis_hash = client.info().genesis_hash; + let sync_service = Arc::new(sync_service); let light_client_request_protocol_config = { // Allow both outgoing and incoming requests. @@ -1234,23 +1238,36 @@ where // install request handlers to `FullNetworkConfiguration` net_config.add_request_response_protocol(light_client_request_protocol_config); - // Initialize IPFS server. - let ipfs_config = net_config.network_config.ipfs_server.then(|| { - let (handler, bitswap_config) = - Net::bitswap_server(client.clone(), metrics_registry.cloned()); - spawn_handle.spawn("bitswap-request-handler", Some("networking"), handler); + let (ipfs_config, bitswap) = if net_config.network_config.ipfs_server { + if !Net::SUPPORTS_IPFS { + return Err(Error::Other( + "the selected network backend does not support Bitswap; \ + set --network-backend litep2p or disable --ipfs-server" + .into(), + )); + } let ipfs_num_blocks = match blocks_pruning { BlocksPruning::KeepAll | BlocksPruning::KeepFinalized => IPFS_MAX_BLOCKS, BlocksPruning::Some(num) => std::cmp::min(num, IPFS_MAX_BLOCKS), }; - IpfsConfig { - bitswap_config, - block_provider: Box::new(IpfsIndexedTransactions::new(client.clone(), ipfs_num_blocks)), - bootnodes: net_config.network_config.ipfs_bootnodes.clone(), - } - }); + let (ipfs_config, litep2p_bitswap_handle) = IpfsConfig::new( + Box::new(IpfsIndexedTransactions::new(client.clone(), ipfs_num_blocks)), + net_config.network_config.ipfs_bootnodes.clone(), + ); + + let (handler, handle) = sc_network_bitswap::start::( + client.clone(), + &*sync_service, + litep2p_bitswap_handle, + metrics_registry, + ); + + (Some(ipfs_config), Some((handler, handle))) + } else { + (None, None) + }; // Create transactions protocol and add it to the list of supported protocols of let (transactions_handler_proto, transactions_config) = @@ -1267,8 +1284,6 @@ where let peer_store = net_config.take_peer_store(); spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - let sync_service = Arc::new(sync_service); - let network_params = sc_network::config::Params::::Hash, Net> { role, executor: { @@ -1291,6 +1306,13 @@ where let network_mut = Net::new(network_params)?; let network = network_mut.network_service().clone(); + // Essential: on storage chains block import depends on the bitswap actor, so its + // death must shut the node down instead of stalling sync silently. + let bitswap_handle = bitswap.map(|(handler, handle)| { + spawn_essential_handle.spawn("bitswap-service", Some("networking"), handler); + handle + }); + let (tx_handler, tx_handler_controller) = transactions_handler_proto.build( network.clone(), sync_service.clone(), @@ -1347,7 +1369,7 @@ where // the service will shut down. spawn_essential_handle.spawn_blocking("network-worker", Some("networking"), future); - Ok((network, system_rpc_tx, tx_handler_controller, sync_service.clone())) + Ok((network, system_rpc_tx, tx_handler_controller, sync_service.clone(), bitswap_handle)) } /// Configuration for [`build_default_syncing_engine`]. diff --git a/substrate/client/storage-chain-sync/Cargo.toml b/substrate/client/storage-chain-sync/Cargo.toml index 291186d4082a..b3b23df88106 100644 --- a/substrate/client/storage-chain-sync/Cargo.toml +++ b/substrate/client/storage-chain-sync/Cargo.toml @@ -15,15 +15,11 @@ workspace = true async-trait = { workspace = true } cid = { workspace = true } codec = { workspace = true, default-features = true } -futures = { workspace = true } -futures-timer = { workspace = true } log = { workspace = true, default-features = true } -rand = { workspace = true, default-features = true } sc-client-api = { workspace = true, default-features = true } sc-client-db = { workspace = true, default-features = true } sc-consensus = { workspace = true, default-features = true } -sc-network = { workspace = true, default-features = true } -sc-network-sync = { workspace = true, default-features = true } +sc-network-bitswap = { workspace = true, default-features = true } sp-api = { workspace = true, default-features = true } sp-blockchain = { workspace = true, default-features = true } sp-consensus = { workspace = true, default-features = true } @@ -34,14 +30,14 @@ sp-state-machine = { workspace = true, default-features = true } sp-transaction-storage-proof = { workspace = true, default-features = true } sp-trie = { workspace = true, default-features = true } thiserror = { workspace = true } +tokio = { workspace = true, default-features = true, features = ["sync", "time"] } [dev-dependencies] -prost = { workspace = true } rstest = { workspace = true } sp-crypto-hashing = { workspace = true, default-features = true } sp-tracing = { workspace = true, default-features = true } sp-version = { workspace = true, default-features = true } -tokio = { workspace = true, default-features = true, features = ["macros", "rt"] } +tokio = { workspace = true, default-features = true, features = ["macros", "rt", "sync"] } tracing = { workspace = true, default-features = true } tracing-log = { workspace = true, default-features = true } tracing-subscriber = { workspace = true, default-features = true } diff --git a/substrate/client/storage-chain-sync/src/fetcher.rs b/substrate/client/storage-chain-sync/src/fetcher.rs index fc4f06cf8c29..b253d34d5bf0 100644 --- a/substrate/client/storage-chain-sync/src/fetcher.rs +++ b/substrate/client/storage-chain-sync/src/fetcher.rs @@ -16,89 +16,62 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Bitswap-based fetcher for indexed-transaction blobs. +//! Indexed-transaction fetching over Bitswap. use crate::RenewWant; -use async_trait::async_trait; -use cid::{multihash::Multihash, Cid}; -use futures::channel::oneshot; -use rand::seq::SliceRandom; -use sc_network::{ - bitswap::{request_bitswap_blocks, FetchOutcome, MAX_WANTED_BLOCKS}, - NetworkRequest, PeerId, -}; -use sc_network_sync::SyncingService; -use sp_runtime::traits::Block as BlockT; +use cid::Cid; +use sc_network_bitswap::{BitswapError, BitswapHandle, FetchItem}; use sp_transaction_storage_proof::ContentHash; use std::{ collections::HashMap, sync::{Arc, OnceLock}, time::Duration, }; +use tokio::sync::mpsc; const LOG_TARGET: &str = "storage-chain-fetcher"; -const BITSWAP_PER_PEER_TIMEOUT: Duration = Duration::from_secs(5); -const MAX_PEERS_PER_IMPORT: usize = 8; -/// Source of currently-connected sync peer IDs. -#[async_trait] -pub trait BitswapPeerSource: Send + Sync { - async fn current_peers(&self) -> Result, oneshot::Canceled>; +const FETCH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Source of Bitswap response streams. +pub trait BitswapRequest: Send + Sync { + /// Requests blocks by CID. Successful stream items contain only requested CIDs. + fn request_stream(&self, cids: Vec) -> Result, BitswapError>; } -#[async_trait] -impl BitswapPeerSource for SyncingService { - async fn current_peers(&self) -> Result, oneshot::Canceled> { - Ok(self - .peers_info() - .await? - .into_iter() - .filter_map(|(peer, info)| info.roles.is_full().then_some(peer)) - .collect()) +impl BitswapRequest for BitswapHandle { + fn request_stream(&self, cids: Vec) -> Result, BitswapError> { + BitswapHandle::request_stream(self, cids) } } -/// Late-bound network request handle, populated once the network is built. -pub type NetworkHandle = Arc>>; -/// Late-bound peer-source handle, populated once the network is built. -pub type SyncingHandle = Arc>>; +/// Late-bound Bitswap request source. +pub type BitswapHandleSlot = Arc>>; -/// Infrastructure-level fetch failure. +/// Indexed-transaction fetch errors. #[derive(Debug, thiserror::Error)] pub enum FetchError { - #[error("network handle not yet set; storage-chain blocks cannot be fetched before build_network completes")] - NetworkHandleUnset, - #[error("sync handle not yet set; storage-chain blocks cannot be fetched before build_network completes")] - SyncingHandleUnset, - #[error("failed to construct multihash for CID: {0}")] - Multihash(String), -} - -/// Fetcher that resolves indexed-transaction hashes via bitswap. -pub struct IndexedTransactionFetcher { - network: NetworkHandle, - peer_source: SyncingHandle, - _phantom: std::marker::PhantomData, + /// Bitswap is unavailable. + #[error("bitswap unavailable: disabled on this node, or network not yet initialized")] + BitswapUnavailable, + /// The Bitswap request failed. + #[error("bitswap service error: {0}")] + Bitswap(#[from] BitswapError), } -impl Clone for IndexedTransactionFetcher { - fn clone(&self) -> Self { - Self { - network: self.network.clone(), - peer_source: self.peer_source.clone(), - _phantom: std::marker::PhantomData, - } - } +/// Fetches indexed transactions through Bitswap. +#[derive(Clone)] +pub struct IndexedTransactionFetcher { + bitswap: BitswapHandleSlot, } -impl IndexedTransactionFetcher { - /// Build a new fetcher backed by the given late-bound handles. - pub fn new(network: NetworkHandle, peer_source: SyncingHandle) -> Self { - Self { network, peer_source, _phantom: std::marker::PhantomData } +impl IndexedTransactionFetcher { + /// Creates a fetcher. + pub fn new(bitswap: BitswapHandleSlot) -> Self { + Self { bitswap } } - /// Resolve a batch of indexed-transaction renew wants via bitswap, rotating across up to - /// `MAX_PEERS_PER_IMPORT` peers. Returns only successfully fetched entries. + /// Fetches a batch of indexed transactions. pub(crate) async fn fetch_many( &self, wants: &[RenewWant], @@ -106,99 +79,64 @@ impl IndexedTransactionFetcher { if wants.is_empty() { return Ok(HashMap::new()); } - let network = self.network.get().ok_or(FetchError::NetworkHandleUnset)?; - let peer_source = self.peer_source.get().ok_or(FetchError::SyncingHandleUnset)?; + let handle = self.bitswap.get().ok_or(FetchError::BitswapUnavailable)?; - let Ok(mut peers) = peer_source.current_peers().await else { - log::warn!(target: LOG_TARGET, "current_peers() channel cancelled"); - return Ok(HashMap::new()); + let cids: Vec = wants.iter().copied().map(Cid::from).collect(); + + let mut rx = match handle.request_stream(cids) { + Ok(rx) => rx, + Err(BitswapError::Overloaded) => { + log::debug!(target: LOG_TARGET, "bitswap service overloaded, deferring fetch"); + return Ok(HashMap::new()); + }, + Err(other) => return Err(FetchError::Bitswap(other)), }; - if peers.is_empty() { - log::debug!( - target: LOG_TARGET, - "no connected sync peers, cannot fetch via bitswap yet", - ); - return Ok(HashMap::new()); - } - // Shuffle peers to not end up with always the same peers. - peers.shuffle(&mut rand::thread_rng()); - - // Build per-want CIDs once; reuse across peers and chunks. - let cids: Vec<(ContentHash, Cid)> = wants - .iter() - .map(|w| { - let mh = Multihash::<64>::wrap(w.hashing.multihash_code(), &w.hash) - .map_err(|e| FetchError::Multihash(e.to_string()))?; - Ok::<_, FetchError>((w.hash, Cid::new_v1(w.cid_codec, mh))) - }) - .collect::>()?; - let mut remaining = cids; - let mut acquired: HashMap> = HashMap::new(); - - for peer in peers.into_iter().take(MAX_PEERS_PER_IMPORT) { - if remaining.is_empty() { - break; + + let deadline = tokio::time::Instant::now() + FETCH_TIMEOUT; + let mut acquired: HashMap> = HashMap::with_capacity(wants.len()); + loop { + match tokio::time::timeout_at(deadline, rx.recv()).await { + Ok(Some(Ok((cid, bytes)))) => { + let hash: ContentHash = cid + .hash() + .digest() + .try_into() + .map_err(|_| BitswapError::InvalidCid { cid })?; + log::debug!( + target: LOG_TARGET, + "bitswap fetched {} bytes for {hash:?}", + bytes.len(), + ); + acquired.insert(hash, bytes); + }, + Ok(Some(Err(BitswapError::ServiceClosed))) => { + log::warn!( + target: LOG_TARGET, + "bitswap service closed mid-stream; returning partial result", + ); + return Ok(acquired); + }, + Ok(Some(Err(BitswapError::Overloaded))) => { + log::debug!( + target: LOG_TARGET, + "bitswap service overloaded; returning partial result", + ); + return Ok(acquired); + }, + Ok(Some(Err(other))) => return Err(FetchError::Bitswap(other)), + Ok(None) => break, + Err(_) => { + log::debug!( + target: LOG_TARGET, + "bitswap fetch timed out with {}/{} entries resolved", + acquired.len(), + wants.len(), + ); + break; + }, } - let from_peer = try_fetch_from_peer(network.as_ref(), peer, &remaining).await; - acquired.extend(from_peer); - remaining.retain(|(hash, _)| !acquired.contains_key(hash)); } Ok(acquired) } } - -/// Try every chunk of `wants` against a single peer in sequence. Returns whatever blocks the -/// peer actually served. A timeout or per-chunk error aborts the remaining chunks for this peer -/// and lets the caller move on to the next one. -async fn try_fetch_from_peer( - network: &N, - peer: PeerId, - wants: &[(ContentHash, Cid)], -) -> HashMap> { - let mut acquired: HashMap> = HashMap::new(); - for chunk in wants.chunks(MAX_WANTED_BLOCKS) { - let cids: Vec = chunk.iter().map(|(_, cid)| *cid).collect(); - match with_timeout(request_bitswap_blocks(network, peer, &cids), BITSWAP_PER_PEER_TIMEOUT) - .await - { - None => { - log::debug!( - target: LOG_TARGET, - "request_bitswap_blocks to {peer:?}: timeout (chunk size {})", - chunk.len(), - ); - return acquired; - }, - Some(Err(e)) => { - log::debug!(target: LOG_TARGET, "request_bitswap_blocks to {peer:?}: {e:?}"); - return acquired; - }, - Some(Ok(per_cid)) => { - for (hash, cid) in chunk { - if let Some(FetchOutcome::Block(data)) = per_cid.get(cid) { - log::debug!( - target: LOG_TARGET, - "fetched {} bytes for {:?} from {peer:?}", - data.len(), - hash, - ); - acquired.insert(*hash, data.clone()); - } - } - }, - } - } - acquired -} - -async fn with_timeout(fut: F, timeout: Duration) -> Option -where - F: std::future::Future, -{ - use futures::FutureExt; - futures::select! { - v = fut.fuse() => Some(v), - _ = futures_timer::Delay::new(timeout).fuse() => None, - } -} diff --git a/substrate/client/storage-chain-sync/src/lib.rs b/substrate/client/storage-chain-sync/src/lib.rs index 428972d2183d..3e426e4b8948 100644 --- a/substrate/client/storage-chain-sync/src/lib.rs +++ b/substrate/client/storage-chain-sync/src/lib.rs @@ -34,8 +34,9 @@ mod fetcher; pub(crate) use fetcher::FetchError; -pub use fetcher::{BitswapPeerSource, IndexedTransactionFetcher, NetworkHandle, SyncingHandle}; +pub use fetcher::{BitswapHandleSlot, BitswapRequest, IndexedTransactionFetcher}; +use cid::{multihash::Multihash, Cid}; use codec::Encode; use sc_client_api::{BlockBackend, PrefetchedIndexedTransactions}; use sc_consensus::{ @@ -113,12 +114,20 @@ pub(crate) struct RenewWant { pub cid_codec: u64, } +impl From for Cid { + fn from(want: RenewWant) -> Self { + let multihash = Multihash::<64>::wrap(want.hashing.multihash_code(), &want.hash) + .expect("a 32-byte content hash always fits into a 64-byte multihash"); + Cid::new_v1(want.cid_codec, multihash) + } +} + /// Block-import wrapper that bitswap-fetches missing TRANSACTION-column entries /// for tip-sync blocks before delegating to the inner block import. pub struct StorageChainBlockImport { inner: Inner, client: Arc, - fetcher: IndexedTransactionFetcher, + fetcher: IndexedTransactionFetcher, _phantom: PhantomData, } @@ -134,11 +143,7 @@ impl Clone for StorageChainBlockImport StorageChainBlockImport { - pub fn new( - inner: Inner, - client: Arc, - fetcher: IndexedTransactionFetcher, - ) -> Self { + pub fn new(inner: Inner, client: Arc, fetcher: IndexedTransactionFetcher) -> Self { Self { inner, client, fetcher, _phantom: PhantomData } } } @@ -577,7 +582,7 @@ fn classify_body( mod tests { use super::*; use codec::Encode; - use sc_network::bitswap::RAW_CODEC; + use sc_network_bitswap::RAW_CODEC; use sp_runtime::{generic, traits::BlakeTwo256, OpaqueExtrinsic}; use std::collections::HashSet; @@ -603,6 +608,18 @@ mod tests { OpaqueExtrinsic::from_blob(bytes.to_vec()) } + #[test] + fn renew_want_converts_to_cid() { + let want = + RenewWant { hash: [0xAB; 32], hashing: HashingAlgorithm::Keccak256, cid_codec: 0x70 }; + + let cid: Cid = want.into(); + + assert_eq!(cid.codec(), 0x70); + assert_eq!(cid.hash().code(), HashingAlgorithm::Keccak256.multihash_code()); + assert_eq!(cid.hash().digest(), &[0xAB; 32]); + } + fn body_info( ext: &OpaqueExtrinsic, extrinsic_index: u32, diff --git a/substrate/client/storage-chain-sync/tests/it.rs b/substrate/client/storage-chain-sync/tests/it.rs index 565f78b4dd66..b838039a288b 100644 --- a/substrate/client/storage-chain-sync/tests/it.rs +++ b/substrate/client/storage-chain-sync/tests/it.rs @@ -66,7 +66,7 @@ fn info( content_hash, size, hashing, - cid_codec: sc_network::bitswap::RAW_CODEC, + cid_codec: sc_network_bitswap::RAW_CODEC, extrinsic_index, } } @@ -152,7 +152,7 @@ async fn import_attached_changes_propagates_runtime_declared_codec_to_bitswap_ci "bitswap request must carry the runtime-declared codec ({DAG_PB_CODEC:#x}); observed: {observed:?}", ); assert!( - observed.iter().all(|cid| cid.codec() != sc_network::bitswap::RAW_CODEC), + observed.iter().all(|cid| cid.codec() != sc_network_bitswap::RAW_CODEC), "no request should fall back to hard-coded RAW_CODEC; observed: {observed:?}", ); @@ -523,24 +523,16 @@ async fn import_gap_sync_without_body_passes_through() { mod mock { use async_trait::async_trait; - use cid::{Cid, Version as CidVersion}; use codec::{Decode, Encode}; - use futures::channel::oneshot; use sc_storage_chain_sync::{ - BitswapPeerSource, IndexedTransactionFetcher, NetworkHandle, StorageChainBlockImport, - SyncingHandle, + BitswapHandleSlot, BitswapRequest, IndexedTransactionFetcher, StorageChainBlockImport, }; use sc_consensus::{ BlockCheckParams, BlockImport, BlockImportParams, ImportResult, ImportedAux, StateAction, StorageChanges as ConsensusStorageChanges, }; - use sc_network::{ - bitswap::{schema::bitswap as bitswap_schema, RAW_CODEC}, - request_responses::{IfDisconnected, RequestFailure}, - types::ProtocolName, - NetworkRequest, PeerId, - }; + use sc_network_bitswap::{BitswapError, Cid as BitswapCid, FetchItem, RAW_CODEC}; use sp_api::{ApiError, ConstructRuntimeApi}; use sp_consensus::{BlockOrigin, Error as ConsensusError}; use sp_core::H256; @@ -555,6 +547,7 @@ mod mock { collections::HashMap, sync::{Arc, Mutex, OnceLock}, }; + use tokio::sync::mpsc; pub(super) type TestBlock = generic::Block, OpaqueExtrinsic>; type TestHeader = generic::Header; @@ -812,13 +805,13 @@ mod mock { } #[derive(Default)] - pub(super) struct MockNetworkRequest { + pub(super) struct MockBitswap { responses: Mutex>>, call_count: Mutex, - observed_cids: Mutex>, + observed_cids: Mutex>, } - impl MockNetworkRequest { + impl MockBitswap { pub(super) fn insert(&self, hash: ContentHash, data: Vec) { self.responses.lock().unwrap().insert(hash, data); } @@ -827,80 +820,35 @@ mod mock { *self.call_count.lock().unwrap() } - pub(super) fn observed_cids(&self) -> Vec { + pub(super) fn observed_cids(&self) -> Vec { self.observed_cids.lock().unwrap().clone() } } - #[async_trait] - impl NetworkRequest for MockNetworkRequest { - async fn request( + impl BitswapRequest for MockBitswap { + fn request_stream( &self, - _target: PeerId, - _protocol: ProtocolName, - request: Vec, - _fallback_request: Option<(Vec, ProtocolName)>, - _connect: IfDisconnected, - ) -> Result<(Vec, ProtocolName), RequestFailure> { - use prost::Message as _; + cids: Vec, + ) -> Result, BitswapError> { *self.call_count.lock().unwrap() += 1; - let message = bitswap_schema::Message::decode(&*request) - .expect("MockNetworkRequest received malformed bitswap request"); + let (tx, rx) = mpsc::channel(cids.len().max(1)); let responses = self.responses.lock().unwrap(); - let mut payload = Vec::new(); - let mut block_presences = Vec::new(); - for entry in message.wantlist.unwrap_or_default().entries { - let Ok(cid) = Cid::read_bytes(entry.block.as_slice()) else { continue }; - self.observed_cids.lock().unwrap().push(cid); + let mut observed = self.observed_cids.lock().unwrap(); + for cid in cids { + observed.push(cid); let digest: Option = cid.hash().digest().try_into().ok(); - match digest.and_then(|d| responses.get(&d).cloned()) { - Some(data) => payload.push(bitswap_schema::message::Block { - prefix: prefix_mirroring_request(&cid), - data, - }), - None => block_presences.push(bitswap_schema::message::BlockPresence { - cid: entry.block, - r#type: bitswap_schema::message::BlockPresenceType::DontHave as i32, - }), + if let Some(bytes) = digest.and_then(|d| responses.get(&d).cloned()) { + tx.try_send(Ok((cid, bytes))).expect("channel sized for cids.len()"); } } - let response = - bitswap_schema::Message { payload, block_presences, ..Default::default() }; - Ok((response.encode_to_vec(), ProtocolName::from("/ipfs/bitswap/1.2.0"))) - } - - fn start_request( - &self, - _target: PeerId, - _protocol: ProtocolName, - _request: Vec, - _fallback_request: Option<(Vec, ProtocolName)>, - _tx: oneshot::Sender, ProtocolName), RequestFailure>>, - _connect: IfDisconnected, - ) { - unreachable!("the bitswap client uses async request(), never start_request()") - } - } - - fn prefix_mirroring_request(cid: &Cid) -> Vec { - sc_network::bitswap::Prefix { - version: CidVersion::V1, - codec: cid.codec(), - mh_type: cid.hash().code(), - mh_len: 32, + Ok(rx) } - .to_bytes() } - struct MockBitswapPeerSource { - peers: Vec, - } - - #[async_trait] - impl BitswapPeerSource for MockBitswapPeerSource { - async fn current_peers(&self) -> Result, oneshot::Canceled> { - Ok(self.peers.clone()) - } + fn populated_bitswap_slot(mock: Arc) -> BitswapHandleSlot { + let slot: BitswapHandleSlot = Arc::new(OnceLock::new()); + let _ = slot.set(mock); + slot } #[allow(dead_code)] @@ -1167,23 +1115,16 @@ mod mock { pub(super) wrapper: StorageChainBlockImport, pub(super) api: Arc, pub(super) captured: Arc>>>, - pub(super) network: Arc, + pub(super) network: Arc, } pub(super) fn make_harness() -> Harness { let api = Arc::new(MockApiClient::default()); - let network: Arc = Arc::new(MockNetworkRequest::default()); + let network: Arc = Arc::new(MockBitswap::default()); let inner = TestInner::recording(); let captured = inner.captured.clone(); - let network_handle: NetworkHandle = Arc::new(OnceLock::new()); - let syncing_handle: SyncingHandle = Arc::new(OnceLock::new()); - let _ = network_handle.set(network.clone() as Arc); - let _ = syncing_handle - .set(Arc::new(MockBitswapPeerSource { peers: vec![PeerId::random()] }) - as Arc); - - let fetcher = IndexedTransactionFetcher::::new(network_handle, syncing_handle); + let fetcher = IndexedTransactionFetcher::new(populated_bitswap_slot(network.clone())); let wrapper = StorageChainBlockImport::new(inner, api.clone(), fetcher); Harness { wrapper, api, captured, network } @@ -1199,19 +1140,12 @@ mod mock { pub(super) fn make_block_execution_harness(data: Vec) -> BlockExecutionHarness { let content_hash = sp_transaction_storage_proof::HashingAlgorithm::Blake2b256.hash(&data); let api = Arc::new(BlockExecutionClient::new(content_hash, data.clone())); - let network: Arc = Arc::new(MockNetworkRequest::default()); + let network: Arc = Arc::new(MockBitswap::default()); network.insert(content_hash, data); let inner = TestInner::recording(); let captured = inner.captured.clone(); - let network_handle: NetworkHandle = Arc::new(OnceLock::new()); - let syncing_handle: SyncingHandle = Arc::new(OnceLock::new()); - let _ = network_handle.set(network as Arc); - let _ = syncing_handle - .set(Arc::new(MockBitswapPeerSource { peers: vec![PeerId::random()] }) - as Arc); - - let fetcher = IndexedTransactionFetcher::::new(network_handle, syncing_handle); + let fetcher = IndexedTransactionFetcher::new(populated_bitswap_slot(network)); let wrapper = StorageChainBlockImport::new(inner, api.clone(), fetcher); BlockExecutionHarness { wrapper, api, captured, content_hash } diff --git a/substrate/frame/revive/dev-node/node/src/service.rs b/substrate/frame/revive/dev-node/node/src/service.rs index bb98d69a4727..16422cacc403 100644 --- a/substrate/frame/revive/dev-node/node/src/service.rs +++ b/substrate/frame/revive/dev-node/node/src/service.rs @@ -127,7 +127,7 @@ pub fn new_full::Ha >::new(&config.network, None); let metrics = Network::register_notification_metrics(None); - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, net_config, diff --git a/templates/minimal/node/src/service.rs b/templates/minimal/node/src/service.rs index da8f225054b2..0ad25e2e3469 100644 --- a/templates/minimal/node/src/service.rs +++ b/templates/minimal/node/src/service.rs @@ -135,7 +135,7 @@ pub fn new_full::Ha config.prometheus_config.as_ref().map(|cfg| &cfg.registry), ); - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, net_config, diff --git a/templates/parachain/node/src/service.rs b/templates/parachain/node/src/service.rs index e58ab306f064..59efae9c26e2 100644 --- a/templates/parachain/node/src/service.rs +++ b/templates/parachain/node/src/service.rs @@ -286,7 +286,7 @@ pub async fn start_parachain_node( // NOTE: because we use Aura here explicitly, we can use `CollatorSybilResistance::Resistant` // when starting the network. - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = build_network(BuildNetworkParams { parachain_config: ¶chain_config, net_config, diff --git a/templates/solochain/node/src/service.rs b/templates/solochain/node/src/service.rs index 331a1409c5fa..106677bedb49 100644 --- a/templates/solochain/node/src/service.rs +++ b/templates/solochain/node/src/service.rs @@ -171,7 +171,7 @@ pub fn new_full< Vec::default(), )); - let (network, system_rpc_tx, tx_handler_controller, sync_service) = + let (network, system_rpc_tx, tx_handler_controller, sync_service, _bitswap_handle) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, net_config, diff --git a/umbrella/Cargo.toml b/umbrella/Cargo.toml index a5b2ac1be8d1..37a0a0191c01 100644 --- a/umbrella/Cargo.toml +++ b/umbrella/Cargo.toml @@ -997,6 +997,7 @@ node = [ "sc-keystore", "sc-mixnet", "sc-network", + "sc-network-bitswap", "sc-network-common", "sc-network-gossip", "sc-network-light", @@ -2796,6 +2797,11 @@ default-features = false optional = true path = "../substrate/client/network" +[dependencies.sc-network-bitswap] +default-features = false +optional = true +path = "../substrate/client/network/bitswap" + [dependencies.sc-network-common] default-features = false optional = true diff --git a/umbrella/src/lib.rs b/umbrella/src/lib.rs index 6bb57891042f..8ef22e08e0c0 100644 --- a/umbrella/src/lib.rs +++ b/umbrella/src/lib.rs @@ -1184,6 +1184,10 @@ pub use sc_mixnet; #[cfg(feature = "sc-network")] pub use sc_network; +/// Substrate Bitswap client/server service. +#[cfg(feature = "sc-network-bitswap")] +pub use sc_network_bitswap; + /// Substrate network common. #[cfg(feature = "sc-network-common")] pub use sc_network_common;