From 631b7483615cdc9c1973833c42a21f587767f43f Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Fri, 14 Nov 2025 13:37:10 -0500
Subject: [PATCH 01/18] add httpx strategy; network simulator for benchmarks

---
 conda_libmamba_solver/index.py               |  17 +-
 conda_libmamba_solver/shards_cache.py        |  27 ++-
 conda_libmamba_solver/shards_subset.py       |  14 +-
 conda_libmamba_solver/shards_subset_http2.py | 110 +++++++++
 tests/requirements.txt                       |   4 +
 tests/test_shards_subset.py                  | 230 ++++++++++++++++++-
 6 files changed, 387 insertions(+), 15 deletions(-)
 create mode 100644 conda_libmamba_solver/shards_subset_http2.py

diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py
index fffdc42c..cb49d1ae 100644
--- a/conda_libmamba_solver/index.py
+++ b/conda_libmamba_solver/index.py
@@ -108,7 +108,7 @@
     PackageInfo,
 )
 
-from conda_libmamba_solver.shards_subset import build_repodata_subset
+from conda_libmamba_solver.shards_subset import RepodataSubset, build_repodata_subset
 
 from .mamba_utils import logger_callback
 
@@ -156,6 +156,21 @@ def _is_sharded_repodata_enabled():
     return context.plugins.use_sharded_repodata is True  # type: ignore
 
 
+def _sharded_repodata_strategy():
+    """
+    Which algorithm should we use to collect sharded repodata?
+    """
+    strategy = context.plugins.sharded_repodata_strategy.lower()  # type: ignore
+    if RepodataSubset.has_strategy(strategy):
+        return strategy
+    log.warning(
+        "Unknown sharded_repodata_strategy '%s', falling back to '%s'.",
+        strategy,
+        RepodataSubset.DEFAULT_STRATEGY,
+    )
+    return RepodataSubset.DEFAULT_STRATEGY
+
+
 _SUPPORTS_PYTHON_SITE_PACKAGES = hasattr(PackageInfo, "python_site_packages_path")
 
 
diff --git a/conda_libmamba_solver/shards_cache.py b/conda_libmamba_solver/shards_cache.py
index e682d2b0..c6b5f420 100644
--- a/conda_libmamba_solver/shards_cache.py
+++ b/conda_libmamba_solver/shards_cache.py
@@ -98,8 +98,6 @@ def insert(self, raw_shard: AnnotatedRawShard):
             package: package name
             raw_shard: msgpack.zst compressed shard data
         """
-        # decompress and return shard for convenience, also to validate? unless
-        # caller would rather retrieve the shard from another thread.
         with self.conn as c:
             c.execute(
                 "INSERT OR IGNORE INTO SHARDS (url, package, shard) VALUES (?, ?, ?)",
@@ -117,11 +115,21 @@ def retrieve(self, url) -> ShardDict | None:
                 else None
             )  # type: ignore
 
-    def retrieve_multiple(self, urls: list[str]) -> dict[str, ShardDict | None]:
+    def retrieve_multiple(self, urls: list[str]) -> dict[str, ShardDict]:
         """
         Query database for cached shard urls.
 
-        Return a dict of urls in cache mapping to the Shard or None if not present.
+        Return a dict of urls in cache mapping to the Shard. If not in cache,
+        that url will not appear in the result.
+        """
+        return {k: v[0] for k, v in self.retrieve_multiple_size(urls).items()}
+
+    def retrieve_multiple_size(self, urls: list[str]) -> dict[str, tuple[ShardDict, int]]:
+        """
+        Query database for cached shard urls; include compressed size.
+
+        Return a dict of urls in cache mapping to the Shard, omit URLs that are
+        not found.
         """
         if not urls:
             return {}  # this optimization does not save a noticeable amount of time.
@@ -132,12 +140,13 @@ def retrieve_multiple(self, urls: list[str]) -> dict[str, ShardDict | None]:
 
         query = f"SELECT url, shard FROM shards WHERE url IN ({','.join(('?',) * len(urls))}) ORDER BY url"
         with self.conn as c:
-            result: dict[str, ShardDict | None] = {
-                row["url"]: msgpack.loads(
-                    dctx.decompress(row["shard"], max_output_size=ZSTD_MAX_SHARD_SIZE)
+            result: dict[str, tuple[ShardDict, int]] = {
+                row["url"]: (
+                    msgpack.loads(
+                        dctx.decompress(row["shard"], max_output_size=ZSTD_MAX_SHARD_SIZE)
+                    ),
+                    len(row["shard"]),
                 )
-                if row
-                else None
                 for row in c.execute(query, urls)  # type: ignore
             }
             return result
diff --git a/conda_libmamba_solver/shards_subset.py b/conda_libmamba_solver/shards_subset.py
index 913bff28..eca21b2b 100644
--- a/conda_libmamba_solver/shards_subset.py
+++ b/conda_libmamba_solver/shards_subset.py
@@ -226,7 +226,12 @@ def reachable_bfs(self, root_packages):
                     if not next_node.visited:
                         node_queue.append(next_node)
 
-    def reachable_pipelined(self, root_packages):
+    def reachable_httpx(self, root_packages):
+        from .shards_subset_http2 import network_fetch_thread_httpx
+
+        return self.reachable_pipelined(root_packages, network_fetch_thread_httpx)
+
+    def reachable_pipelined(self, root_packages, network_worker=None):
         """
         Fetch all packages reachable from `root_packages`' by following
         dependencies.
@@ -249,8 +254,11 @@ def reachable_pipelined(self, root_packages):
             daemon=True,  # may have to set to False if we ever want to run in a subinterpreter
         )
 
+        if network_worker is None:
+            network_worker = network_fetch_thread
+
         network_thread = threading.Thread(
-            target=network_fetch_thread,
+            target=network_worker,
             args=(cache_miss_queue, shard_out_queue, cache, self.shardlikes),
             daemon=True,
         )
@@ -397,7 +405,7 @@ def drain_pending(
 def build_repodata_subset(
     root_packages: Iterable[str],
     channels: dict[str, Channel],
-    algorithm: Literal["bfs", "pipelined"] = RepodataSubset.DEFAULT_STRATEGY,
+    algorithm: Literal["bfs", "pipelined", "httpx"] = RepodataSubset.DEFAULT_STRATEGY,
 ) -> dict[str, ShardBase]:
     """
     Retrieve all necessary information to build a repodata subset.
diff --git a/conda_libmamba_solver/shards_subset_http2.py b/conda_libmamba_solver/shards_subset_http2.py
new file mode 100644
index 00000000..93cb663e
--- /dev/null
+++ b/conda_libmamba_solver/shards_subset_http2.py
@@ -0,0 +1,110 @@
+# Copyright (C) 2022 Anaconda, Inc
+# Copyright (C) 2023 conda
+# SPDX-License-Identifier: BSD-3-Clause
+"""
+http/2 network fetching of shards, proof of concept.
+
+The greater concurrency of http/2 should be very beneficial for sharded
+repodata.
+"""
+
+import asyncio
+import concurrent.futures
+import logging
+from queue import Queue
+
+import httpx
+import msgpack
+import zstandard
+
+from .shards import ShardBase, Shards
+from .shards_cache import ZSTD_MAX_SHARD_SIZE, AnnotatedRawShard, ShardCache
+from .shards_subset import NodeId, exception_to_queue
+from .shards_typing import ShardDict
+
+log = logging.getLogger(__name__)
+
+
+async def _network_fetch_loop_httpx(
+    in_queue: Queue[list[NodeId] | None],
+    shard_out_queue: Queue[list[tuple[NodeId, ShardDict] | Exception] | None],
+    cache: ShardCache,
+    shardlikes: list[ShardBase],
+):
+    cache = cache.copy()
+    dctx = zstandard.ZstdDecompressor()
+    shardlikes_by_url = {s.url: s for s in shardlikes}
+
+    async def fetch(client: httpx.AsyncClient, url: str, node_id: NodeId):
+        response = await client.get(url)
+        response.raise_for_status()
+        data = response.content
+        return (url, node_id, data)
+
+    async def submit(client, node_id):
+        # this worker should only recieve network node_id's:
+        shardlike = shardlikes_by_url[node_id.channel]
+        if not isinstance(shardlike, Shards):
+            log.warning("network_fetch_thread got non-network shardlike")
+            return
+        url = shardlikes_by_url[node_id.channel].shard_url(node_id.package)
+        return await fetch(client, url, node_id)
+
+    async def get_work():
+        with concurrent.futures.ThreadPoolExecutor() as thread_pool:
+            while True:
+                batch = await asyncio.get_running_loop().run_in_executor(thread_pool, in_queue.get)
+                if batch is None:
+                    break
+                for node_id in batch:
+                    yield node_id
+
+    def handle_result(task: asyncio.Task):
+        tasks.remove(task)
+        url, node_id, data = task.result()  # can raise exceptions from fetch
+        log.debug("Fetch %s (%s bytes)", url, len(data))
+        # Decompress and parse. If it decodes as
+        # msgpack.zst, insert into cache. Then put "known
+        # good" shard into out queue.
+        shard: ShardDict = msgpack.loads(
+            dctx.decompress(data, max_output_size=ZSTD_MAX_SHARD_SIZE)
+        )  # type: ignore[assign]
+        cache.insert(AnnotatedRawShard(url, node_id.package, data))
+        shard_out_queue.put([(node_id, shard)])
+
+    async with httpx.AsyncClient(http2=True) as client:
+        results: list[tuple[NodeId, ShardDict] | Exception] = []
+
+        tasks = set()
+        async for node_id in get_work():
+            task = asyncio.create_task(submit(client, node_id))
+            task.add_done_callback(handle_result)
+            tasks.add(task)
+
+        # this loop is unnecessary as handle_result sends results to shard_out_queue
+        for task in asyncio.as_completed(tasks):
+            try:
+                result = await task
+                results.append(result)
+            except Exception as e:
+                results.append(e)
+
+
+@exception_to_queue
+def network_fetch_thread_httpx(
+    in_queue: Queue[list[NodeId] | None],
+    shard_out_queue: Queue[list[tuple[NodeId, ShardDict] | Exception] | None],
+    cache: ShardCache,
+    shardlikes: list[ShardBase],
+):
+    """
+    in_queue contains urls to fetch over the network.
+    While the in_queue has not received a sentinel None, empty everything from
+    the queue. Fetch all of them over the network. Fetched shards go to
+    shard_out_queue.
+    Unhandled exceptions also go to shard_out_queue, and exit this thread.
+    """
+    for shard in shardlikes:
+        if not shard.url.startswith("http"):
+            raise ValueError(f"Unsupported shard in http/2 fetch thread: {shard.url}")
+    asyncio.run(_network_fetch_loop_httpx(in_queue, shard_out_queue, cache, shardlikes))
diff --git a/tests/requirements.txt b/tests/requirements.txt
index af22ac83..d526143c 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -7,3 +7,7 @@ pytest-codspeed >=4
 conda-forge::pytest-mock
 # needed for many conda tests
 flask
+httpx
+h2
+hpack
+hyperframe
diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 18a712ef..f9ea29db 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: BSD-3-Clause
 from __future__ import annotations
 
+import asyncio
 import concurrent.futures
 import random
 import threading
 import time
+import typing
 import urllib.parse
 from contextlib import suppress
+from dataclasses import dataclass
 from pathlib import Path
 from queue import Empty, SimpleQueue
 from typing import TYPE_CHECKING
@@ -17,12 +20,13 @@
 import conda.gateways.repodata
 import pytest
 import pytest_codspeed
+import requests
 from conda.common.compat import on_win
 from conda.core.subdir_data import SubdirData
 from conda.models.channel import Channel
 from requests.exceptions import HTTPError
 
-from conda_libmamba_solver import shards_cache, shards_subset
+from conda_libmamba_solver import shards_cache, shards_subset, shards_subset_http2
 from conda_libmamba_solver.shards import ShardLike, fetch_channels, fetch_shards_index
 from conda_libmamba_solver.shards_subset import (
     NodeId,
@@ -123,7 +127,7 @@ def clean_cache(conda_cli: CondaCLIFixture):
 
 @pytest.mark.skipif(not codspeed_supported(), reason="pytest-codspeed-version-4")
 @pytest.mark.parametrize("cache_state", ("cold", "warm"))
-@pytest.mark.parametrize("algorithm", ("bfs", "pipelined"))
+@pytest.mark.parametrize("algorithm", ("bfs", "pipelined", "httpx"))
 @pytest.mark.parametrize(
     "scenario",
     TESTING_SCENARIOS,
@@ -331,6 +335,44 @@ def test_shards_network_thread(http_server_shards, shard_cache_with_data):
     network_thread.join(5)
 
 
+def test_shards_network_thread_httpx(http_server_shards, shard_cache_with_data):
+    """
+    Test network retrieval thread, meant to be chained after the sqlite3 thread
+    by having network_in_queue = sqlite3 thread's network_out_queue.
+    """
+    cache, fake_shards = shard_cache_with_data
+    channel = Channel.from_url(f"{http_server_shards}/noarch")
+    subdir_data = SubdirData(channel)
+    found = fetch_shards_index(subdir_data)
+    assert found
+
+    network_in_queue: SimpleQueue[list[NodeId] | None] = SimpleQueue()
+    shard_out_queue: SimpleQueue[list[tuple[NodeId, ShardDict]]] = SimpleQueue()
+
+    # this kind of thread can crash, and we don't hear back without our own
+    # handling.
+    network_thread = threading.Thread(
+        target=shards_subset_http2.network_fetch_thread_httpx,
+        args=(network_in_queue, shard_out_queue, cache, [found]),
+        daemon=False,
+    )
+
+    node_ids = [NodeId(package, found.url) for package in found.package_names]
+
+    # several batches, then None "finish thread" sentinel
+    network_in_queue.put([node_ids[0]])
+    network_in_queue.put(node_ids[1:])
+    network_in_queue.put(None)
+
+    network_thread.start()
+
+    while batch := shard_out_queue.get(timeout=1):
+        for url, shard in batch:
+            print(url, len(shard), "bytes")
+
+    network_thread.join(5)
+
+
 # endregion
 
 
@@ -748,3 +790,187 @@ def test_repodata_subset_misc():
     assert tuple(
         RepodataSubset.has_strategy(strategy) for strategy in ("bfs", "pipelined", "squirrel")
     ) == (True, True, False)
+
+
+@dataclass
+class ShardFetchResult:
+    node_id: NodeId
+    shard: ShardDict
+    size: int
+
+
+class NetworkSimulator:
+    """
+    Simulate a network with configurable parallelism and bandwidth, for
+    RepodataSubset.reachable_pipelined.
+    """
+
+    def __init__(
+        self,
+        connections: int,
+        bandwidth_mbps: float,
+        latency_ms: int,
+        cache: shards_cache.ShardCache,
+        initial_delay_bytes=0,  # wait for this many bytes before processing nodes
+    ):
+        self.connections = connections
+        self.bandwidth_mbps = bandwidth_mbps
+        self.latency_ns = latency_ms * 1_000_000
+        self.initial_delay_bytes = initial_delay_bytes
+
+        self.in_queue: SimpleQueue[Sequence[NodeId] | None] = SimpleQueue()
+        self.async_queue = asyncio.Queue()
+        self.out_queue: SimpleQueue[list[tuple[NodeId, ShardDict] | Exception] | None] = (
+            SimpleQueue()
+        )
+
+        self.cache = cache
+
+    def __str__(self):
+        return (
+            f"NetworkSimulator(connections={self.connections}, "
+            f"bandwidth_mbps={self.bandwidth_mbps}, "
+            f"latency_ms={self.latency_ns / 1_000_000}, "
+            f"initial_delay_bytes={self.initial_delay_bytes})"
+        )
+
+    def transfer_time(self, byte_count: int):
+        """
+        Time to transfer byte_count bytes at configured bandwidth.
+        """
+        return (byte_count * 8) / (self.bandwidth_mbps * 1_000_000)
+
+    async def delay_shards(self):
+        """
+        Delay shards based on connections, bandwidth, and latency.
+        """
+        cache = self.cache.copy()
+
+        async def get_work() -> typing.AsyncGenerator[Sequence[NodeId], None]:
+            with concurrent.futures.ThreadPoolExecutor() as thread_pool:
+                while True:
+                    batch = await asyncio.get_running_loop().run_in_executor(
+                        thread_pool, self.in_queue.get
+                    )
+                    if batch is None:
+                        break
+                    yield batch
+
+        connection_pool = asyncio.Semaphore(self.connections)
+        bandwidth_pool = asyncio.Semaphore(1)
+
+        async def latency_task(item: ShardFetchResult):
+            async with connection_pool:
+                # print("Get", Channel(item.node_id.channel), item.node_id.package)
+                await asyncio.sleep(self.latency_ns / 1_000_000_000)
+                asyncio.create_task(bandwidth_task(item))
+
+        async def bandwidth_task(item: ShardFetchResult):
+            async with bandwidth_pool:
+                # one task at a time waits proportional to size / bandwidth
+                transfer_time = self.transfer_time(item.size)
+                await asyncio.sleep(transfer_time)
+                # print("Got", Channel(item.node_id.channel), item.node_id.package)
+                self.out_queue.put([(item.node_id, item.shard)])
+
+        async def process_nodes():
+            async for node_ids in get_work():
+                # latency needs to be added after we go into the request queue. ignore the sqlite3 latency.
+                cached = cache.retrieve_multiple_size([node_id.shard_url for node_id in node_ids])
+                found: list[tuple[NodeId, tuple[ShardDict, int]]] = []
+                not_found: list[NodeId] = []
+                print("Batch size", len(node_ids))
+                for node_id in node_ids:
+                    if shard_info := cached.get(node_id.shard_url):
+                        found.append((node_id, shard_info))
+                    else:
+                        not_found.append(node_id)
+                        print(f"Simulator wants full cache, but missing {node_id.shard_url}")
+                        self.out_queue.put(None)
+                        return
+
+                for node_id, (shard, size) in found:
+                    asyncio.create_task(
+                        latency_task(
+                            ShardFetchResult(
+                                node_id=node_id,
+                                shard=shard,
+                                size=size,
+                            )
+                        )
+                    )
+
+        print("Processing nodes")
+        await process_nodes()
+
+    def index_transfer_delay(self):
+        """
+        Call before RepodataSubset.build_repodata_subset to simulate initial index transfer delay.
+        """
+        index_transfer = self.transfer_time(self.initial_delay_bytes)
+        print(
+            f"Wait to transfer repodata_shards.msgpack.zst ({self.initial_delay_bytes} bytes, {index_transfer:.2f} s)"
+        )
+        time.sleep(index_transfer)
+
+    def run(self):
+        asyncio.run(self.delay_shards())
+
+
+@pytest.fixture(scope="session")
+def repodata_index_transfer_size():
+    """
+    Determine the transfer size of the shards index for use in network simulator.
+    """
+    channel = Channel("conda-forge-sharded/linux-64")
+    urls = channel.urls()
+    result = {}
+    session = requests.Session()
+    for filename in "repodata_shards.msgpack.zst", "repodata.json.zst":
+        total_size = 0
+        for url in urls:
+            total_size += int(session.head(f"{url}/{filename}").headers.get("Content-Length", 0))
+        result[filename] = total_size
+    return result
+
+
+def test_repodata_subset_network_simulator(repodata_index_transfer_size):
+    """
+    Test RepodataSubset.reachable_pipelined with a simulated network.
+    """
+
+    # May remove debugging "is thread alive" from pipelined_main_thread later.
+    class FakeThread:
+        def is_alive(self):
+            return True
+
+        def join(self, timeout=None):
+            pass
+
+    # Set up test channel and root packages
+    channel = Channel("conda-forge-sharded/linux-64")
+    channel_data = fetch_channels([channel])
+    root_packages = ["python", "vaex"]
+
+    # populate cache
+    build_repodata_subset(root_packages, [channel], algorithm="httpx")
+
+    # set up network simulator
+    simulator = NetworkSimulator(
+        connections=10,
+        bandwidth_mbps=10.0,
+        latency_ms=100,
+        cache=list(channel_data.values())[0].shards_cache,  # type: ignore[arg-type]
+        initial_delay_bytes=repodata_index_transfer_size["repodata_shards.msgpack.zst"],
+    )
+
+    simulator_thread = threading.Thread(target=simulator.run)
+    simulator_thread.start()
+
+    print()
+    print(simulator)
+    simulator.index_transfer_delay()
+    subset = RepodataSubset(channel_data.values())
+    subset.pipelined_main_thread(
+        root_packages, simulator.in_queue, simulator.out_queue, FakeThread(), FakeThread()
+    )

From 17b582e8ab7c980697c1d0d2f12e2551db484839 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Fri, 14 Nov 2025 14:23:56 -0500
Subject: [PATCH 02/18] run simulation under different scenarios

---
 conda_libmamba_solver/shards.py |  6 +++
 tests/test_shards_subset.py     | 92 ++++++++++++++++++++++++---------
 2 files changed, 73 insertions(+), 25 deletions(-)

diff --git a/conda_libmamba_solver/shards.py b/conda_libmamba_solver/shards.py
index ccdbc539..16e73a5f 100644
--- a/conda_libmamba_solver/shards.py
+++ b/conda_libmamba_solver/shards.py
@@ -203,6 +203,12 @@ def build_repodata(self) -> RepodataDict:
                 repodata[package_group].update(shard[package_group])
         return repodata
 
+    def reset(self):
+        """
+        Clear visited shards.
+        """
+        self.visited.clear()
+
 
 class ShardLike(ShardBase):
     """
diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index f9ea29db..56441d1f 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -14,7 +14,6 @@
 from dataclasses import dataclass
 from pathlib import Path
 from queue import Empty, SimpleQueue
-from typing import TYPE_CHECKING
 from unittest.mock import patch
 
 import conda.gateways.repodata
@@ -42,7 +41,7 @@
     _timer,
 )
 
-if TYPE_CHECKING:
+if typing.TYPE_CHECKING:
     from collections.abc import Sequence
 
     from conda.testing.fixtures import CondaCLIFixture
@@ -817,6 +816,7 @@ def __init__(
         self.bandwidth_mbps = bandwidth_mbps
         self.latency_ns = latency_ms * 1_000_000
         self.initial_delay_bytes = initial_delay_bytes
+        self.bytes_transferred = 0
 
         self.in_queue: SimpleQueue[Sequence[NodeId] | None] = SimpleQueue()
         self.async_queue = asyncio.Queue()
@@ -870,16 +870,22 @@ async def bandwidth_task(item: ShardFetchResult):
                 # one task at a time waits proportional to size / bandwidth
                 transfer_time = self.transfer_time(item.size)
                 await asyncio.sleep(transfer_time)
-                # print("Got", Channel(item.node_id.channel), item.node_id.package)
+                self.bytes_transferred += item.size
                 self.out_queue.put([(item.node_id, item.shard)])
 
         async def process_nodes():
+            total_nodes = 0
+            wrap_nodes = 0
             async for node_ids in get_work():
                 # latency needs to be added after we go into the request queue. ignore the sqlite3 latency.
                 cached = cache.retrieve_multiple_size([node_id.shard_url for node_id in node_ids])
                 found: list[tuple[NodeId, tuple[ShardDict, int]]] = []
                 not_found: list[NodeId] = []
-                print("Batch size", len(node_ids))
+                wrap_nodes += len(node_ids)
+                total_nodes += len(node_ids)
+                if wrap_nodes > 100:
+                    print(f"Simulated {total_nodes} nodes")
+                    wrap_nodes %= 100
                 for node_id in node_ids:
                     if shard_info := cached.get(node_id.shard_url):
                         found.append((node_id, shard_info))
@@ -915,6 +921,7 @@ def index_transfer_delay(self):
 
     def run(self):
         asyncio.run(self.delay_shards())
+        print("NetworkSimulator end")
 
 
 @pytest.fixture(scope="session")
@@ -934,7 +941,25 @@ def repodata_index_transfer_size():
     return result
 
 
-def test_repodata_subset_network_simulator(repodata_index_transfer_size):
+# Different latency/bandwidth scenarios, loosely based on Firefox debug console.
+NETWORK_SCENARIOS = {
+    "3G": {"bandwidth_mbps": 4, "latency_ms": 100},  # want a high latency
+    "5G": {"bandwidth_mbps": 30, "latency_ms": 30},  # medium
+    "HALFGIG": {
+        "bandwidth_mbps": 500,
+        "latency_ms": 20,
+    },  # and low (ping time to cloudflare) option
+}
+
+
+@pytest.mark.parametrize("scenario_name", NETWORK_SCENARIOS.keys())
+@pytest.mark.parametrize("connections", [10, 20, 100])
+@pytest.mark.parametrize(
+    "packages", [["python", "celery"], ["numpy", "pandas", "scipy"]], ids=["web", "sci"]
+)
+def test_repodata_subset_network_simulator(
+    benchmark, repodata_index_transfer_size, scenario_name, connections, packages
+):
     """
     Test RepodataSubset.reachable_pipelined with a simulated network.
     """
@@ -950,27 +975,44 @@ def join(self, timeout=None):
     # Set up test channel and root packages
     channel = Channel("conda-forge-sharded/linux-64")
     channel_data = fetch_channels([channel])
-    root_packages = ["python", "vaex"]
 
     # populate cache
-    build_repodata_subset(root_packages, [channel], algorithm="httpx")
-
-    # set up network simulator
-    simulator = NetworkSimulator(
-        connections=10,
-        bandwidth_mbps=10.0,
-        latency_ms=100,
-        cache=list(channel_data.values())[0].shards_cache,  # type: ignore[arg-type]
-        initial_delay_bytes=repodata_index_transfer_size["repodata_shards.msgpack.zst"],
-    )
+    build_repodata_subset(packages, [channel], algorithm="httpx")
+
+    def build_simulator():
+        # set up network simulator
+        simulator = NetworkSimulator(
+            connections=connections,
+            cache=list(channel_data.values())[0].shards_cache,  # type: ignore[arg-type]
+            initial_delay_bytes=repodata_index_transfer_size["repodata_shards.msgpack.zst"],
+            **NETWORK_SCENARIOS[scenario_name],
+        )
 
-    simulator_thread = threading.Thread(target=simulator.run)
-    simulator_thread.start()
+        simulator_thread = threading.Thread(target=simulator.run)
+        simulator_thread.start()
 
-    print()
-    print(simulator)
-    simulator.index_transfer_delay()
-    subset = RepodataSubset(channel_data.values())
-    subset.pipelined_main_thread(
-        root_packages, simulator.in_queue, simulator.out_queue, FakeThread(), FakeThread()
-    )
+        return simulator
+
+    @benchmark
+    def simulate():
+        simulator = build_simulator()
+        monolithic_transfer = simulator.transfer_time(
+            repodata_index_transfer_size["repodata.json.zst"]
+        )
+        with _timer(f"Non-shards download {monolithic_transfer:.2f}s vs simulated traversal"):
+            simulator.index_transfer_delay()
+            for value in channel_data.values():
+                value.reset()  # avoid already-loaded shortcut
+            subset = RepodataSubset(channel_data.values())
+            subset.pipelined_main_thread(
+                packages, simulator.in_queue, simulator.out_queue, FakeThread(), FakeThread()
+            )
+            print(f"{len(subset.nodes)} nodes found.")
+            shards_mib = (
+                simulator.bytes_transferred + repodata_index_transfer_size["repodata.json.zst"]
+            ) / (2**20)
+            repodata_mib = repodata_index_transfer_size["repodata.json.zst"] / (2**20)
+            print(f"Shards {shards_mib:0.2f}MiB vs repodata.json.zst {repodata_mib:0.2f}MiB")
+
+    # Missing here is the "load into LibMambaIndexHelper" step. When bandwidth
+    # is high the time spent parsing repodata vs shards can dominate.

From ccbdb25e9f27991805a63b7831667ea407da3495 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Fri, 14 Nov 2025 14:34:21 -0500
Subject: [PATCH 03/18] include packages in output

---
 tests/test_shards_subset.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 56441d1f..037259c5 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -962,6 +962,12 @@ def test_repodata_subset_network_simulator(
 ):
     """
     Test RepodataSubset.reachable_pipelined with a simulated network.
+
+    This underestimates the time taken to download shards, possibly due to
+    compounded async.sleep() error. We could measure the actual start and end
+    time and then sleep more or less the next time based on the error(); or, we
+    could increment a counter with the correct latency and bandwidth delays
+    instead of sleeping in real time.
     """
 
     # May remove debugging "is thread alive" from pipelined_main_thread later.
@@ -1012,7 +1018,9 @@ def simulate():
                 simulator.bytes_transferred + repodata_index_transfer_size["repodata.json.zst"]
             ) / (2**20)
             repodata_mib = repodata_index_transfer_size["repodata.json.zst"] / (2**20)
-            print(f"Shards {shards_mib:0.2f}MiB vs repodata.json.zst {repodata_mib:0.2f}MiB")
+            print(
+                f"Shards {shards_mib:0.2f}MiB vs repodata.json.zst {repodata_mib:0.2f}MiB for {packages}"
+            )
 
     # Missing here is the "load into LibMambaIndexHelper" step. When bandwidth
     # is high the time spent parsing repodata vs shards can dominate.

From 7f650b4208c12ec55a358ca9eb80220255e36391 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Fri, 14 Nov 2025 14:58:09 -0500
Subject: [PATCH 04/18] track some timer error; correct shards MiB total

---
 tests/test_shards_subset.py | 48 ++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 037259c5..05228afb 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -814,7 +814,7 @@ def __init__(
     ):
         self.connections = connections
         self.bandwidth_mbps = bandwidth_mbps
-        self.latency_ns = latency_ms * 1_000_000
+        self.latency_ms = latency_ms
         self.initial_delay_bytes = initial_delay_bytes
         self.bytes_transferred = 0
 
@@ -830,8 +830,9 @@ def __str__(self):
         return (
             f"NetworkSimulator(connections={self.connections}, "
             f"bandwidth_mbps={self.bandwidth_mbps}, "
-            f"latency_ms={self.latency_ns / 1_000_000}, "
-            f"initial_delay_bytes={self.initial_delay_bytes})"
+            f"latency_ms={self.latency_ms}, "
+            f"initial_delay_bytes={self.initial_delay_bytes}), "
+            f"cache={self.cache.base}"
         )
 
     def transfer_time(self, byte_count: int):
@@ -859,17 +860,29 @@ async def get_work() -> typing.AsyncGenerator[Sequence[NodeId], None]:
         connection_pool = asyncio.Semaphore(self.connections)
         bandwidth_pool = asyncio.Semaphore(1)
 
+        latency_error = 0.0
+
         async def latency_task(item: ShardFetchResult):
+            nonlocal latency_error
             async with connection_pool:
                 # print("Get", Channel(item.node_id.channel), item.node_id.package)
-                await asyncio.sleep(self.latency_ns / 1_000_000_000)
+                latency_start = time.monotonic()
+                await asyncio.sleep(self.latency_ms / 1000.0)
+                latency_end = time.monotonic()
+                latency_error += (self.latency_ms / 1000.0) - (latency_end - latency_start)
                 asyncio.create_task(bandwidth_task(item))
 
+        transfer_error = 0.0
+
         async def bandwidth_task(item: ShardFetchResult):
+            nonlocal transfer_error
             async with bandwidth_pool:
                 # one task at a time waits proportional to size / bandwidth
                 transfer_time = self.transfer_time(item.size)
+                transfer_start = time.monotonic()
                 await asyncio.sleep(transfer_time)
+                transfer_end = time.monotonic()
+                transfer_error += transfer_time - (transfer_end - transfer_start)
                 self.bytes_transferred += item.size
                 self.out_queue.put([(item.node_id, item.shard)])
 
@@ -908,6 +921,8 @@ async def process_nodes():
 
         print("Processing nodes")
         await process_nodes()
+        print(f"Bandwidth error {transfer_error:0.3f}")
+        print(f"Latency error {latency_error:0.3f}")
 
     def index_transfer_delay(self):
         """
@@ -943,8 +958,8 @@ def repodata_index_transfer_size():
 
 # Different latency/bandwidth scenarios, loosely based on Firefox debug console.
 NETWORK_SCENARIOS = {
-    "3G": {"bandwidth_mbps": 4, "latency_ms": 100},  # want a high latency
-    "5G": {"bandwidth_mbps": 30, "latency_ms": 30},  # medium
+    "4MBPS": {"bandwidth_mbps": 4, "latency_ms": 100},  # want a high latency
+    "30MBPS": {"bandwidth_mbps": 30, "latency_ms": 30},  # medium
     "HALFGIG": {
         "bandwidth_mbps": 500,
         "latency_ms": 20,
@@ -961,13 +976,8 @@ def test_repodata_subset_network_simulator(
     benchmark, repodata_index_transfer_size, scenario_name, connections, packages
 ):
     """
-    Test RepodataSubset.reachable_pipelined with a simulated network.
-
-    This underestimates the time taken to download shards, possibly due to
-    compounded async.sleep() error. We could measure the actual start and end
-    time and then sleep more or less the next time based on the error(); or, we
-    could increment a counter with the correct latency and bandwidth delays
-    instead of sleeping in real time.
+    Test RepodataSubset.reachable_pipelined with a simulated network. Real-world
+    results may vary.
     """
 
     # May remove debugging "is thread alive" from pipelined_main_thread later.
@@ -1005,7 +1015,7 @@ def simulate():
         monolithic_transfer = simulator.transfer_time(
             repodata_index_transfer_size["repodata.json.zst"]
         )
-        with _timer(f"Non-shards download {monolithic_transfer:.2f}s vs simulated traversal"):
+        with _timer(f"Non-shards transfer {monolithic_transfer:.2f}s vs simulated traversal"):
             simulator.index_transfer_delay()
             for value in channel_data.values():
                 value.reset()  # avoid already-loaded shortcut
@@ -1013,13 +1023,13 @@ def simulate():
             subset.pipelined_main_thread(
                 packages, simulator.in_queue, simulator.out_queue, FakeThread(), FakeThread()
             )
-            print(f"{len(subset.nodes)} nodes found.")
-            shards_mib = (
-                simulator.bytes_transferred + repodata_index_transfer_size["repodata.json.zst"]
-            ) / (2**20)
+            shards_mib = (simulator.bytes_transferred) / (2**20)
+            shards_index_mib = +repodata_index_transfer_size["repodata_shards.msgpack.zst"] / (
+                2**20
+            )
             repodata_mib = repodata_index_transfer_size["repodata.json.zst"] / (2**20)
             print(
-                f"Shards {shards_mib:0.2f}MiB vs repodata.json.zst {repodata_mib:0.2f}MiB for {packages}"
+                f"Shards {shards_index_mib:0.2f}MiB+{shards_mib:0.2f}MiB vs repodata.json.zst {repodata_mib:0.2f}MiB for {packages}, {len(subset.nodes)} nodes"
             )
 
     # Missing here is the "load into LibMambaIndexHelper" step. When bandwidth

From 52c50751d203e22125f10ed948c4a0cf57571dae Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Fri, 14 Nov 2025 15:02:07 -0500
Subject: [PATCH 05/18] adjust scenarios

---
 tests/test_shards_subset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 05228afb..f405ada1 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -970,7 +970,7 @@ def repodata_index_transfer_size():
 @pytest.mark.parametrize("scenario_name", NETWORK_SCENARIOS.keys())
 @pytest.mark.parametrize("connections", [10, 20, 100])
 @pytest.mark.parametrize(
-    "packages", [["python", "celery"], ["numpy", "pandas", "scipy"]], ids=["web", "sci"]
+    "packages", [["python"], ["django", "celery"], ["vaex"]], ids=["python", "django", "vaex"]
 )
 def test_repodata_subset_network_simulator(
     benchmark, repodata_index_transfer_size, scenario_name, connections, packages

From 3f3ea0aaf6596d25f87fabaeb82cf9db914fa860 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Fri, 14 Nov 2025 15:06:20 -0500
Subject: [PATCH 06/18] rename variable

---
 tests/test_shards_subset.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index f405ada1..16be9a6e 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -810,12 +810,12 @@ def __init__(
         bandwidth_mbps: float,
         latency_ms: int,
         cache: shards_cache.ShardCache,
-        initial_delay_bytes=0,  # wait for this many bytes before processing nodes
+        index_delay_bytes=0,  # wait for this many bytes before processing nodes
     ):
         self.connections = connections
         self.bandwidth_mbps = bandwidth_mbps
         self.latency_ms = latency_ms
-        self.initial_delay_bytes = initial_delay_bytes
+        self.index_delay_bytes = index_delay_bytes
         self.bytes_transferred = 0
 
         self.in_queue: SimpleQueue[Sequence[NodeId] | None] = SimpleQueue()
@@ -831,7 +831,7 @@ def __str__(self):
             f"NetworkSimulator(connections={self.connections}, "
             f"bandwidth_mbps={self.bandwidth_mbps}, "
             f"latency_ms={self.latency_ms}, "
-            f"initial_delay_bytes={self.initial_delay_bytes}), "
+            f"index_delay_bytes={self.index_delay_bytes}), "
             f"cache={self.cache.base}"
         )
 
@@ -928,9 +928,9 @@ def index_transfer_delay(self):
         """
         Call before RepodataSubset.build_repodata_subset to simulate initial index transfer delay.
         """
-        index_transfer = self.transfer_time(self.initial_delay_bytes)
+        index_transfer = self.transfer_time(self.index_delay_bytes)
         print(
-            f"Wait to transfer repodata_shards.msgpack.zst ({self.initial_delay_bytes} bytes, {index_transfer:.2f} s)"
+            f"Wait to transfer repodata_shards.msgpack.zst ({self.index_delay_bytes} bytes, {index_transfer:.2f} s)"
         )
         time.sleep(index_transfer)
 
@@ -1000,7 +1000,7 @@ def build_simulator():
         simulator = NetworkSimulator(
             connections=connections,
             cache=list(channel_data.values())[0].shards_cache,  # type: ignore[arg-type]
-            initial_delay_bytes=repodata_index_transfer_size["repodata_shards.msgpack.zst"],
+            index_delay_bytes=repodata_index_transfer_size["repodata_shards.msgpack.zst"],
             **NETWORK_SCENARIOS[scenario_name],
         )
 

From b5928c8a7e65467bd91d80be4c1d20e1e8cfa661 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Fri, 14 Nov 2025 15:38:50 -0500
Subject: [PATCH 07/18] minor tweaks

---
 tests/test_shards_subset.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 16be9a6e..73dff614 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -802,6 +802,9 @@ class NetworkSimulator:
     """
     Simulate a network with configurable parallelism and bandwidth, for
     RepodataSubset.reachable_pipelined.
+
+    Not sure this does a great job of simulating http/1 vs. http/2 "10 versus
+    much more than 10 simultaneous requests".
     """
 
     def __init__(
@@ -870,7 +873,7 @@ async def latency_task(item: ShardFetchResult):
                 await asyncio.sleep(self.latency_ms / 1000.0)
                 latency_end = time.monotonic()
                 latency_error += (self.latency_ms / 1000.0) - (latency_end - latency_start)
-                asyncio.create_task(bandwidth_task(item))
+            asyncio.create_task(bandwidth_task(item))
 
         transfer_error = 0.0
 
@@ -891,6 +894,7 @@ async def process_nodes():
             wrap_nodes = 0
             async for node_ids in get_work():
                 # latency needs to be added after we go into the request queue. ignore the sqlite3 latency.
+                # we could afford to cache these in RAM.
                 cached = cache.retrieve_multiple_size([node_id.shard_url for node_id in node_ids])
                 found: list[tuple[NodeId, tuple[ShardDict, int]]] = []
                 not_found: list[NodeId] = []
@@ -1012,6 +1016,7 @@ def build_simulator():
     @benchmark
     def simulate():
         simulator = build_simulator()
+        print(simulator)
         monolithic_transfer = simulator.transfer_time(
             repodata_index_transfer_size["repodata.json.zst"]
         )

From a3e562a203b4c54c1e116b5f27ecb74d8c87d95a Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 20 Nov 2025 11:58:32 +0100
Subject: [PATCH 08/18] tests: remove unused "prefetch_packages"

---
 tests/test_shards_subset.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 73dff614..8582774d 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -55,42 +55,36 @@
     {
         "name": "python",
         "packages": ["python"],
-        "prefetch_packages": [],
         "channel": "conda-forge-sharded",
         "platform": "linux-64",
     },
     {
         "name": "data_science_ml",
         "packages": ["scikit-learn", "matplotlib"],
-        "prefetch_packages": ["python", "numpy"],
         "channel": "conda-forge-sharded",
         "platform": "linux-64",
     },
     {
         "name": "web_development",
         "packages": ["django", "celery"],
-        "prefetch_packages": ["python", "requests"],
         "channel": "conda-forge-sharded",
         "platform": "linux-64",
     },
     {
         "name": "scientific_computing",
         "packages": ["scipy", "sympy", "pytorch"],
-        "prefetch_packages": ["python", "numpy", "pandas"],
         "channel": "conda-forge-sharded",
         "platform": "linux-64",
     },
     {
         "name": "devops_automation",
         "packages": ["ansible", "pyyaml", "jinja2"],
-        "prefetch_packages": ["python"],
         "channel": "conda-forge-sharded",
         "platform": "linux-64",
     },
     {
         "name": "vaex",
         "packages": ["vaex"],
-        "prefetch_packages": ["python", "numpy", "pandas"],
         "channel": "conda-forge-sharded",
         "platform": "linux-64",
     },

From 966b277600689b8a7ab96c7a8a2a80c1ee03e7ed Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 20 Nov 2025 11:59:23 +0100
Subject: [PATCH 09/18] benchmark test_load_channel_repo_info_shards

---
 tests/run_in_profiler.py    | 19 ++++++++++++++++++-
 tests/test_index.py         |  4 +++-
 tests/test_shards_subset.py | 12 ++++++++++--
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/tests/run_in_profiler.py b/tests/run_in_profiler.py
index 1e78cc04..04eb66a9 100644
--- a/tests/run_in_profiler.py
+++ b/tests/run_in_profiler.py
@@ -10,8 +10,10 @@
 import os
 import pathlib
 
+import pytest
 from conda.base.context import reset_context
 
+import tests.test_index
 import tests.test_shards_subset
 from conda_libmamba_solver import shards, shards_cache, shards_subset
 
@@ -29,4 +31,19 @@
 for module in (shards, shards_cache, shards_subset):
     module.log.setLevel(logging.DEBUG)
 
-tests.test_shards_subset.test_build_repodata_subset_pipelined(None, tmp_path)
+# tests.test_shards_subset.test_build_repodata_subset_pipelined(None, tmp_path)
+
+
+class Benchmark:
+    def pedantic(self, fn, rounds: int = 1):
+        return fn()
+        # for _ in range(1):
+        #     rc = fn()
+        # return rc
+
+
+monkeypatch = pytest.MonkeyPatch()
+for i in range(16):
+    tests.test_index.test_load_channel_repo_info_shards(
+        "shard", ("django", "celery"), tmp_path, None, monkeypatch, Benchmark()
+    )
diff --git a/tests/test_index.py b/tests/test_index.py
index be8e5e8a..763f1962 100644
--- a/tests/test_index.py
+++ b/tests/test_index.py
@@ -136,7 +136,6 @@ def test_load_channel_repo_info_shards(
     load_type: str,
     requested: tuple[str, ...],
     tmp_path: Path,
-    conda_cli: CondaCLIFixture,
     monkeypatch: pytest.MonkeyPatch,
     benchmark: BenchmarkFixture,
 ):
@@ -165,7 +164,10 @@ def index():
             pkgs_dirs=(),  # do not load local cache as a channel
             in_state=in_state,
         )
+        pass
 
+    # this fails for some reason if run twice
+    # cuda finder function crashes when run twice, sys.exit() called?
     index_helper = benchmark.pedantic(index, rounds=1)
 
     assert len(index_helper.repos) > 0
diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 8582774d..b0ae19ea 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -277,7 +277,7 @@ def test_shards_network_thread(http_server_shards, shard_cache_with_data):
     Test network retrieval thread, meant to be chained after the sqlite3 thread
     by having network_in_queue = sqlite3 thread's network_out_queue.
     """
-    cache, fake_shards = shard_cache_with_data
+    cache, _ = shard_cache_with_data
     channel = Channel.from_url(f"{http_server_shards}/noarch")
     subdir_data = SubdirData(channel)
     found = fetch_shards_index(subdir_data)
@@ -883,13 +883,20 @@ async def bandwidth_task(item: ShardFetchResult):
                 self.bytes_transferred += item.size
                 self.out_queue.put([(item.node_id, item.shard)])
 
+        sqlite_time = 0.0
+
         async def process_nodes():
+            nonlocal sqlite_time
             total_nodes = 0
             wrap_nodes = 0
             async for node_ids in get_work():
                 # latency needs to be added after we go into the request queue. ignore the sqlite3 latency.
                 # we could afford to cache these in RAM.
+                sqlite_start = time.monotonic()
                 cached = cache.retrieve_multiple_size([node_id.shard_url for node_id in node_ids])
+                sqlite_end = time.monotonic()
+                sqlite_time += sqlite_end - sqlite_start
+
                 found: list[tuple[NodeId, tuple[ShardDict, int]]] = []
                 not_found: list[NodeId] = []
                 wrap_nodes += len(node_ids)
@@ -921,6 +928,7 @@ async def process_nodes():
         await process_nodes()
         print(f"Bandwidth error {transfer_error:0.3f}")
         print(f"Latency error {latency_error:0.3f}")
+        print(f"SQLite took {sqlite_time:0.3f}")
 
     def index_transfer_delay(self):
         """
@@ -966,7 +974,7 @@ def repodata_index_transfer_size():
 
 
 @pytest.mark.parametrize("scenario_name", NETWORK_SCENARIOS.keys())
-@pytest.mark.parametrize("connections", [10, 20, 100])
+@pytest.mark.parametrize("connections", [5, 20, 80])
 @pytest.mark.parametrize(
     "packages", [["python"], ["django", "celery"], ["vaex"]], ids=["python", "django", "vaex"]
 )

From c81bd52b5dd90de2ca5365c107e867ac73ec254e Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 20 Nov 2025 13:41:18 +0100
Subject: [PATCH 10/18] add test to try to split out parse time from network
 time

---
 conda_libmamba_solver/index.py |  18 ++++--
 tests/test_index.py            | 103 ++++++++++++++++++++++++++++++++-
 tests/test_shards_subset.py    |   1 +
 3 files changed, 115 insertions(+), 7 deletions(-)

diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py
index cb49d1ae..f457c585 100644
--- a/conda_libmamba_solver/index.py
+++ b/conda_libmamba_solver/index.py
@@ -389,15 +389,21 @@ def _load_channel_repo_info_shards(
         self, urls_to_channel: dict[str, Channel]
     ) -> list[_ChannelRepoInfo]:
         """
-        Load repository information from sharded repodata cache.
+        Load repository information by fetching and processing repodata shards.
         """
         # make a subset of possible dependencies
         root_packages = (*self.in_state.installed.keys(), *self.in_state.requested)
-        channel_data = build_repodata_subset(root_packages, urls_to_channel)
-        channel_repo_infos = self._load_repo_info_from_repodata_dict(channel_data)
+        channel_data = self._build_repodata_subset(root_packages, urls_to_channel)
+        channel_repo_infos = self._load_repo_info_from_repodata_shards(channel_data)
 
         return channel_repo_infos
 
+    def _build_repodata_subset(
+        self, root_packages: tuple[str, ...], urls_to_channel: dict[str, Channel]
+    ) -> dict[str, ShardBase]:
+        # split into a separate method for tests
+        return build_repodata_subset(root_packages, urls_to_channel)
+
     def _load_channel_repo_info_json(
         self, urls_to_channel: dict[str, Channel], try_solv: bool
     ) -> list[_ChannelRepoInfo]:
@@ -609,12 +615,12 @@ def _load_pkgs_cache(self, pkgs_dirs: PathsType) -> list[RepoInfo]:
         return repos
 
     @time_recorder(module_name=__name__)
-    def _load_repo_info_from_repodata_dict(
+    def _load_repo_info_from_repodata_shards(
         self, repodata_subset: dict[str, ShardBase]
     ) -> list[_ChannelRepoInfo]:
         """
-        Load repository information from deserialized repodata.json-like
-        structures.
+        Load repository information from already-fetched ShardBase objects, that
+        produce in-memory repodata.json-like dicts.
         """
         repos = []
         for channel_url, shardlike in repodata_subset.items():
diff --git a/tests/test_index.py b/tests/test_index.py
index 763f1962..50cc4156 100644
--- a/tests/test_index.py
+++ b/tests/test_index.py
@@ -17,14 +17,18 @@
 from conda.models.channel import Channel
 
 from conda_libmamba_solver.index import LibMambaIndexHelper, _is_sharded_repodata_enabled
+from conda_libmamba_solver.shards_subset import build_repodata_subset
 from conda_libmamba_solver.state import SolverInputState
 
 if TYPE_CHECKING:
     import os
+    from collections.abc import Iterable
 
-    from conda.testing.fixtures import CondaCLIFixture
+    from conda.gateways.repodata import RepodataState
     from pytest_benchmark.plugin import BenchmarkFixture
 
+    from conda_libmamba_solver.shards import ShardBase
+
 
 initialize_logging()
 DATA = Path(__file__).parent / "data"
@@ -171,3 +175,100 @@ def index():
     index_helper = benchmark.pedantic(index, rounds=1)
 
     assert len(index_helper.repos) > 0
+
+
+@pytest.mark.parametrize(
+    "load_type,requested",
+    [
+        ("shard", ("python",)),
+        ("shard", ("django", "celery")),
+        ("shard", ("vaex",)),
+        ("repodata", ("vaex",)),
+        ("main", ()),
+    ],
+    ids=["shard-small", "shard-medium", "shard-large", "noshard", "main"],
+)
+def test_load_channel_repo_info_shards_parse_only(
+    load_type: str,
+    requested: tuple[str, ...],
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    benchmark: BenchmarkFixture,
+):
+    """
+    Benchmark shards/not-shards under different dependency tree sizes.
+
+    Only measure parse time, not "download + parse" time.
+
+    TODO: This test should eventually switch to just using conda-forge when that channel
+          supports shards and not the `conda-forge-sharded` channel.
+    """
+    # defaults is more than one channel but I don't think it is too significant for the test.
+    load_channel = "main" if load_type == "main" else "conda-forge-sharded"
+
+    monkeypatch.setattr(context.plugins, "use_sharded_repodata", load_type == "shard")
+    assert _is_sharded_repodata_enabled() == (load_type == "shard")
+
+    in_state = SolverInputState(str(tmp_path / "env"), requested=requested)
+
+    urls_to_channel_ = {
+        f"https://conda.anaconda.org/{load_channel}/noarch": Channel(f"{load_channel}/noarch"),
+        f"https://conda.anaconda.org/{load_channel}/linux-64": Channel(f"{load_channel}/linux-64"),
+    }
+
+    repodata_jsons = {}
+    channel_data = {}
+    root_packages_ = ()
+    if load_type == "shard":  # shards
+        root_packages_ = (*in_state.installed.keys(), *in_state.requested)
+        channel_data = build_repodata_subset(root_packages_, urls_to_channel_)
+
+    else:  # no shards
+        # this is an inefficient way to fetch repodata.json in the format
+        # expected by LibMambaIndexHelper since it loads it into the solver.
+        helper = LibMambaIndexHelper(
+            # this is expanded to noarch, linux-64 for shards.
+            channels=[Channel(f"{load_channel}/linux-64")],
+            subdirs=(
+                "noarch",
+                "linux-64",
+            ),
+            installed_records=(),  # do not load installed
+            pkgs_dirs=(),  # do not load local cache as a channel
+            in_state=in_state,
+        )
+        repodata_jsons = helper._fetch_repodata_jsons(urls_to_channel_.keys())
+
+    class LibMambaIndexHelperParseOnly(LibMambaIndexHelper):
+        def _build_repodata_subset(
+            self, root_packages: tuple[str, ...], urls_to_channel: dict[str, Channel]
+        ) -> dict[str, ShardBase]:
+            assert root_packages == root_packages_
+            assert urls_to_channel == urls_to_channel
+            return channel_data
+
+        def _fetch_repodata_jsons(
+            self, urls: Iterable[str]
+        ) -> dict[str, tuple[str, RepodataState]]:
+            assert sorted(urls) == sorted(urls_to_channel_.keys())
+            return repodata_jsons
+
+    def index():
+        return LibMambaIndexHelperParseOnly(
+            # this is expanded to noarch, linux-64 for shards.
+            channels=[Channel(f"{load_channel}/linux-64")],
+            subdirs=(
+                "noarch",
+                "linux-64",
+            ),
+            installed_records=(),  # do not load installed
+            pkgs_dirs=(),  # do not load local cache as a channel
+            in_state=in_state,
+        )
+        pass
+
+    # this fails for some reason if run twice
+    # cuda finder function crashes when rounds > 1, sys.exit() called?
+    index_helper = benchmark.pedantic(index, rounds=1)
+
+    assert len(index_helper.repos) > 0
diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index b0ae19ea..77e32dcb 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -965,6 +965,7 @@ def repodata_index_transfer_size():
 # Different latency/bandwidth scenarios, loosely based on Firefox debug console.
 NETWORK_SCENARIOS = {
     "4MBPS": {"bandwidth_mbps": 4, "latency_ms": 100},  # want a high latency
+    "11MBPS": {"bandwidth_mbps": 11, "latency_ms": 100},  # want a high latency
     "30MBPS": {"bandwidth_mbps": 30, "latency_ms": 30},  # medium
     "HALFGIG": {
         "bandwidth_mbps": 500,

From 7e2f4524b3e0bd284a94d7517a1d1611fc2a68b0 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 20 Nov 2025 16:15:03 +0100
Subject: [PATCH 11/18] begin two-container devcontainer

---
 .devcontainer/docker-compose.yml              | 22 ++++++++++++
 .devcontainer/network-simulator/Dockerfile    |  4 +++
 .../network-simulator/devcontainer.json       | 34 +++++++++++++++++++
 3 files changed, 60 insertions(+)
 create mode 100644 .devcontainer/docker-compose.yml
 create mode 100644 .devcontainer/network-simulator/Dockerfile
 create mode 100644 .devcontainer/network-simulator/devcontainer.json

diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
new file mode 100644
index 00000000..4c5ea991
--- /dev/null
+++ b/.devcontainer/docker-compose.yml
@@ -0,0 +1,22 @@
+services:
+  app:
+    build:
+      context: .
+      dockerfile: ./network-simulator/Dockerfile
+    volumes:
+      - ../../conda:/workspace/conda:cached
+      - ../../mamba:/workspace/mamba:cached
+      - ../../conda-libmamba-solver:/workspace/conda-libmamba-solver
+    cap_add:
+      - NET_ADMIN
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+  repo:
+    image: nginx
+    restart: unless-stopped
+    volumes:
+      - .:/usr/share/nginx/html:ro # instead, the test shards directory
+    cap_add:
+      - NET_ADMIN
diff --git a/.devcontainer/network-simulator/Dockerfile b/.devcontainer/network-simulator/Dockerfile
new file mode 100644
index 00000000..962e0e74
--- /dev/null
+++ b/.devcontainer/network-simulator/Dockerfile
@@ -0,0 +1,4 @@
+FROM continuumio/miniconda3:latest
+COPY apt-deps.txt post_create.sh post_start.sh ./
+RUN bash ./post_create.sh
+CMD [ "bash", "post_start.sh" ]
\ No newline at end of file
diff --git a/.devcontainer/network-simulator/devcontainer.json b/.devcontainer/network-simulator/devcontainer.json
new file mode 100644
index 00000000..7de5537d
--- /dev/null
+++ b/.devcontainer/network-simulator/devcontainer.json
@@ -0,0 +1,34 @@
+// For format details, see https://aka.ms/devcontainer.json
+{
+    "name": "Network Simulator Dev Container",
+    "dockerComposeFile": [
+        "../docker-compose.yml"
+    ],
+    "service": "app",
+    "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+    "mounts": [
+        "source=${localWorkspaceFolder}/../conda,target=/workspaces/conda,type=bind,consistency=cached",
+        "source=${localWorkspaceFolder}/../mamba,target=/workspaces/mamba,type=bind,consistency=cached"
+    ],
+    // Configure tool-specific properties.
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "python.defaultInterpreterPath": "/opt/conda/bin/python",
+                "python.testing.pytestArgs": [
+                    "tests"
+                ],
+                "python.testing.unittestEnabled": false,
+                "python.testing.pytestEnabled": true
+            },
+            "extensions": [
+                "charliermarsh.ruff",
+                "eamodio.gitlens",
+                "ms-toolsai.jupyter",
+                "be5invis.toml"
+            ]
+        }
+    }
+    // Adjust to connect as non-root instead. More info: https://aka.ms/dev-containers-non-root.
+    // "remoteUser": "root",
+}
\ No newline at end of file

From 0e99b04d99e67868917e64abbb6f60d6c0ad8354 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 20 Nov 2025 16:35:41 +0100
Subject: [PATCH 12/18] split apt, conda setup into separate docker layers

---
 .devcontainer/apt_deps.sh                  | 22 ++++++++++++++++
 .devcontainer/conda_deps.sh                | 30 ++++++++++++++++++++++
 .devcontainer/docker-compose.yml           |  6 ++---
 .devcontainer/network-simulator/Dockerfile |  7 ++---
 .devcontainer/post_create.sh               |  1 +
 5 files changed, 60 insertions(+), 6 deletions(-)
 create mode 100644 .devcontainer/apt_deps.sh
 create mode 100644 .devcontainer/conda_deps.sh

diff --git a/.devcontainer/apt_deps.sh b/.devcontainer/apt_deps.sh
new file mode 100644
index 00000000..74522fea
--- /dev/null
+++ b/.devcontainer/apt_deps.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# This script assumes we are running in a Miniconda container where:
+# - /opt/conda is the Miniconda or Miniforge installation directory
+# - https://github.com/conda/conda is mounted at /workspaces/conda
+# - https://github.com/conda/conda-libmamba-solver is mounted at
+#   /workspaces/conda-libmamba-solver
+# - https://github.com/mamba-org/mamba is (optionally) mounted at
+#   /workspaces/mamba
+
+set -euo pipefail
+
+HERE=$(dirname $0)
+BASE_CONDA=${BASE_CONDA:-/opt/conda}
+SRC_CONDA=${SRC_CONDA:-/workspaces/conda}
+SRC_CONDA_LIBMAMBA_SOLVER=${SRC_CONDA_LIBMAMBA_SOLVER:-/workspaces/conda-libmamba-solver}
+
+if which apt-get > /dev/null; then
+    echo "Installing system dependencies"
+    apt-get update
+    DEBIAN_FRONTEND=noninteractive xargs -a "$HERE/apt-deps.txt" apt-get install -y
+fi
diff --git a/.devcontainer/conda_deps.sh b/.devcontainer/conda_deps.sh
new file mode 100644
index 00000000..61c42606
--- /dev/null
+++ b/.devcontainer/conda_deps.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# This script assumes we are running in a Miniconda container where:
+# - /opt/conda is the Miniconda or Miniforge installation directory
+# - https://github.com/conda/conda is mounted at /workspaces/conda
+# - https://github.com/conda/conda-libmamba-solver is mounted at
+#   /workspaces/conda-libmamba-solver
+# - https://github.com/mamba-org/mamba is (optionally) mounted at
+#   /workspaces/mamba
+
+set -euo pipefail
+
+if [ ! -f "$SRC_CONDA/pyproject.toml" ]; then
+    echo "https://github.com/conda/conda not found! Please clone or mount to $SRC_CONDA"
+    exit 1
+fi
+
+# Clear history to avoid unneeded conflicts
+echo "Clearing base history..."
+echo '' > "$BASE_CONDA/conda-meta/history"
+
+echo "Installing dev & test dependencies..."
+"$BASE_CONDA/bin/conda" install -n base --yes --quiet \
+    --file="$SRC_CONDA/tests/requirements.txt" \
+    --file="$SRC_CONDA/tests/requirements-ci.txt" \
+    --file="$SRC_CONDA/tests/requirements-Linux.txt" \
+    --file="$SRC_CONDA/tests/requirements-s3.txt" \
+    --file="$SRC_CONDA_LIBMAMBA_SOLVER/dev/requirements.txt" \
+    --file="$SRC_CONDA_LIBMAMBA_SOLVER/tests/requirements.txt"\
+    pre-commit
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
index 4c5ea991..87866d3e 100644
--- a/.devcontainer/docker-compose.yml
+++ b/.devcontainer/docker-compose.yml
@@ -4,9 +4,9 @@ services:
       context: .
       dockerfile: ./network-simulator/Dockerfile
     volumes:
-      - ../../conda:/workspace/conda:cached
-      - ../../mamba:/workspace/mamba:cached
-      - ../../conda-libmamba-solver:/workspace/conda-libmamba-solver
+      - ../../conda:/workspaces/conda:cached
+      - ../../mamba:/workspaces/mamba:cached
+      - ../:/workspaces/conda-libmamba-solver
     cap_add:
       - NET_ADMIN
 
diff --git a/.devcontainer/network-simulator/Dockerfile b/.devcontainer/network-simulator/Dockerfile
index 962e0e74..1af713e9 100644
--- a/.devcontainer/network-simulator/Dockerfile
+++ b/.devcontainer/network-simulator/Dockerfile
@@ -1,4 +1,5 @@
 FROM continuumio/miniconda3:latest
-COPY apt-deps.txt post_create.sh post_start.sh ./
-RUN bash ./post_create.sh
-CMD [ "bash", "post_start.sh" ]
\ No newline at end of file
+COPY apt-deps.txt apt_deps.sh conda_deps.sh post_create.sh post_start.sh ./
+RUN bash ./apt_deps.sh
+RUN bash ./conda_deps.sh
+CMD [ "sleep", "infinity" ]
\ No newline at end of file
diff --git a/.devcontainer/post_create.sh b/.devcontainer/post_create.sh
index b90a1d79..a51a8830 100644
--- a/.devcontainer/post_create.sh
+++ b/.devcontainer/post_create.sh
@@ -21,6 +21,7 @@ if which apt-get > /dev/null; then
     DEBIAN_FRONTEND=noninteractive xargs -a "$HERE/apt-deps.txt" apt-get install -y
 fi
 
+# this would make sense in a separate docker layer:
 
 if [ ! -f "$SRC_CONDA/pyproject.toml" ]; then
     echo "https://github.com/conda/conda not found! Please clone or mount to $SRC_CONDA"

From bd5d70e11d53a75d089334ebfab570b1c2829379 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 20 Nov 2025 16:38:08 +0100
Subject: [PATCH 13/18] define variables in shell script

---
 .devcontainer/conda_deps.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.devcontainer/conda_deps.sh b/.devcontainer/conda_deps.sh
index 61c42606..c569c5dd 100644
--- a/.devcontainer/conda_deps.sh
+++ b/.devcontainer/conda_deps.sh
@@ -10,6 +10,11 @@
 
 set -euo pipefail
 
+HERE=$(dirname $0)
+BASE_CONDA=${BASE_CONDA:-/opt/conda}
+SRC_CONDA=${SRC_CONDA:-/workspaces/conda}
+SRC_CONDA_LIBMAMBA_SOLVER=${SRC_CONDA_LIBMAMBA_SOLVER:-/workspaces/conda-libmamba-solver}
+
 if [ ! -f "$SRC_CONDA/pyproject.toml" ]; then
     echo "https://github.com/conda/conda not found! Please clone or mount to $SRC_CONDA"
     exit 1

From 845150dfcaf8f9836b98c4a0fae5653f3c31fb7a Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 20 Nov 2025 15:42:33 +0000
Subject: [PATCH 14/18] note script mounts prereq

---
 .devcontainer/network-simulator/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.devcontainer/network-simulator/Dockerfile b/.devcontainer/network-simulator/Dockerfile
index 1af713e9..31393a43 100644
--- a/.devcontainer/network-simulator/Dockerfile
+++ b/.devcontainer/network-simulator/Dockerfile
@@ -1,5 +1,5 @@
 FROM continuumio/miniconda3:latest
 COPY apt-deps.txt apt_deps.sh conda_deps.sh post_create.sh post_start.sh ./
 RUN bash ./apt_deps.sh
-RUN bash ./conda_deps.sh
-CMD [ "sleep", "infinity" ]
\ No newline at end of file
+# conda_deps.sh wants mounts, which don't exist until the container runs.
+CMD [ "bash", "./conda_deps.sh" ]
\ No newline at end of file

From c1fcf13b6f7bcfacf843d82724f734414ccf9e05 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Mon, 24 Nov 2025 11:23:09 -0500
Subject: [PATCH 15/18] restore configurable shard traversal setting

---
 conda_libmamba_solver/index.py  | 4 +++-
 conda_libmamba_solver/plugin.py | 6 ++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py
index f457c585..9f03ecc2 100644
--- a/conda_libmamba_solver/index.py
+++ b/conda_libmamba_solver/index.py
@@ -402,7 +402,9 @@ def _build_repodata_subset(
         self, root_packages: tuple[str, ...], urls_to_channel: dict[str, Channel]
     ) -> dict[str, ShardBase]:
         # split into a separate method for tests
-        return build_repodata_subset(root_packages, urls_to_channel)
+        return build_repodata_subset(
+            root_packages, urls_to_channel, algorithm=_sharded_repodata_strategy()
+        )
 
     def _load_channel_repo_info_json(
         self, urls_to_channel: dict[str, Channel], try_solv: bool
diff --git a/conda_libmamba_solver/plugin.py b/conda_libmamba_solver/plugin.py
index 61dd314e..5ae7afd3 100644
--- a/conda_libmamba_solver/plugin.py
+++ b/conda_libmamba_solver/plugin.py
@@ -44,3 +44,9 @@ def conda_settings():
         description="Enable use of sharded repodata when available.",
         parameter=PrimitiveParameter(False, element_type=bool),
     )
+
+    yield CondaSetting(
+        name="sharded_repodata_strategy",
+        description="Strategy to collect sharded repodata.",
+        parameter=PrimitiveParameter("auto", element_type=str),
+    )

From db8735d003136157ddeb08fd46852832975e22ce Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Mon, 24 Nov 2025 15:35:52 -0500
Subject: [PATCH 16/18] test subsetting non-shards conda-forge repodata.json

---
 tests/test_index.py         | 33 ++++++++++++++++++++++++---------
 tests/test_shards_subset.py | 11 ++++++++---
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/tests/test_index.py b/tests/test_index.py
index 50cc4156..839f7454 100644
--- a/tests/test_index.py
+++ b/tests/test_index.py
@@ -19,6 +19,7 @@
 from conda_libmamba_solver.index import LibMambaIndexHelper, _is_sharded_repodata_enabled
 from conda_libmamba_solver.shards_subset import build_repodata_subset
 from conda_libmamba_solver.state import SolverInputState
+from tests.test_shards import _timer
 
 if TYPE_CHECKING:
     import os
@@ -185,8 +186,16 @@ def index():
         ("shard", ("vaex",)),
         ("repodata", ("vaex",)),
         ("main", ()),
+        ("shard-unavailable", ("vaex",)),
+    ],
+    ids=[
+        "shard-small",
+        "shard-medium",
+        "shard-large",
+        "noshard",
+        "main",
+        "shard-unavailable",
     ],
-    ids=["shard-small", "shard-medium", "shard-large", "noshard", "main"],
 )
 def test_load_channel_repo_info_shards_parse_only(
     load_type: str,
@@ -203,11 +212,17 @@ def test_load_channel_repo_info_shards_parse_only(
     TODO: This test should eventually switch to just using conda-forge when that channel
           supports shards and not the `conda-forge-sharded` channel.
     """
-    # defaults is more than one channel but I don't think it is too significant for the test.
-    load_channel = "main" if load_type == "main" else "conda-forge-sharded"
+    # defaults is more than one channel but I don't think that matters here.
+    load_channel = {"main": "main", "shard-unavailable": "conda-forge"}.get(
+        load_type, "conda-forge-sharded"
+    )
 
-    monkeypatch.setattr(context.plugins, "use_sharded_repodata", load_type == "shard")
-    assert _is_sharded_repodata_enabled() == (load_type == "shard")
+    monkeypatch.setattr(
+        context.plugins,
+        "use_sharded_repodata",
+        load_type in ("shard", "shard-unavailable"),
+    )
+    assert _is_sharded_repodata_enabled() == (load_type in ("shard", "shard-unavailable"))
 
     in_state = SolverInputState(str(tmp_path / "env"), requested=requested)
 
@@ -218,10 +233,10 @@ def test_load_channel_repo_info_shards_parse_only(
 
     repodata_jsons = {}
     channel_data = {}
-    root_packages_ = ()
-    if load_type == "shard":  # shards
-        root_packages_ = (*in_state.installed.keys(), *in_state.requested)
-        channel_data = build_repodata_subset(root_packages_, urls_to_channel_)
+    root_packages_ = (*in_state.installed.keys(), *in_state.requested)
+    if load_type in ("shard", "shard-unavailable"):  # shards
+        with _timer("subset monolithic repodata.json"):
+            channel_data = build_repodata_subset(root_packages_, urls_to_channel_)
 
     else:  # no shards
         # this is an inefficient way to fetch repodata.json in the format
diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 77e32dcb..a4bea70a 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -785,6 +785,11 @@ def test_repodata_subset_misc():
     ) == (True, True, False)
 
 
+# endregion
+
+# region simulator
+
+
 @dataclass
 class ShardFetchResult:
     node_id: NodeId
@@ -965,8 +970,8 @@ def repodata_index_transfer_size():
 # Different latency/bandwidth scenarios, loosely based on Firefox debug console.
 NETWORK_SCENARIOS = {
     "4MBPS": {"bandwidth_mbps": 4, "latency_ms": 100},  # want a high latency
-    "11MBPS": {"bandwidth_mbps": 11, "latency_ms": 100},  # want a high latency
-    "30MBPS": {"bandwidth_mbps": 30, "latency_ms": 30},  # medium
+    "10MBPS": {"bandwidth_mbps": 10, "latency_ms": 100},  # want a high latency
+    # "30MBPS": {"bandwidth_mbps": 30, "latency_ms": 30},  # medium
     "HALFGIG": {
         "bandwidth_mbps": 500,
         "latency_ms": 20,
@@ -1037,7 +1042,7 @@ def simulate():
             )
             repodata_mib = repodata_index_transfer_size["repodata.json.zst"] / (2**20)
             print(
-                f"Shards {shards_index_mib:0.2f}MiB+{shards_mib:0.2f}MiB vs repodata.json.zst {repodata_mib:0.2f}MiB for {packages}, {len(subset.nodes)} nodes"
+                f"Shards index {shards_index_mib:0.2f}MiB+shards {shards_mib:0.2f}MiB vs repodata.json.zst {repodata_mib:0.2f}MiB for {packages}, {len(subset.nodes)} nodes"
             )
 
     # Missing here is the "load into LibMambaIndexHelper" step. When bandwidth

From 918c8974600f1f7d440fba67481c3dafc93bdb6b Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Thu, 11 Dec 2025 10:08:03 -0500
Subject: [PATCH 17/18] revert .devcontainer changes

---
 .devcontainer/apt_deps.sh                     | 22 ------------
 .devcontainer/conda_deps.sh                   | 35 -------------------
 .devcontainer/docker-compose.yml              | 22 ------------
 .devcontainer/network-simulator/Dockerfile    |  5 ---
 .../network-simulator/devcontainer.json       | 34 ------------------
 .devcontainer/post_create.sh                  |  1 -
 6 files changed, 119 deletions(-)
 delete mode 100644 .devcontainer/apt_deps.sh
 delete mode 100644 .devcontainer/conda_deps.sh
 delete mode 100644 .devcontainer/docker-compose.yml
 delete mode 100644 .devcontainer/network-simulator/Dockerfile
 delete mode 100644 .devcontainer/network-simulator/devcontainer.json

diff --git a/.devcontainer/apt_deps.sh b/.devcontainer/apt_deps.sh
deleted file mode 100644
index 74522fea..00000000
--- a/.devcontainer/apt_deps.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# This script assumes we are running in a Miniconda container where:
-# - /opt/conda is the Miniconda or Miniforge installation directory
-# - https://github.com/conda/conda is mounted at /workspaces/conda
-# - https://github.com/conda/conda-libmamba-solver is mounted at
-#   /workspaces/conda-libmamba-solver
-# - https://github.com/mamba-org/mamba is (optionally) mounted at
-#   /workspaces/mamba
-
-set -euo pipefail
-
-HERE=$(dirname $0)
-BASE_CONDA=${BASE_CONDA:-/opt/conda}
-SRC_CONDA=${SRC_CONDA:-/workspaces/conda}
-SRC_CONDA_LIBMAMBA_SOLVER=${SRC_CONDA_LIBMAMBA_SOLVER:-/workspaces/conda-libmamba-solver}
-
-if which apt-get > /dev/null; then
-    echo "Installing system dependencies"
-    apt-get update
-    DEBIAN_FRONTEND=noninteractive xargs -a "$HERE/apt-deps.txt" apt-get install -y
-fi
diff --git a/.devcontainer/conda_deps.sh b/.devcontainer/conda_deps.sh
deleted file mode 100644
index c569c5dd..00000000
--- a/.devcontainer/conda_deps.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-
-# This script assumes we are running in a Miniconda container where:
-# - /opt/conda is the Miniconda or Miniforge installation directory
-# - https://github.com/conda/conda is mounted at /workspaces/conda
-# - https://github.com/conda/conda-libmamba-solver is mounted at
-#   /workspaces/conda-libmamba-solver
-# - https://github.com/mamba-org/mamba is (optionally) mounted at
-#   /workspaces/mamba
-
-set -euo pipefail
-
-HERE=$(dirname $0)
-BASE_CONDA=${BASE_CONDA:-/opt/conda}
-SRC_CONDA=${SRC_CONDA:-/workspaces/conda}
-SRC_CONDA_LIBMAMBA_SOLVER=${SRC_CONDA_LIBMAMBA_SOLVER:-/workspaces/conda-libmamba-solver}
-
-if [ ! -f "$SRC_CONDA/pyproject.toml" ]; then
-    echo "https://github.com/conda/conda not found! Please clone or mount to $SRC_CONDA"
-    exit 1
-fi
-
-# Clear history to avoid unneeded conflicts
-echo "Clearing base history..."
-echo '' > "$BASE_CONDA/conda-meta/history"
-
-echo "Installing dev & test dependencies..."
-"$BASE_CONDA/bin/conda" install -n base --yes --quiet \
-    --file="$SRC_CONDA/tests/requirements.txt" \
-    --file="$SRC_CONDA/tests/requirements-ci.txt" \
-    --file="$SRC_CONDA/tests/requirements-Linux.txt" \
-    --file="$SRC_CONDA/tests/requirements-s3.txt" \
-    --file="$SRC_CONDA_LIBMAMBA_SOLVER/dev/requirements.txt" \
-    --file="$SRC_CONDA_LIBMAMBA_SOLVER/tests/requirements.txt"\
-    pre-commit
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
deleted file mode 100644
index 87866d3e..00000000
--- a/.devcontainer/docker-compose.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-services:
-  app:
-    build:
-      context: .
-      dockerfile: ./network-simulator/Dockerfile
-    volumes:
-      - ../../conda:/workspaces/conda:cached
-      - ../../mamba:/workspaces/mamba:cached
-      - ../:/workspaces/conda-libmamba-solver
-    cap_add:
-      - NET_ADMIN
-
-    # Overrides default command so things don't shut down after the process ends.
-    command: sleep infinity
-
-  repo:
-    image: nginx
-    restart: unless-stopped
-    volumes:
-      - .:/usr/share/nginx/html:ro # instead, the test shards directory
-    cap_add:
-      - NET_ADMIN
diff --git a/.devcontainer/network-simulator/Dockerfile b/.devcontainer/network-simulator/Dockerfile
deleted file mode 100644
index 31393a43..00000000
--- a/.devcontainer/network-simulator/Dockerfile
+++ /dev/null
@@ -1,5 +0,0 @@
-FROM continuumio/miniconda3:latest
-COPY apt-deps.txt apt_deps.sh conda_deps.sh post_create.sh post_start.sh ./
-RUN bash ./apt_deps.sh
-# conda_deps.sh wants mounts, which don't exist until the container runs.
-CMD [ "bash", "./conda_deps.sh" ]
\ No newline at end of file
diff --git a/.devcontainer/network-simulator/devcontainer.json b/.devcontainer/network-simulator/devcontainer.json
deleted file mode 100644
index 7de5537d..00000000
--- a/.devcontainer/network-simulator/devcontainer.json
+++ /dev/null
@@ -1,34 +0,0 @@
-// For format details, see https://aka.ms/devcontainer.json
-{
-    "name": "Network Simulator Dev Container",
-    "dockerComposeFile": [
-        "../docker-compose.yml"
-    ],
-    "service": "app",
-    "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-    "mounts": [
-        "source=${localWorkspaceFolder}/../conda,target=/workspaces/conda,type=bind,consistency=cached",
-        "source=${localWorkspaceFolder}/../mamba,target=/workspaces/mamba,type=bind,consistency=cached"
-    ],
-    // Configure tool-specific properties.
-    "customizations": {
-        "vscode": {
-            "settings": {
-                "python.defaultInterpreterPath": "/opt/conda/bin/python",
-                "python.testing.pytestArgs": [
-                    "tests"
-                ],
-                "python.testing.unittestEnabled": false,
-                "python.testing.pytestEnabled": true
-            },
-            "extensions": [
-                "charliermarsh.ruff",
-                "eamodio.gitlens",
-                "ms-toolsai.jupyter",
-                "be5invis.toml"
-            ]
-        }
-    }
-    // Adjust to connect as non-root instead. More info: https://aka.ms/dev-containers-non-root.
-    // "remoteUser": "root",
-}
\ No newline at end of file
diff --git a/.devcontainer/post_create.sh b/.devcontainer/post_create.sh
index a51a8830..b90a1d79 100644
--- a/.devcontainer/post_create.sh
+++ b/.devcontainer/post_create.sh
@@ -21,7 +21,6 @@ if which apt-get > /dev/null; then
     DEBIAN_FRONTEND=noninteractive xargs -a "$HERE/apt-deps.txt" apt-get install -y
 fi
 
-# this would make sense in a separate docker layer:
 
 if [ ! -f "$SRC_CONDA/pyproject.toml" ]; then
     echo "https://github.com/conda/conda not found! Please clone or mount to $SRC_CONDA"

From cd522dae12a1c5c6c9299afa42cfe7b6878bfae4 Mon Sep 17 00:00:00 2001
From: Daniel Holth <dholth@anaconda.com>
Date: Sat, 4 Apr 2026 16:04:21 -0400
Subject: [PATCH 18/18] test pycurl transport

---
 conda_libmamba_solver/shards_subset.py       | 21 ++++++-
 conda_libmamba_solver/shards_subset_http2.py | 62 +++++++++++++++++++-
 tests/test_shards_subset.py                  |  9 ++-
 3 files changed, 88 insertions(+), 4 deletions(-)

diff --git a/conda_libmamba_solver/shards_subset.py b/conda_libmamba_solver/shards_subset.py
index 909fd789..556c16ae 100644
--- a/conda_libmamba_solver/shards_subset.py
+++ b/conda_libmamba_solver/shards_subset.py
@@ -294,6 +294,24 @@ def reachable_httpx(self, root_packages):
 
         return self.reachable_pipelined(root_packages, network_fetch_thread_httpx)
 
+    def reachable_pycurl(self, root_packages):
+        """
+        httpx + pycurl transport for httpx (curl sockets in event loop async implementation)
+        """
+        from .shards_subset_http2 import network_fetch_thread_httpx_pycurl
+
+        return self.reachable_pipelined(root_packages, network_fetch_thread_httpx_pycurl)
+
+    def reachable_pycurl2(self, root_packages):
+        """
+        httpx + pycurl transport for httpx (delegate to thread async implementation)
+        """
+        from .shards_subset_http2 import network_fetch_thread_httpx_pycurl2
+
+        return self.reachable_pipelined(root_packages, network_fetch_thread_httpx_pycurl2)
+
+    # could try sync + threads http/2 also...
+
     def reachable_pipelined(self, root_packages, network_worker=None):
         """
         Fetch all packages reachable from `root_packages`' by following
@@ -308,7 +326,8 @@ def reachable_pipelined(self, root_packages, network_worker=None):
         # empty shards.
         if context.offline:
             network_worker = offline_nofetch_thread
-        else:
+
+        if network_worker is None:
             network_worker = network_fetch_thread
 
         # Ignore cache on shards object, use our own. Necessary if there are no
diff --git a/conda_libmamba_solver/shards_subset_http2.py b/conda_libmamba_solver/shards_subset_http2.py
index 93cb663e..a5bc34c6 100644
--- a/conda_libmamba_solver/shards_subset_http2.py
+++ b/conda_libmamba_solver/shards_subset_http2.py
@@ -15,6 +15,7 @@
 
 import httpx
 import msgpack
+import pycurltx
 import zstandard
 
 from .shards import ShardBase, Shards
@@ -30,6 +31,7 @@ async def _network_fetch_loop_httpx(
     shard_out_queue: Queue[list[tuple[NodeId, ShardDict] | Exception] | None],
     cache: ShardCache,
     shardlikes: list[ShardBase],
+    transport=None,
 ):
     cache = cache.copy()
     dctx = zstandard.ZstdDecompressor()
@@ -52,6 +54,7 @@ async def submit(client, node_id):
 
     async def get_work():
         with concurrent.futures.ThreadPoolExecutor() as thread_pool:
+            # this may be faster than asyncio.to_thread()
             while True:
                 batch = await asyncio.get_running_loop().run_in_executor(thread_pool, in_queue.get)
                 if batch is None:
@@ -72,7 +75,8 @@ def handle_result(task: asyncio.Task):
         cache.insert(AnnotatedRawShard(url, node_id.package, data))
         shard_out_queue.put([(node_id, shard)])
 
-    async with httpx.AsyncClient(http2=True) as client:
+    # if transport is not None, http2 comes from pycurl
+    async with httpx.AsyncClient(http2=(transport is None), transport=transport) as client:
         results: list[tuple[NodeId, ShardDict] | Exception] = []
 
         tasks = set()
@@ -108,3 +112,59 @@ def network_fetch_thread_httpx(
         if not shard.url.startswith("http"):
             raise ValueError(f"Unsupported shard in http/2 fetch thread: {shard.url}")
     asyncio.run(_network_fetch_loop_httpx(in_queue, shard_out_queue, cache, shardlikes))
+
+
+@exception_to_queue
+def network_fetch_thread_httpx_pycurl(
+    in_queue: Queue[list[NodeId] | None],
+    shard_out_queue: Queue[list[tuple[NodeId, ShardDict] | Exception] | None],
+    cache: ShardCache,
+    shardlikes: list[ShardBase],
+):
+    """
+    in_queue contains urls to fetch over the network.
+    While the in_queue has not received a sentinel None, empty everything from
+    the queue. Fetch all of them over the network. Fetched shards go to
+    shard_out_queue.
+    Unhandled exceptions also go to shard_out_queue, and exit this thread.
+    """
+    for shard in shardlikes:
+        if not shard.url.startswith("http"):
+            raise ValueError(f"Unsupported shard in http/2 fetch thread: {shard.url}")
+    asyncio.run(
+        _network_fetch_loop_httpx(
+            in_queue,
+            shard_out_queue,
+            cache,
+            shardlikes,
+            transport=pycurltx.PyCurlAsyncMultiSocketTransport(max_connections=100),
+        )
+    )
+
+
+@exception_to_queue
+def network_fetch_thread_httpx_pycurl2(
+    in_queue: Queue[list[NodeId] | None],
+    shard_out_queue: Queue[list[tuple[NodeId, ShardDict] | Exception] | None],
+    cache: ShardCache,
+    shardlikes: list[ShardBase],
+):
+    """
+    in_queue contains urls to fetch over the network.
+    While the in_queue has not received a sentinel None, empty everything from
+    the queue. Fetch all of them over the network. Fetched shards go to
+    shard_out_queue.
+    Unhandled exceptions also go to shard_out_queue, and exit this thread.
+    """
+    for shard in shardlikes:
+        if not shard.url.startswith("http"):
+            raise ValueError(f"Unsupported shard in http/2 fetch thread: {shard.url}")
+    asyncio.run(
+        _network_fetch_loop_httpx(
+            in_queue,
+            shard_out_queue,
+            cache,
+            shardlikes,
+            transport=pycurltx.PyCurlAsyncTransport(timeout=10.0),
+        )
+    )
diff --git a/tests/test_shards_subset.py b/tests/test_shards_subset.py
index 445f3ea8..926682ff 100644
--- a/tests/test_shards_subset.py
+++ b/tests/test_shards_subset.py
@@ -112,7 +112,12 @@
     TESTING_SCENARIOS = [
         scenario
         for scenario in TESTING_SCENARIOS
-        if scenario["name"] in ("python", "devops_automation")
+        if scenario["name"]
+        in (
+            "python",
+            "devops_automation",
+            "vaex",
+        )
     ]
 
 
@@ -160,7 +165,7 @@ def repodata_subset_size(channel_data):
 
 @pytest.mark.skipif(not codspeed_supported(), reason="pytest-codspeed-version-4")
 @pytest.mark.parametrize("cache_state", ("cold", "warm"))
-@pytest.mark.parametrize("algorithm", ("bfs", "pipelined", "httpx"))
+@pytest.mark.parametrize("algorithm", ("pipelined", "httpx", "pycurl", "pycurl2"))
 @pytest.mark.parametrize(
     "scenario",
     TESTING_SCENARIOS,