e2e/qa: run testnet shred-subscription settlement on solana devnet

config/constants.go

@@ -19,18 +19,20 @@ const (
 	MainnetUSDCMint                     = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"
 
 	// Testnet constants.
-	TestnetLedgerPublicRPCURL            = "https://doublezerolocalnet.rpcpool.com/8a4fd3f4-0977-449f-88c7-63d4b0f10f16"
-	TestnetServiceabilityProgramID       = "DZtnuQ839pSaDMFG5q1ad2V95G82S5EC4RrB3Ndw2Heb"
-	TestnetTelemetryProgramID            = "3KogTMmVxc5eUHtjZnwm136H5P8tvPwVu4ufbGPvM7p1"
-	TestnetInternetLatencyCollectorPK    = "HWGQSTmXWMB85NY2vFLhM1nGpXA8f4VCARRyeGNbqDF1"
-	TestnetDeviceLocalASN                = 65342
-	TestnetTwoZOracleURL                 = "https://sol-2z-oracle-api-v1.testnet.doublezero.xyz"
-	TestnetSolanaRPC                     = "https://api.testnet.solana.com"
-	TestnetTelemetryFlowIngestURL        = "http://telemetry-flow-in.testnet.doublezero.xyz"
-	TestnetTelemetryStateIngestURL       = "https://telemetry-state-in-testnet.doublezero.xyz"
-	TestnetGeolocationProgramID          = "3AG2BCA7gAm47Q6xZzPQcUUYvnBjxAvPKnPz919cxHF4"
-	TestnetShredSubscriptionProgramID    = "dzshrr3yL57SB13sJPYHYo3TV8Bo1i1FxkyrZr3bKNE"
-	TestnetUSDCMint                      = "uSDZq2RMuxrEf7gqgDjR8wJCtCyaDAQk2e5jLAaoeeM"
+	TestnetLedgerPublicRPCURL         = "https://doublezerolocalnet.rpcpool.com/8a4fd3f4-0977-449f-88c7-63d4b0f10f16"
+	TestnetServiceabilityProgramID    = "DZtnuQ839pSaDMFG5q1ad2V95G82S5EC4RrB3Ndw2Heb"
+	TestnetTelemetryProgramID         = "3KogTMmVxc5eUHtjZnwm136H5P8tvPwVu4ufbGPvM7p1"
+	TestnetInternetLatencyCollectorPK = "HWGQSTmXWMB85NY2vFLhM1nGpXA8f4VCARRyeGNbqDF1"
+	TestnetDeviceLocalASN             = 65342
+	TestnetTwoZOracleURL              = "https://sol-2z-oracle-api-v1.testnet.doublezero.xyz"
+	TestnetSolanaRPC                  = "https://api.testnet.solana.com"
+	TestnetTelemetryFlowIngestURL     = "http://telemetry-flow-in.testnet.doublezero.xyz"
+	TestnetTelemetryStateIngestURL    = "https://telemetry-state-in-testnet.doublezero.xyz"
+	TestnetGeolocationProgramID       = "3AG2BCA7gAm47Q6xZzPQcUUYvnBjxAvPKnPz919cxHF4"
+	TestnetShredSubscriptionProgramID = "dzshrr3yL57SB13sJPYHYo3TV8Bo1i1FxkyrZr3bKNE"
+	// Testnet shred-subscription runs on Solana devnet, so settlement uses the
+	// Solana devnet USDC mint (not the DZ ledger testnet mint).
+	TestnetUSDCMint                      = "4zMMC9srt5Ri5X14GAgXhaHii3GnPAEERYPJgZJDncDU"
 	TestnetTelemetryGNMITunnelServerAddr = "gnmic-testnet.doublezero.xyz:443"
 
 	// Devnet constants.

e2e/internal/qa/client.go

@@ -26,10 +26,18 @@ import (
 )
 
 const (
-	disconnectTimeout                = 150 * time.Second
-	waitForStatusUpTimeout           = 90 * time.Second
-	waitForStatusDisconnectedTimeout = 90 * time.Second
-	waitForUserDeletionTimeout       = 90 * time.Second
+	disconnectTimeout      = 150 * time.Second
+	waitForStatusUpTimeout = 90 * time.Second
+	// Multicast (shred-subscription) tunnel up/down is driven by the oracle's
+	// ~60s reconcile loop (subscribe once the seat is active; unsubscribe after
+	// withdrawal), NOT on seat-allocation ack / withdrawal — observed latency up
+	// to ~3.5 min on devnet. Generous, dedicated timeouts so up/down don't race
+	// the cadence. TEMPORARY debug padding to surface downstream failures; the
+	// real fix is event-driven subscribe/unsubscribe in the oracle.
+	waitForMulticastStatusUpTimeout           = 360 * time.Second
+	waitForMulticastStatusDisconnectedTimeout = 360 * time.Second
+	waitForStatusDisconnectedTimeout          = 90 * time.Second
+	waitForUserDeletionTimeout                = 90 * time.Second
 
 	// NOTE: This needs to be longer than 1m since BGP can sometimes throttle activity for that
 	// amount of time if too much is happening consecutively for the same peers.
@@ -125,7 +133,8 @@ type Client struct {
 
 	// Settlement config passed to doublezero-solana shreds commands.
 	// SolanaRPCURL is the Solana RPC endpoint for settlement transactions (--url).
-	// On testnet this is the DZ ledger URL; on mainnet it's the public Solana RPC.
+	// On testnet this is Solana devnet (via SOLANA_RPC_URL); on mainnet the public
+	// Solana RPC; on devnet the DZ ledger URL.
 	SolanaRPCURL               string
 	ShredSubscriptionProgramID string
 	DZLedgerURL                string
@@ -155,11 +164,14 @@ func NewClient(ctx context.Context, log *slog.Logger, hostname string, port int,
 
 	serviceabilityClient := serviceability.New(rpc.New(networkConfig.LedgerPublicRPCURL), networkConfig.ServiceabilityProgramID)
 
-	// Settlement transactions on testnet/devnet use the DZ ledger RPC endpoint
-	// (which hosts the settlement programs). Mainnet and localnet use the
+	// Settlement transactions on devnet use the DZ ledger RPC endpoint (which
+	// hosts the settlement programs there). Testnet reads/writes the
+	// shred-subscription program on Solana devnet via networkConfig.SolanaRPCURL
+	// (the SOLANA_RPC_URL override, a Solana devnet RPC endpoint in CI;
+	// defaults to the public Solana endpoint). Mainnet and localnet use the
 	// standard Solana RPC.
 	solanaRPCURL := networkConfig.SolanaRPCURL
-	if networkConfig.Moniker == config.EnvTestnet || networkConfig.Moniker == config.EnvDevnet {
+	if networkConfig.Moniker == config.EnvDevnet {
 		solanaRPCURL = networkConfig.LedgerPublicRPCURL
 	}
 
@@ -393,19 +405,19 @@ func (c *Client) WaitForStatusUp(ctx context.Context) error {
 // its session is up. Prefer this over WaitForStatusUp in multi-tunnel contexts
 // where other tunnel types may already be present.
 func (c *Client) WaitForUnicastStatusUp(ctx context.Context) error {
-	return c.waitForUserTypeStatusUp(ctx, "IBRL", FindIBRLStatus)
+	return c.waitForUserTypeStatusUp(ctx, "IBRL", FindIBRLStatus, waitForStatusUpTimeout)
 }
 
 // WaitForMulticastStatusUp polls until a Multicast status entry exists and
 // its session is up. Prefer this over WaitForStatusUp in multi-tunnel contexts
 // where other tunnel types may already be present.
 func (c *Client) WaitForMulticastStatusUp(ctx context.Context) error {
-	return c.waitForUserTypeStatusUp(ctx, "Multicast", FindMulticastStatus)
+	return c.waitForUserTypeStatusUp(ctx, "Multicast", FindMulticastStatus, waitForMulticastStatusUpTimeout)
 }
 
 // waitForUserTypeStatusUp polls until find returns a non-nil status whose
 // session is up. userType is used only for log context.
-func (c *Client) waitForUserTypeStatusUp(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status) error {
+func (c *Client) waitForUserTypeStatusUp(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status, timeout time.Duration) error {
 	c.log.Debug("Waiting for status to be up", "host", c.Host, "userType", userType)
 	err := poll.Until(ctx, func() (bool, error) {
 		resp, err := c.grpcClient.GetStatus(ctx, &emptypb.Empty{})
@@ -414,7 +426,7 @@ func (c *Client) waitForUserTypeStatusUp(ctx context.Context, userType string, f
 		}
 		s := find(resp.Status)
 		return s != nil && IsStatusUp(s.SessionStatus), nil
-	}, waitForStatusUpTimeout, waitInterval)
+	}, timeout, waitInterval)
 	if err != nil {
 		return fmt.Errorf("failed to wait for %s status to be up on host %s: %w", userType, c.Host, err)
 	}
@@ -503,7 +515,7 @@ func (c *Client) WaitForStatusDisconnected(ctx context.Context) error {
 // Prefer this over WaitForStatusDisconnected in multi-tunnel contexts where
 // other tunnel types (e.g. IBRL) remain up after a multicast seat is withdrawn.
 func (c *Client) WaitForMulticastStatusDisconnected(ctx context.Context) error {
-	return c.waitForUserTypeStatusDisconnected(ctx, "Multicast", FindMulticastStatus)
+	return c.waitForUserTypeStatusDisconnected(ctx, "Multicast", FindMulticastStatus, waitForMulticastStatusDisconnectedTimeout)
 }
 
 // WaitForIBRLStatusDisconnected polls until no IBRL (or IBRLWithAllocatedIP)
@@ -537,7 +549,7 @@ func (c *Client) WaitForIBRLStatusDisconnected(ctx context.Context) error {
 
 // waitForUserTypeStatusDisconnected polls until find returns nil or a status
 // whose session is disconnected. userType is used only for log context.
-func (c *Client) waitForUserTypeStatusDisconnected(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status) error {
+func (c *Client) waitForUserTypeStatusDisconnected(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status, timeout time.Duration) error {
 	c.log.Debug("Waiting for status to be disconnected", "host", c.Host, "userType", userType)
 	err := poll.Until(ctx, func() (bool, error) {
 		resp, err := c.grpcClient.GetStatus(ctx, &emptypb.Empty{})
@@ -546,7 +558,7 @@ func (c *Client) waitForUserTypeStatusDisconnected(ctx context.Context, userType
 		}
 		s := find(resp.Status)
 		return s == nil || s.SessionStatus == UserStatusDisconnected, nil
-	}, waitForStatusDisconnectedTimeout, waitInterval)
+	}, timeout, waitInterval)
 	if err != nil {
 		return fmt.Errorf("failed to wait for %s status to be disconnected on host %s: %w", userType, c.Host, err)
 	}

e2e/internal/qa/client_settlement.go

@@ -106,11 +106,13 @@ func (c *Client) ClosestDevice(ctx context.Context) (*Device, error) {
 	return device, nil
 }
 
-// FeedSeatPrice calls the FeedSeatPrice RPC to query device seat prices.
-// This is an idempotent read, so on RPC failure it fails over to the next
-// endpoint and retries.
-func (c *Client) FeedSeatPrice(ctx context.Context) ([]*pb.DevicePrice, error) {
-	c.log.Debug("Querying seat prices", "host", c.Host)
+// FeedSeatPrice calls the FeedSeatPrice RPC to query seat pricing for a single
+// device (by pubkey). Querying by pubkey avoids device-code resolution, which
+// the CLI refuses when it can't classify the cluster (e.g. a private Solana
+// devnet RPC URL). This is an idempotent read, so on RPC failure it fails over
+// to the next endpoint and retries.
+func (c *Client) FeedSeatPrice(ctx context.Context, devicePubkey string) ([]*pb.DevicePrice, error) {
+	c.log.Debug("Querying seat prices", "host", c.Host, "device", devicePubkey)
 	var prices []*pb.DevicePrice
 	err := c.withReadFailover(func(rpcURL string) error {
 		resp, err := c.grpcClient.FeedSeatPrice(ctx, &pb.FeedSeatPriceRequest{
@@ -119,6 +121,7 @@ func (c *Client) FeedSeatPrice(ctx context.Context) ([]*pb.DevicePrice, error) {
 			UsdcMint:                   c.USDCMint,
 			Keypair:                    c.Keypair,
 			ShredSubscriptionProgramId: c.ShredSubscriptionProgramID,
+			DevicePubkey:               devicePubkey,
 		})
 		if err != nil {
 			return err

e2e/internal/rpc/agent.go

@@ -362,6 +362,13 @@ func (q *QAAgent) FeedSeatPrice(ctx context.Context, req *pb.FeedSeatPriceReques
 	if req.GetSolanaRpcUrl() != "" {
 		args = append(args, "--url", req.GetSolanaRpcUrl())
 	}
+	// Query a single device by pubkey rather than listing all. The list path
+	// resolves device codes via serviceability, which the CLI refuses when it
+	// can't classify the cluster (e.g. a private Solana devnet RPC URL, seen as
+	// localnet); passing --device sidesteps code resolution entirely.
+	if req.GetDevicePubkey() != "" {
+		args = append(args, "--device", req.GetDevicePubkey())
+	}
 
 	cmdCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()

e2e/proto/qa/agent.proto

@@ -238,6 +238,7 @@ message FeedSeatPriceRequest {
   string usdc_mint = 3;
   string keypair = 4;
   string shred_subscription_program_id = 5;
+  string device_pubkey = 6;
 }
 
 message DevicePrice {

e2e/qa_multicast_settlement_test.go

@@ -130,12 +130,14 @@ func TestQA_MulticastSettlement(t *testing.T) {
 	}
 
 	if !t.Run("query_seat_price", func(t *testing.T) {
-		prices, err := client.FeedSeatPrice(ctx)
+		prices, err := client.FeedSeatPrice(ctx, device.PubKey)
 		require.NoError(t, err, "failed to get seat prices")
 
+		// Match by pubkey, not code: querying by --device skips code resolution,
+		// so the returned rows may not carry a device_code.
 		var price *pb.DevicePrice
 		for _, p := range prices {
-			if p.DeviceCode == device.Code {
+			if p.DevicePubkey == device.PubKey {
 				price = p
 				break
 			}